Files
orbit/internal/storage/mysql/worker_results.go

133 lines
5.3 KiB
Go

package mysql
import (
"context"
"database/sql"
"fmt"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.Binding, message core.ID, in worker.FinishInput) (worker.CommandResult, error) {
var out worker.CommandResult
digest, err := requestDigest(struct {
Identity any
Outcome string
Result core.JSON
ErrorCode string
}{attemptIdentity(in.Ref), in.Outcome, core.JSON(in.Result), in.ErrorCode})
if err != nil {
return out, err
}
err = s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
rows, err := s.lockExecution(ctx, tx, token, b, p, in.Ref, false)
if err != nil {
return err
}
cached, err := readReceipt(ctx, tx, token.NamespaceID, in.Ref.AttemptID, message, "finish", digest)
if err != nil {
return err
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
status := leaseStatus(rows, b, now)
if cached != nil {
out = worker.CommandResult{Receipt: *cached, Lease: status, Deduplicated: true}
return nil
}
task, a := rows.task, rows.attempt
receipt := worker.Receipt{Operation: "finish", TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: a.LeaseUntil, Disposition: "APPLIED"}
if a.ResultDigest != nil && *a.ResultDigest == digest {
// 新消息 ID 重报已保存的同一结果,也不能重复事件、许可释放或状态推进。
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
return err
}
out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true}
return nil
}
reason := ""
if a.ResultDigest != nil {
reason = "RESULT_CONFLICT"
} else if a.AuthorizedAt == nil {
reason = "NOT_STARTED"
} else if !status.Active || a.State != pulse.AttemptRunning {
reason = "LEASE_LOST"
}
if reason != "" {
if err := saveEvidence(ctx, tx, token.NamespaceID, message, in, digest, reason); err != nil {
return err
}
receipt.Disposition = "EVIDENCE_ONLY"
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
return err
}
out = worker.CommandResult{Receipt: receipt, Lease: status}
return nil
}
attemptState, taskState := pulse.AttemptState(in.Outcome), pulse.TaskState(in.Outcome)
var completed *time.Time = &now
if in.Outcome == "UNKNOWN" {
taskState = pulse.TaskNeedsReview
completed = nil
}
changed, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state=?,ended_at=?,error_code=NULLIF(?,''),result=?,result_digest=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, attemptState, now, in.ErrorCode, string(in.Result), digest, now, token.NamespaceID, a.ID, a.Revision)
if err := exactlyOne(changed, err); err != nil {
return err
}
changed, err = tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,result=?,lease_until=NULL,completed_at=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, taskState, string(in.Result), completed, now, token.NamespaceID, in.Ref.TaskID, task.view.Revision)
if err := exactlyOne(changed, err); err != nil {
return err
}
if err := releasePermits(ctx, tx, token.NamespaceID, a.ID, now); err != nil {
return err
}
event := map[string]any{"attempt_id": a.ID, "epoch": fmt.Sprint(a.Epoch), "state": taskState, "outcome": in.Outcome, "error_code": in.ErrorCode}
if err := emitEvent(ctx, tx, token.NamespaceID, in.Ref.TaskID, task.view.Revision+1, "task", resultEvent(taskState), event); err != nil {
return err
}
if err := appendAudit(ctx, tx, token, message, in.Ref.TaskID, "task", "task.result", "worker result", map[string]any{"state": task.view.State}, event); err != nil {
return err
}
receipt.TaskState = taskState
receipt.AttemptState = attemptState
receipt.TaskRevision++
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, "finish", digest, receipt); err != nil {
return err
}
out = worker.CommandResult{Receipt: receipt, Lease: worker.LeaseStatus{State: attemptState, LeaseUntil: a.LeaseUntil, ServerTime: now}}
return nil
})
return out, err
}
func resultEvent(state pulse.TaskState) string {
switch state {
case pulse.TaskSucceeded:
return "task.succeeded"
case pulse.TaskFailed:
return "task.failed"
case pulse.TaskNeedsReview:
return "task.needs_review"
case pulse.TaskWaiting:
return "task.requeued"
default:
return "task.cancelled"
}
}
func saveEvidence(ctx context.Context, tx *sql.Tx, ns, message core.ID, in worker.FinishInput, digest core.Digest, reason string) error {
id, err := core.NewID()
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempt_evidence(namespace_id,id,attempt_id,message_id,outcome,error_code,payload,request_digest,reason) VALUES (?,?,?,?,?,NULLIF(?,''),?,?,?)`, ns, id, in.Ref.AttemptID, message, in.Outcome, in.ErrorCode, string(in.Result), digest, reason)
if err != nil {
return err
}
return emitEvent(ctx, tx, ns, id, 1, "attempt_evidence", "attempt.evidence_received", map[string]any{"attempt_id": in.Ref.AttemptID, "task_id": in.Ref.TaskID, "reason": reason, "outcome": in.Outcome})
}