// workflow-recovery 注入本地渠道提交后断线,使用明确证据恢复,保留UNKNOWN尝试和暂停门控。 package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "os" "sync" "sync/atomic" "time" "git.yuntuo.hk/k2safe/orbit/examples/internal/flowdemo" orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" ) const exactSequence uint64 = 9007199254740993 type request struct { OperationID string `json:"operation_id"` // OperationID 原外部操作身份,核查时保持不变。 Sequence uint64 `json:"sequence"` // Sequence 精确业务序号,不使用float64解析。 } type receipt struct { OperationID string `json:"operation_id"` // OperationID 已完成操作的稳定身份。 Sequence uint64 `json:"sequence"` // Sequence 渠道已保存的精确序号。 State string `json:"state"` // State 本夹具唯一可核实的成功状态CONFIRMED。 } func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } func run() error { api, queue, operation, err := flowdemo.Client() if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() var mu sync.Mutex var committed json.RawMessage var submissions atomic.Int32 var dropped atomic.Bool // 本地夹具代表唯一受控外部系统:先保存结果,再关闭HTTP连接而不发送回执。 // 真实人工核查需要持久业务证据及执行已结束的证明,不能只凭GET成功推断所有执行已停止。 remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/objects/"+operation { http.NotFound(w, r) return } if r.Method == "GET" { mu.Lock() raw := append([]byte(nil), committed...) mu.Unlock() if len(raw) == 0 { http.NotFound(w, r) return } w.Header().Set("Content-Type", "application/json") _, _ = w.Write(raw) return } if r.Method != "POST" { w.WriteHeader(http.StatusMethodNotAllowed) return } raw, err := io.ReadAll(io.LimitReader(r.Body, 4097)) var in request if err != nil || len(raw) > 4096 || json.Unmarshal(raw, &in) != nil || in.OperationID != operation || in.Sequence != exactSequence { http.Error(w, "invalid command", 400) return } if submissions.Add(1) != 1 { http.Error(w, "unexpected resubmission", 409) return } evidence, _ := json.Marshal(receipt{operation, exactSequence, "CONFIRMED"}) mu.Lock() committed = evidence mu.Unlock() conn, _, err := w.(http.Hijacker).Hijack() if err == nil { dropped.Store(true) _ = conn.Close() } })) defer remote.Close() httpClient := remote.Client() httpClient.Timeout = 3 * time.Second requestType := orbit.WorkflowType{Kind: "OBJECT", Fields: map[string]orbit.WorkflowType{"operation_id": {Kind: "STRING"}, "sequence": {Kind: "NUMBER"}}} receiptType := orbit.WorkflowType{Kind: "OBJECT", Fields: map[string]orbit.WorkflowType{"operation_id": {Kind: "STRING"}, "sequence": {Kind: "NUMBER"}, "state": {Kind: "STRING", Enum: []string{"CONFIRMED"}}}} publishHandler, recordHandler := "object.publish."+operation, "object.record."+operation publish := orbit.WorkflowTask{QueueID: queue, Handler: publishHandler, HandlerVersion: "v1", PayloadVersion: "v1", InputType: requestType, Input: orbit.WorkflowRef("input"), ResultType: receiptType, TimeoutMS: 30000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 5000}, RecoverySafety: orbit.RecoveryReconcileOnly} record := publish record.Handler = recordHandler record.InputType = receiptType record.Input = orbit.WorkflowRef("publish") record.RecoverySafety = orbit.RecoveryIdempotent spec := orbit.WorkflowSpecification{SchemaVersion: 1, InputType: requestType, TimeoutMS: 60000, FailurePolicy: "REVIEW", Body: orbit.WorkflowBlock{Steps: []orbit.WorkflowStep{{Key: "publish", Kind: "TASK", Task: &publish}, {Key: "record", Kind: "TASK", Task: &record}}, OutputType: receiptType, Output: orbit.WorkflowRef("record")}} input, _ := json.Marshal(request{operation, exactSequence}) v, err := flowdemo.PublishStart(ctx, api, operation, "recovery", spec, input) if err != nil { return err } runner, err := flowdemo.StartWorker(ctx, api, queue, orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: publishHandler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) { req, err := http.NewRequestWithContext(ctx, "POST", remote.URL+"/objects/"+operation, bytes.NewReader(e.Task.Payload)) if err != nil { return nil, err } resp, err := httpClient.Do(req) if err != nil { return nil, err } // 回执丢失只能报UNKNOWN,不能声明明确失败以触发重发。 resp.Body.Close() return nil, errors.New("fixture expected a lost response; inspect remote operation") }}, orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: recordHandler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) { if err := ctx.Err(); err != nil { return nil, err } if err := verifyReceipt(e.Task.Payload, operation); err != nil { return nil, &orbit.WorkerFailure{Code: "INVALID_RECEIPT"} } return e.Task.Payload, nil }}) if err != nil { return err } defer runner.Close() step, err := runner.Step(ctx, api, v.ID, "publish", func(d orbit.WorkflowStepDetail) bool { return d.Step.State == "NEEDS_REVIEW" }) if err != nil { return err } if step.Step.TaskID == nil || step.Step.ActivationID == nil { return errors.New("review source missing") } taskID, activation := *step.Step.TaskID, *step.Step.ActivationID task, err := api.GetTask(ctx, taskID) if err != nil { return err } attempts, err := api.ListTaskAttempts(ctx, taskID, orbit.HistoryPageRequest{Limit: 2}) if err != nil { return err } if task.State != orbit.TaskNeedsReview || len(attempts.Items) != 1 || attempts.Items[0].State != "UNKNOWN" || attempts.Items[0].EndedAt == nil || submissions.Load() != 1 || !dropped.Load() { return errors.New("lost response did not retain one UNKNOWN execution") } if err := runner.Wait(ctx, func() (bool, error) { s := runner.Worker.Stats() return s.Running == 0 && s.ReportsConfirmed == 1, nil }); err != nil { return err } current, err := api.GetWorkflow(ctx, v.ID) if err != nil { return err } if _, err = api.ControlWorkflow(ctx, v.ID, operation+".pause", orbit.WorkflowControlRequest{Operation: "PAUSE", ExpectedControlRevision: current.ControlRevision, Reason: "hold successor while operator verifies evidence"}); err != nil { return err } // 这里只在受控夹具证明“唯一提交已经结束、操作身份和结果一致”后模拟管理员决定。 req, err := http.NewRequestWithContext(ctx, "GET", remote.URL+"/objects/"+operation, nil) if err != nil { return err } resp, err := httpClient.Do(req) if err != nil { return err } evidence, readErr := io.ReadAll(io.LimitReader(resp.Body, 4097)) resp.Body.Close() if readErr != nil { return readErr } if resp.StatusCode != 200 || len(evidence) > 4096 { return errors.New("verified external evidence unavailable") } if err := verifyReceipt(evidence, operation); err != nil { return err } step, err = api.GetWorkflowStep(ctx, v.ID, "publish") if err != nil { return err } evidenceRef := "evidence:local-fixture/" + operation decision := orbit.WorkflowRecoveryDecision{ExpectedRevision: step.Revision, Reason: "verified fixture commit, original operation and settled execution", EvidenceRef: evidenceRef} command := orbit.ReviewWorkflowTaskRequest{WorkflowRecoveryDecision: decision, TaskID: taskID, ExpectedTaskRevision: task.Revision, ExecutionSettled: true, Outcome: orbit.TaskSucceeded, Result: evidence} reviewed, err := api.ReviewWorkflowTask(ctx, v.ID, operation+".review", command) if err != nil { return err } again, err := api.ReviewWorkflowTask(ctx, v.ID, operation+".review", command) if err != nil { return err } if !again.Deduplicated || again.AuditID != reviewed.AuditID || reviewed.Task == nil || reviewed.TaskAuditID == nil { return errors.New("review identity or dual audit missing") } step, err = runner.Step(ctx, api, v.ID, "publish", func(d orbit.WorkflowStepDetail) bool { return d.Revision > reviewed.Instance.Revision }) if err != nil { return err } if step.Step.ActivationID == nil || *step.Step.ActivationID != activation { return errors.New("review changed activation") } decision.ExpectedRevision = step.Revision resolved, err := api.ResolveWorkflowStep(ctx, v.ID, operation+".resolve", orbit.ResolveWorkflowStepRequest{WorkflowStepDecision: orbit.WorkflowStepDecision{WorkflowRecoveryDecision: decision, StepKey: "publish", ActivationID: activation, ExpectedStepRevision: step.Step.Revision}, ExecutionSettled: true, Outcome: "SUCCEEDED", Result: evidence}) if err != nil { return err } current, err = api.GetWorkflow(ctx, v.ID) if err != nil { return err } decision.ExpectedRevision = current.Revision continued, err := api.ContinueWorkflow(ctx, v.ID, operation+".continue", orbit.ContinueWorkflowRequest{WorkflowRecoveryDecision: decision, ReviewResolved: true}) if err != nil { return err } next, err := api.GetWorkflowStep(ctx, v.ID, "record") if err != nil { return err } if !continued.Instance.Paused || next.Step.ActivationID != nil || next.Step.TaskID != nil || next.Step.State != "BLOCKED" { return errors.New("manual recovery bypassed pause gate") } for action, id := range map[string]string{"workflow.task.review": reviewed.AuditID, "workflow.step.resolve": resolved.AuditID, "workflow.review.continue": continued.AuditID} { audit, err := api.GetWorkflowOperation(ctx, v.ID, id) if err != nil { return err } if audit.Action != action || audit.EvidenceRef != evidenceRef || audit.Reason != decision.Reason || audit.ActorID == "" { return errors.New("management evidence or actor missing") } } current, err = api.GetWorkflow(ctx, v.ID) if err != nil { return err } if _, err = api.ControlWorkflow(ctx, v.ID, operation+".resume", orbit.WorkflowControlRequest{Operation: "RESUME", ExpectedControlRevision: current.ControlRevision, Reason: "verified review complete; explicitly release pause"}); err != nil { return err } final, err := runner.Workflow(ctx, api, v.ID, "SUCCEEDED") if err != nil { return err } if err := verifyReceipt(final.Output, operation); err != nil { return err } old, err := api.GetTaskAttempt(ctx, taskID, attempts.Items[0].ID) if err != nil { return err } if old.State != "UNKNOWN" || old.EndedAt == nil || !old.EndedAt.Equal(*attempts.Items[0].EndedAt) || submissions.Load() != 1 || !final.DeadlineAt.Equal(v.DeadlineAt) { return errors.New("recovery rewrote history, repeated submission or extended deadline") } if err := runner.Close(); err != nil { return err } stats := runner.Worker.Stats() if stats.Started != 2 || stats.ReportsConfirmed != 2 || stats.Running != 0 { return errors.New("unexpected executions or incomplete drain") } fmt.Printf("recovery_verified=true state=%s submissions=%d unknown_history_preserved=true review_deduplicated=true pause_preserved=true audits=3 sequence=%d worker_drained=true\n", final.State, submissions.Load(), exactSequence) return nil } func verifyReceipt(raw json.RawMessage, operation string) error { var out receipt if json.Unmarshal(raw, &out) != nil || out.OperationID != operation || out.Sequence != exactSequence || out.State != "CONFIRMED" { return errors.New("external receipt identity or exact sequence mismatch") } return nil }