Files
orbit/sdk/go/workflow_instances_integration_test.go

404 lines
18 KiB
Go

package orbit
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/http/httputil"
"net/url"
"sync/atomic"
"testing"
"time"
)
var workflowExact = json.RawMessage(`{"n":9007199254740993}`)
func workflowSDKSpec(queue, handler string) WorkflowSpecification {
typ := WorkflowType{Kind: "OBJECT", Fields: map[string]WorkflowType{"n": {Kind: "NUMBER"}}}
task := WorkflowTask{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", InputType: typ, Input: WorkflowRef("input"), ResultType: typ, TimeoutMS: 60000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 10000, InitialDelayMS: 100, MaxDelayMS: 100}, RecoverySafety: RecoveryReconcileOnly}
return WorkflowSpecification{SchemaVersion: 1, InputType: typ, TimeoutMS: 120000, FailurePolicy: "REVIEW", Body: WorkflowBlock{Steps: []WorkflowStep{{Key: "work", Kind: "TASK", Task: &task}}, OutputType: typ, Output: WorkflowRef("work")}}
}
func publishWorkflowSDK(t *testing.T, ctx context.Context, api *Client, spec WorkflowSpecification) StartWorkflowRequest {
t.Helper()
key := workerTestID(t)
d, err := api.CreateWorkflowDefinition(ctx, key, CreateWorkflowDefinitionRequest{Name: "runtime." + key})
if err != nil {
t.Fatal(err)
}
if _, err = api.PublishWorkflowVersion(ctx, d.Definition.ID, key, PublishWorkflowVersionRequest{Version: "v1", Specification: spec, Reason: "verify public workflow chain"}); err != nil {
t.Fatal(err)
}
return StartWorkflowRequest{DefinitionID: d.Definition.ID, Version: "v1", Input: workflowExact, BusinessKey: "作业关联 "}
}
func startWorkflowSDK(t *testing.T, ctx context.Context, api *Client, spec WorkflowSpecification) WorkflowInstance {
t.Helper()
out, err := api.StartWorkflow(ctx, workerTestID(t), publishWorkflowSDK(t, ctx, api, spec))
if err != nil {
t.Fatal(err)
}
return out.Instance
}
func waitWorkflowSDK(t *testing.T, ctx context.Context, api *Client, id, state string) WorkflowInstance {
t.Helper()
for {
v, err := api.GetWorkflow(ctx, id)
if err != nil {
t.Fatal(err)
}
if v.State == state {
return v
}
select {
case <-ctx.Done():
t.Fatalf("workflow wanted %s, got %s", state, v.State)
case <-time.After(40 * time.Millisecond):
}
}
}
func waitWorkflowStepSDK(t *testing.T, ctx context.Context, api *Client, id, key string, predicate func(WorkflowStepDetail) bool) WorkflowStepDetail {
t.Helper()
for {
v, err := api.GetWorkflowStep(ctx, id, key)
if err != nil {
t.Fatal(err)
}
if predicate(v) {
return v
}
select {
case <-ctx.Done():
t.Fatalf("workflow step timed out: %s %s", key, v.Step.State)
case <-time.After(40 * time.Millisecond):
}
}
}
func workflowWorkerSDK(t *testing.T, ctx context.Context, api *Client, queue, handler string) *WorkerConnection {
t.Helper()
reg, err := NewWorkerRegistration(queue, 1, []WorkerCapability{{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}})
if err != nil {
t.Fatal(err)
}
w, err := api.DialWorker(ctx, reg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { w.Close() })
if _, err = w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
t.Fatal(err)
}
return w
}
func finishWorkflowSDK(t *testing.T, ctx context.Context, w *WorkerConnection, outcome WorkerOutcome, result json.RawMessage) WorkerAssignment {
t.Helper()
a, err := w.NextAssignment(ctx)
if err != nil {
t.Fatal(err)
}
if _, err = w.Start(ctx, workerTestID(t), a.Attempt()); err != nil {
t.Fatal(err)
}
code := ""
if outcome != WorkerSucceeded {
code = "EXAMPLE_EXECUTION_RESULT"
}
if _, err = w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: outcome, Result: result, ErrorCode: code}); err != nil {
t.Fatal(err)
}
return a
}
func workflowDecision(v WorkflowStepDetail) WorkflowStepDecision {
return WorkflowStepDecision{WorkflowRecoveryDecision: WorkflowRecoveryDecision{ExpectedRevision: v.Revision, Reason: "已核实模拟执行及等待", EvidenceRef: "evidence:workflow-sdk"}, StepKey: v.Step.Key, ActivationID: *v.Step.ActivationID, ExpectedStepRevision: v.Step.Revision}
}
func continueWorkflowSDK(t *testing.T, ctx context.Context, api *Client, id string) {
t.Helper()
v, err := api.GetWorkflow(ctx, id)
if err != nil {
t.Fatal(err)
}
key := workerTestID(t)
in := ContinueWorkflowRequest{WorkflowRecoveryDecision: WorkflowRecoveryDecision{ExpectedRevision: v.Revision, Reason: "核查事项已结清", EvidenceRef: "evidence:workflow-sdk"}, ReviewResolved: true}
first, err := api.ContinueWorkflow(ctx, id, key, in)
if err != nil {
t.Fatal(err)
}
again, err := api.ContinueWorkflow(ctx, id, key, in)
if err != nil || !again.Deduplicated || again.AuditID != first.AuditID {
t.Fatal("continue receipt", err)
}
}
func workflowAPIError(t *testing.T, err error, code string) {
t.Helper()
var v *APIError
if !errors.As(err, &v) || v.Code != code {
t.Fatalf("want %s got %v", code, err)
}
}
func TestIntegrationWorkflowInstancesSDK(t *testing.T) {
api, queue, token := integrationWorkerClient(t)
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
handler := "workflow." + workerTestID(t)
t.Run("start_receipt_control_timer_signal_and_audit", func(t *testing.T) {
spec := workflowSDKSpec(queue, handler)
task := *spec.Body.Steps[0].Task
spec.Body.Steps = append(spec.Body.Steps, WorkflowStep{Key: "delay", Kind: "TIMER", Timer: &WorkflowTimer{DelayMS: 500}}, WorkflowStep{Key: "receipt", Kind: "SIGNAL", Signal: &WorkflowSignal{Name: "done", PayloadVersion: "v1", Correlation: WorkflowValue{Kind: "LITERAL", Literal: json.RawMessage(`"operation "`)}, PayloadType: spec.InputType, TimeoutMS: 60000}}, WorkflowStep{Key: "tail", Kind: "TASK", Task: &task})
spec.Body.Output = WorkflowRef("tail")
in := publishWorkflowSDK(t, ctx, api, spec)
target, _ := url.Parse(api.baseURL)
proxy := httputil.NewSingleHostReverseProxy(target)
var dropped atomic.Bool
var calls atomic.Int32
proxy.ModifyResponse = func(r *http.Response) error {
calls.Add(1)
if dropped.CompareAndSwap(false, true) {
raw, e := io.ReadAll(r.Body)
r.Body.Close()
if e != nil {
return e
}
if r.StatusCode != 201 || !json.Valid(raw) {
t.Error("start did not commit")
}
return io.ErrUnexpectedEOF
}
return nil
}
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) {
conn, _, err := w.(http.Hijacker).Hijack()
if err == nil {
conn.Close()
}
}
gateway := httptest.NewServer(proxy)
defer gateway.Close()
transport := &http.Transport{DisableKeepAlives: true}
defer transport.CloseIdleConnections()
lost, err := NewClient(gateway.URL, Options{Token: token, HTTPClient: &http.Client{Transport: transport}})
if err != nil {
t.Fatal(err)
}
key := workerTestID(t)
if _, err = lost.StartWorkflow(ctx, key, in); err == nil || calls.Load() != 1 {
t.Fatal("start loss hidden", err)
}
started, err := api.StartWorkflow(ctx, key, in)
if err != nil || !started.Deduplicated {
t.Fatal("start receipt", err)
}
id := started.Instance.ID
before, err := api.GetWorkflowSteps(ctx, id)
if err != nil || len(before.Steps) != 4 {
t.Fatal(err)
}
pause := WorkflowControlRequest{Operation: "PAUSE", ExpectedControlRevision: started.Instance.ControlRevision, Reason: "接收事实但暂停新执行"}
pauseKey := workerTestID(t)
paused, err := api.ControlWorkflow(ctx, id, pauseKey, pause)
if err != nil || !paused.Instance.Paused {
t.Fatal(err)
}
same, err := api.ControlWorkflow(ctx, id, pauseKey, pause)
if err != nil || !same.Deduplicated {
t.Fatal(err)
}
_, err = api.ControlWorkflow(ctx, id, workerTestID(t), pause)
workflowAPIError(t, err, "STATE_CONFLICT")
if _, err = api.ControlWorkflow(ctx, id, workerTestID(t), WorkflowControlRequest{Operation: "RESUME", ExpectedControlRevision: paused.Instance.ControlRevision, Reason: "恢复执行"}); err != nil {
t.Fatal(err)
}
w := workflowWorkerSDK(t, ctx, api, queue, handler)
defer w.Close()
first := finishWorkflowSDK(t, ctx, w, WorkerSucceeded, workflowExact)
work, err := api.GetWorkflowStep(ctx, id, "work")
if err != nil || work.Step.TaskID == nil || *work.Step.TaskID != first.Task.ID || work.Step.ActivationSequence != 1 {
t.Fatal("step metadata", err)
}
wait := waitWorkflowStepSDK(t, ctx, api, id, "receipt", func(v WorkflowStepDetail) bool { return v.Step.State == "WAITING" })
if wait.Wait == nil || wait.Wait.CorrelationKey != "operation " {
t.Fatal("wait metadata")
}
signal := SendWorkflowSignalRequest{Name: "done", PayloadVersion: "v1", TargetStepKey: "receipt", TargetActivationSequence: wait.Step.ActivationSequence, CorrelationKey: wait.Wait.CorrelationKey, Payload: workflowExact, Reason: "external receipt"}
future := signal
future.TargetActivationSequence++
_, err = api.SendWorkflowSignal(ctx, id, workerTestID(t), future)
workflowAPIError(t, err, "STATE_CONFLICT")
signalKey := workerTestID(t)
accepted, err := api.SendWorkflowSignal(ctx, id, signalKey, signal)
if err != nil {
t.Fatal(err)
}
repeated, err := api.SendWorkflowSignal(ctx, id, signalKey, signal)
if err != nil || !repeated.Deduplicated || repeated.Signal.ID != accepted.Signal.ID {
t.Fatal("signal dedupe", err)
}
finishWorkflowSDK(t, ctx, w, WorkerSucceeded, workflowExact)
done := waitWorkflowSDK(t, ctx, api, id, "SUCCEEDED")
if string(done.Output) != string(workflowExact) {
t.Fatal("precision lost", string(done.Output))
}
receipt, err := api.GetWorkflowSignal(ctx, id, accepted.Signal.ID)
if err != nil || receipt.State != "CONSUMED" {
t.Fatal(err)
}
old, err := api.StartWorkflow(ctx, key, in)
if err != nil || old.Instance.Revision != started.Instance.Revision || old.Instance.DeadlineAt != started.Instance.DeadlineAt {
t.Fatal("start receipt changed", err)
}
page, err := api.ListWorkflowOperations(ctx, id, HistoryPageRequest{Limit: 1})
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
t.Fatal("audit pagination", err)
}
audit, err := api.GetWorkflowOperation(ctx, id, page.Items[0].ID)
if err != nil || audit.WorkflowID != id {
t.Fatal(err)
}
timer := workflowSDKSpec(queue, handler)
timer.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "wait", Kind: "TIMER", Timer: &WorkflowTimer{DelayMS: 30000}}}, OutputType: WorkflowType{Kind: "NULL"}, Output: WorkflowValue{Kind: "LITERAL", Literal: json.RawMessage(`null`)}}
other := startWorkflowSDK(t, ctx, api, timer)
_, err = api.ListWorkflowOperations(ctx, other.ID, HistoryPageRequest{Limit: 1, Cursor: page.NextCursor})
workflowAPIError(t, err, "INVALID_ARGUMENT")
_, err = api.ControlWorkflow(ctx, other.ID, workerTestID(t), WorkflowControlRequest{Operation: "CANCEL", ExpectedControlRevision: other.ControlRevision, Reason: "stop unused wait"})
if err != nil {
t.Fatal(err)
}
waitWorkflowSDK(t, ctx, api, other.ID, "CANCELLED")
})
t.Run("unknown_review_resolve_and_continue", func(t *testing.T) {
v := startWorkflowSDK(t, ctx, api, workflowSDKSpec(queue, handler))
w := workflowWorkerSDK(t, ctx, api, queue, handler)
defer w.Close()
a := finishWorkflowSDK(t, ctx, w, WorkerUnknown, json.RawMessage(`{"unknown":true}`))
step := waitWorkflowStepSDK(t, ctx, api, v.ID, "work", func(v WorkflowStepDetail) bool { return v.Step.State == "NEEDS_REVIEW" })
task, err := api.GetTask(ctx, a.Task.ID)
if err != nil {
t.Fatal(err)
}
in := ReviewWorkflowTaskRequest{WorkflowRecoveryDecision: workflowDecision(step).WorkflowRecoveryDecision, TaskID: task.ID, ExpectedTaskRevision: task.Revision, ExecutionSettled: true, Outcome: TaskSucceeded, Result: workflowExact}
key := workerTestID(t)
reviewed, err := api.ReviewWorkflowTask(ctx, v.ID, key, in)
if err != nil || reviewed.Task == nil || reviewed.TaskAuditID == nil {
t.Fatal(err)
}
duplicate, err := api.ReviewWorkflowTask(ctx, v.ID, key, in)
if err != nil || !duplicate.Deduplicated || duplicate.AuditID != reviewed.AuditID {
t.Fatal(err)
}
step = waitWorkflowStepSDK(t, ctx, api, v.ID, "work", func(v WorkflowStepDetail) bool { return v.Revision > reviewed.Instance.Revision })
resolved, err := api.ResolveWorkflowStep(ctx, v.ID, workerTestID(t), ResolveWorkflowStepRequest{WorkflowStepDecision: workflowDecision(step), ExecutionSettled: true, Outcome: "SUCCEEDED", Result: workflowExact})
if err != nil || resolved.Step.State != "SUCCEEDED" {
t.Fatal(err)
}
audit, err := api.GetWorkflowOperation(ctx, v.ID, resolved.AuditID)
if err != nil || audit.Action != "workflow.step.resolve" || audit.EvidenceRef != "evidence:workflow-sdk" {
t.Fatal(err)
}
continueWorkflowSDK(t, ctx, api, v.ID)
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
attempt, err := api.GetTaskAttempt(ctx, task.ID, a.AttemptID)
if err != nil || attempt.State != "UNKNOWN" {
t.Fatal("old evidence rewritten", err)
}
})
t.Run("retry_task_and_rearm_signal", func(t *testing.T) {
spec := workflowSDKSpec(queue, handler)
task := spec.Body.Steps[0]
wait := WorkflowStep{Key: "receipt", Kind: "SIGNAL", Signal: &WorkflowSignal{Name: "done", PayloadVersion: "v1", Correlation: WorkflowValue{Kind: "LITERAL", Literal: json.RawMessage(`"shared"`)}, PayloadType: spec.InputType, TimeoutMS: 60000}}
typ := WorkflowType{Kind: "OBJECT", Fields: map[string]WorkflowType{"left": spec.InputType, "right": spec.InputType}}
spec.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "pair", Kind: "PARALLEL", Parallel: &WorkflowParallel{Branches: []WorkflowParallelBranch{{Name: "left", Body: WorkflowBlock{Steps: []WorkflowStep{task}, OutputType: spec.InputType, Output: WorkflowRef("work")}}, {Name: "right", Body: WorkflowBlock{Steps: []WorkflowStep{wait}, OutputType: spec.InputType, Output: WorkflowRef("receipt")}}}}}}, OutputType: typ, Output: WorkflowRef("pair")}
v := startWorkflowSDK(t, ctx, api, spec)
w := workflowWorkerSDK(t, ctx, api, queue, handler)
defer w.Close()
finishWorkflowSDK(t, ctx, w, WorkerFailed, json.RawMessage(`{"known_failure":true}`))
waitWorkflowStepSDK(t, ctx, api, v.ID, "work", func(v WorkflowStepDetail) bool { return v.Step.State == "FAILED" })
old, err := api.GetWorkflowStep(ctx, v.ID, "receipt")
if err != nil {
t.Fatal(err)
}
key := workerTestID(t)
rearm := RearmWorkflowWaitRequest{WorkflowStepDecision: workflowDecision(old), WaitVerified: true}
restored, err := api.RearmWorkflowWait(ctx, v.ID, key, rearm)
if err != nil || restored.Step.ActivationSequence != 2 {
t.Fatal("new wait generation", err)
}
same, err := api.RearmWorkflowWait(ctx, v.ID, key, rearm)
if err != nil || !same.Deduplicated || same.AuditID != restored.AuditID {
t.Fatal(err)
}
current, err := api.GetWorkflowStep(ctx, v.ID, "receipt")
if err != nil || current.Wait.DeadlineAt != old.Wait.DeadlineAt {
t.Fatal("wait extended", err)
}
stale, err := api.SendWorkflowSignal(ctx, v.ID, workerTestID(t), SendWorkflowSignalRequest{Name: "done", PayloadVersion: "v1", TargetStepKey: "receipt", TargetActivationSequence: 1, CorrelationKey: "shared", Payload: workflowExact, Reason: "late prior generation"})
if err != nil || stale.Signal.State != "REJECTED" {
t.Fatal("old signal not fenced", err)
}
step, err := api.GetWorkflowStep(ctx, v.ID, "work")
if err != nil {
t.Fatal(err)
}
retryKey := workerTestID(t)
retry := RetryWorkflowStepRequest{WorkflowStepDecision: workflowDecision(step), SafeToRetry: true}
child, err := api.RetryWorkflowStep(ctx, v.ID, retryKey, retry)
if err != nil || child.Step.ActivationSequence != 2 || child.Task.Retry.MaxAttempts != 1 {
t.Fatal("task budget restored incorrectly", err)
}
repeated, err := api.RetryWorkflowStep(ctx, v.ID, retryKey, retry)
if err != nil || !repeated.Deduplicated || repeated.Task.ID != child.Task.ID {
t.Fatal(err)
}
continueWorkflowSDK(t, ctx, api, v.ID)
a := finishWorkflowSDK(t, ctx, w, WorkerSucceeded, workflowExact)
if a.Task.ID != child.Task.ID {
t.Fatal("wrong retry task")
}
_, err = api.SendWorkflowSignal(ctx, v.ID, workerTestID(t), SendWorkflowSignalRequest{Name: "done", PayloadVersion: "v1", TargetStepKey: "receipt", TargetActivationSequence: 2, CorrelationKey: "shared", Payload: workflowExact, Reason: "verified new wait"})
if err != nil {
t.Fatal(err)
}
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
})
t.Run("review_pending_then_continue_original_poll", func(t *testing.T) {
spec := workflowSDKSpec(queue, handler)
task := *spec.Body.Steps[0].Task
task.ResultType = WorkflowType{Kind: "STRING", Enum: []string{"PENDING", "DONE"}}
spec.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "query", Kind: "POLL", Poll: &WorkflowPoll{Task: task, MaxRounds: 3, TimeoutMS: 60000, IntervalMS: 1000, Pending: []string{"PENDING"}, Succeeded: []string{"DONE"}}}}, OutputType: task.ResultType, Output: WorkflowRef("query")}
v := startWorkflowSDK(t, ctx, api, spec)
w := workflowWorkerSDK(t, ctx, api, queue, handler)
defer w.Close()
finishWorkflowSDK(t, ctx, w, WorkerUnknown, json.RawMessage(`"unknown"`))
step := waitWorkflowStepSDK(t, ctx, api, v.ID, "query", func(v WorkflowStepDetail) bool { return v.Step.State == "NEEDS_REVIEW" })
source, err := api.GetTask(ctx, step.Poll.LastTaskID)
if err != nil {
t.Fatal(err)
}
reviewed, err := api.ReviewWorkflowTask(ctx, v.ID, workerTestID(t), ReviewWorkflowTaskRequest{WorkflowRecoveryDecision: workflowDecision(step).WorkflowRecoveryDecision, TaskID: source.ID, ExpectedTaskRevision: source.Revision, ExecutionSettled: true, Outcome: TaskSucceeded, Result: json.RawMessage(`"PENDING"`)})
if err != nil {
t.Fatal(err)
}
step = waitWorkflowStepSDK(t, ctx, api, v.ID, "query", func(v WorkflowStepDetail) bool { return v.Revision > reviewed.Instance.Revision })
key := workerTestID(t)
in := ContinueWorkflowPollRequest{WorkflowStepDecision: workflowDecision(step), PendingConfirmed: true}
out, err := api.ContinueWorkflowPoll(ctx, v.ID, key, in)
if err != nil || out.Step.ActivationSequence != 1 {
t.Fatal(err)
}
same, err := api.ContinueWorkflowPoll(ctx, v.ID, key, in)
if err != nil || !same.Deduplicated || same.AuditID != out.AuditID {
t.Fatal(err)
}
continueWorkflowSDK(t, ctx, api, v.ID)
finishWorkflowSDK(t, ctx, w, WorkerSucceeded, json.RawMessage(`"DONE"`))
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
final, err := api.GetWorkflowStep(ctx, v.ID, "query")
if err != nil || final.Poll.UsedRounds != 2 || final.Poll.DeadlineAt != step.Poll.DeadlineAt {
t.Fatal("poll budget changed", err)
}
})
if !t.Failed() {
t.Log("public workflow start/query/control/signal, all six recovery commands, audit and real Worker execution verified without server imports")
}
}