514 lines
25 KiB
Go
514 lines
25 KiB
Go
package orbit
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/http/httputil"
|
|
"net/url"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
var workflowExact = json.RawMessage(`{"n":9007199254740993}`)
|
|
|
|
func workflowSDKSpec(queue, handler string) WorkflowSpecification {
|
|
typ := WorkflowType{Kind: "OBJECT", Fields: map[string]WorkflowType{"n": {Kind: "NUMBER"}}}
|
|
task := WorkflowTask{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", InputType: typ, Input: WorkflowRef("input"), ResultType: typ, TimeoutMS: 60000, Retry: RetryPolicy{MaxAttempts: 2, AttemptTimeoutMS: 10000, InitialDelayMS: 100, MaxDelayMS: 100}, RecoverySafety: RecoveryReconcileOnly}
|
|
return WorkflowSpecification{SchemaVersion: 1, InputType: typ, TimeoutMS: 120000, FailurePolicy: "REVIEW", Body: WorkflowBlock{Steps: []WorkflowStep{{Key: "work", Kind: "TASK", Task: &task}}, OutputType: typ, Output: WorkflowRef("work")}}
|
|
}
|
|
func publishWorkflowSDK(t *testing.T, ctx context.Context, api *Client, spec WorkflowSpecification) StartWorkflowRequest {
|
|
t.Helper()
|
|
key := workerTestID(t)
|
|
d, err := api.CreateWorkflowDefinition(ctx, key, CreateWorkflowDefinitionRequest{Name: "runtime." + key})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = api.PublishWorkflowVersion(ctx, d.Definition.ID, key, PublishWorkflowVersionRequest{Version: "v1", Specification: spec, Reason: "verify public workflow chain"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return StartWorkflowRequest{DefinitionID: d.Definition.ID, Version: "v1", Input: workflowExact, BusinessKey: "作业关联 "}
|
|
}
|
|
func startWorkflowSDK(t *testing.T, ctx context.Context, api *Client, spec WorkflowSpecification) WorkflowInstance {
|
|
t.Helper()
|
|
out, err := api.StartWorkflow(ctx, workerTestID(t), publishWorkflowSDK(t, ctx, api, spec))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Instance
|
|
}
|
|
func waitWorkflowSDK(t *testing.T, ctx context.Context, api *Client, id, state string) WorkflowInstance {
|
|
t.Helper()
|
|
for {
|
|
v, err := api.GetWorkflow(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v.State == state {
|
|
return v
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Fatalf("workflow wanted %s, got %s", state, v.State)
|
|
case <-time.After(40 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
func waitWorkflowStepSDK(t *testing.T, ctx context.Context, api *Client, id, key string, predicate func(WorkflowStepDetail) bool) WorkflowStepDetail {
|
|
t.Helper()
|
|
for {
|
|
v, err := api.GetWorkflowStep(ctx, id, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if predicate(v) {
|
|
return v
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Fatalf("workflow step timed out: %s %s", key, v.Step.State)
|
|
case <-time.After(40 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
func workflowWorkerSDK(t *testing.T, ctx context.Context, api *Client, queue, handler string) *WorkerConnection {
|
|
t.Helper()
|
|
reg, err := NewWorkerRegistration(queue, 1, []WorkerCapability{{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w, err := api.DialWorker(ctx, reg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { w.Close() })
|
|
if _, err = w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return w
|
|
}
|
|
func finishWorkflowSDK(t *testing.T, ctx context.Context, w *WorkerConnection, outcome WorkerOutcome, result json.RawMessage) WorkerAssignment {
|
|
t.Helper()
|
|
a, err := w.NextAssignment(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = w.Start(ctx, workerTestID(t), a.Attempt()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code := ""
|
|
if outcome != WorkerSucceeded {
|
|
code = "EXAMPLE_EXECUTION_RESULT"
|
|
}
|
|
if _, err = w.Finish(ctx, workerTestID(t), WorkerFinish{Ref: a.Attempt(), Outcome: outcome, Result: result, ErrorCode: code}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return a
|
|
}
|
|
func workflowDecision(v WorkflowStepDetail) WorkflowStepDecision {
|
|
return WorkflowStepDecision{WorkflowRecoveryDecision: WorkflowRecoveryDecision{ExpectedRevision: v.Revision, Reason: "已核实模拟执行及等待", EvidenceRef: "evidence:workflow-sdk"}, StepKey: v.Step.Key, ActivationID: *v.Step.ActivationID, ExpectedStepRevision: v.Step.Revision}
|
|
}
|
|
func continueWorkflowSDK(t *testing.T, ctx context.Context, api *Client, id string) {
|
|
t.Helper()
|
|
v, err := api.GetWorkflow(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key := workerTestID(t)
|
|
in := ContinueWorkflowRequest{WorkflowRecoveryDecision: WorkflowRecoveryDecision{ExpectedRevision: v.Revision, Reason: "核查事项已结清", EvidenceRef: "evidence:workflow-sdk"}, ReviewResolved: true}
|
|
first, err := api.ContinueWorkflow(ctx, id, key, in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := api.ContinueWorkflow(ctx, id, key, in)
|
|
if err != nil || !again.Deduplicated || again.AuditID != first.AuditID {
|
|
t.Fatal("continue receipt", err)
|
|
}
|
|
}
|
|
func workflowAPIError(t *testing.T, err error, code string) {
|
|
t.Helper()
|
|
var v *APIError
|
|
if !errors.As(err, &v) || v.Code != code {
|
|
t.Fatalf("want %s got %v", code, err)
|
|
}
|
|
}
|
|
|
|
func assertWorkflowSourceSDK(t *testing.T, ctx context.Context, api *Client, v WorkflowInstance, task string, sequence, round uint32, parent string) {
|
|
t.Helper()
|
|
source, err := api.GetTaskSource(ctx, task)
|
|
if err != nil || source.Workflow == nil || source.Kind != "WORKFLOW" {
|
|
t.Fatal("workflow source", err)
|
|
}
|
|
w := source.Workflow
|
|
if w.WorkflowID != v.ID || w.DefinitionID != v.DefinitionID || w.DefinitionVersionID != v.DefinitionVersionID || w.Version != v.Version || w.ActivationSequence != sequence || w.PollRound != round || parent == "" && w.RetryOf != nil || parent != "" && (w.RetryOf == nil || *w.RetryOf != parent) {
|
|
t.Fatal("fixed source mismatch", w)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationWorkflowInstancesSDK(t *testing.T) {
|
|
api, queue, token := integrationWorkerClient(t)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
|
defer cancel()
|
|
handler := "workflow." + workerTestID(t)
|
|
t.Run("start_receipt_control_timer_signal_and_audit", func(t *testing.T) {
|
|
spec := workflowSDKSpec(queue, handler)
|
|
task := *spec.Body.Steps[0].Task
|
|
spec.Body.Steps = append(spec.Body.Steps, WorkflowStep{Key: "delay", Kind: "TIMER", Timer: &WorkflowTimer{DelayMS: 500}}, WorkflowStep{Key: "receipt", Kind: "SIGNAL", Signal: &WorkflowSignal{Name: "done", PayloadVersion: "v1", Correlation: WorkflowValue{Kind: "LITERAL", Literal: json.RawMessage(`"operation "`)}, PayloadType: spec.InputType, TimeoutMS: 60000}}, WorkflowStep{Key: "tail", Kind: "TASK", Task: &task})
|
|
spec.Body.Output = WorkflowRef("tail")
|
|
in := publishWorkflowSDK(t, ctx, api, spec)
|
|
target, _ := url.Parse(api.baseURL)
|
|
proxy := httputil.NewSingleHostReverseProxy(target)
|
|
var dropped atomic.Bool
|
|
var calls atomic.Int32
|
|
proxy.ModifyResponse = func(r *http.Response) error {
|
|
calls.Add(1)
|
|
if dropped.CompareAndSwap(false, true) {
|
|
raw, e := io.ReadAll(r.Body)
|
|
r.Body.Close()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if r.StatusCode != 201 || !json.Valid(raw) {
|
|
t.Error("start did not commit")
|
|
}
|
|
return io.ErrUnexpectedEOF
|
|
}
|
|
return nil
|
|
}
|
|
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) {
|
|
conn, _, err := w.(http.Hijacker).Hijack()
|
|
if err == nil {
|
|
conn.Close()
|
|
}
|
|
}
|
|
gateway := httptest.NewServer(proxy)
|
|
defer gateway.Close()
|
|
transport := &http.Transport{DisableKeepAlives: true}
|
|
defer transport.CloseIdleConnections()
|
|
lost, err := NewClient(gateway.URL, Options{Token: token, HTTPClient: &http.Client{Transport: transport}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key := workerTestID(t)
|
|
if _, err = lost.StartWorkflow(ctx, key, in); err == nil || calls.Load() != 1 {
|
|
t.Fatal("start loss hidden", err)
|
|
}
|
|
started, err := api.StartWorkflow(ctx, key, in)
|
|
if err != nil || !started.Deduplicated {
|
|
t.Fatal("start receipt", err)
|
|
}
|
|
id := started.Instance.ID
|
|
|
|
// 所有新增历史路由必须拒绝未声明参数，不能悄悄扩大读取范围。
|
|
for _, suffix := range []string{"/steps/work/activations", "/steps/work/activations/" + id, "/steps/work/activations/" + id + "/rounds", "/signals", "/signals/" + id + "/operations", "/signals/" + id + "/operations/" + id} {
|
|
var unused any
|
|
err := api.call(ctx, "GET", "/v1/workflows/"+id+suffix+"?unknown=1", "", nil, &unused)
|
|
workflowAPIError(t, err, "INVALID_ARGUMENT")
|
|
}
|
|
before, err := api.GetWorkflowSteps(ctx, id)
|
|
if err != nil || len(before.Steps) != 4 {
|
|
t.Fatal(err)
|
|
}
|
|
pause := WorkflowControlRequest{Operation: "PAUSE", ExpectedControlRevision: started.Instance.ControlRevision, Reason: "接收事实但暂停新执行"}
|
|
pauseKey := workerTestID(t)
|
|
paused, err := api.ControlWorkflow(ctx, id, pauseKey, pause)
|
|
if err != nil || !paused.Instance.Paused {
|
|
t.Fatal(err)
|
|
}
|
|
same, err := api.ControlWorkflow(ctx, id, pauseKey, pause)
|
|
if err != nil || !same.Deduplicated {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = api.ControlWorkflow(ctx, id, workerTestID(t), pause)
|
|
workflowAPIError(t, err, "STATE_CONFLICT")
|
|
if _, err = api.ControlWorkflow(ctx, id, workerTestID(t), WorkflowControlRequest{Operation: "RESUME", ExpectedControlRevision: paused.Instance.ControlRevision, Reason: "恢复执行"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := workflowWorkerSDK(t, ctx, api, queue, handler)
|
|
defer w.Close()
|
|
first := finishWorkflowSDK(t, ctx, w, WorkerSucceeded, workflowExact)
|
|
assertWorkflowSourceSDK(t, ctx, api, started.Instance, first.Task.ID, 1, 0, "")
|
|
work, err := api.GetWorkflowStep(ctx, id, "work")
|
|
if err != nil || work.Step.TaskID == nil || *work.Step.TaskID != first.Task.ID || work.Step.ActivationSequence != 1 {
|
|
t.Fatal("step metadata", err)
|
|
}
|
|
wait := waitWorkflowStepSDK(t, ctx, api, id, "receipt", func(v WorkflowStepDetail) bool { return v.Step.State == "WAITING" })
|
|
if wait.Wait == nil || wait.Wait.CorrelationKey != "operation " {
|
|
t.Fatal("wait metadata")
|
|
}
|
|
signal := SendWorkflowSignalRequest{Name: "done", PayloadVersion: "v1", TargetStepKey: "receipt", TargetActivationSequence: wait.Step.ActivationSequence, CorrelationKey: wait.Wait.CorrelationKey, Payload: workflowExact, Reason: "external receipt"}
|
|
future := signal
|
|
future.TargetActivationSequence++
|
|
_, err = api.SendWorkflowSignal(ctx, id, workerTestID(t), future)
|
|
workflowAPIError(t, err, "STATE_CONFLICT")
|
|
signalKey := workerTestID(t)
|
|
accepted, err := api.SendWorkflowSignal(ctx, id, signalKey, signal)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repeated, err := api.SendWorkflowSignal(ctx, id, signalKey, signal)
|
|
if err != nil || !repeated.Deduplicated || repeated.Signal.ID != accepted.Signal.ID {
|
|
t.Fatal("signal dedupe", err)
|
|
}
|
|
finishWorkflowSDK(t, ctx, w, WorkerSucceeded, workflowExact)
|
|
done := waitWorkflowSDK(t, ctx, api, id, "SUCCEEDED")
|
|
if string(done.Output) != string(workflowExact) {
|
|
t.Fatal("precision lost", string(done.Output))
|
|
}
|
|
receipt, err := api.GetWorkflowSignal(ctx, id, accepted.Signal.ID)
|
|
if err != nil || receipt.State != "CONSUMED" {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
signals, err := api.ListWorkflowSignals(ctx, id, HistoryPageRequest{Limit: 1})
|
|
if err != nil || len(signals.Items) != 1 || signals.Items[0].ID != accepted.Signal.ID || signals.Items[0].State != "CONSUMED" {
|
|
t.Fatal("signal history", err)
|
|
}
|
|
operations, err := api.ListWorkflowSignalOperations(ctx, id, accepted.Signal.ID, HistoryPageRequest{Limit: 1})
|
|
if err != nil || len(operations.Items) != 1 {
|
|
t.Fatal("signal operations", err)
|
|
}
|
|
operation, err := api.GetWorkflowSignalOperation(ctx, id, accepted.Signal.ID, operations.Items[0].ID)
|
|
if err != nil || operation.Reason != signal.Reason || operation.Action != "workflow.signal" {
|
|
t.Fatal("signal operation detail", err)
|
|
}
|
|
historical, err := api.GetWorkflowStep(ctx, id, "receipt")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitHistory, err := api.GetWorkflowActivation(ctx, id, "receipt", *historical.Step.ActivationID)
|
|
if err != nil || waitHistory.Wait.WinnerSignalID == nil || *waitHistory.Wait.WinnerSignalID != accepted.Signal.ID {
|
|
t.Fatal("historical winner", err)
|
|
}
|
|
old, err := api.StartWorkflow(ctx, key, in)
|
|
if err != nil || old.Instance.Revision != started.Instance.Revision || old.Instance.DeadlineAt != started.Instance.DeadlineAt {
|
|
t.Fatal("start receipt changed", err)
|
|
}
|
|
page, err := api.ListWorkflowOperations(ctx, id, HistoryPageRequest{Limit: 1})
|
|
if err != nil || len(page.Items) != 1 || page.NextCursor == "" {
|
|
t.Fatal("audit pagination", err)
|
|
}
|
|
audit, err := api.GetWorkflowOperation(ctx, id, page.Items[0].ID)
|
|
if err != nil || audit.WorkflowID != id {
|
|
t.Fatal(err)
|
|
}
|
|
timer := workflowSDKSpec(queue, handler)
|
|
timer.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "wait", Kind: "TIMER", Timer: &WorkflowTimer{DelayMS: 30000}}}, OutputType: WorkflowType{Kind: "NULL"}, Output: WorkflowValue{Kind: "LITERAL", Literal: json.RawMessage(`null`)}}
|
|
other := startWorkflowSDK(t, ctx, api, timer)
|
|
_, err = api.ListWorkflowOperations(ctx, other.ID, HistoryPageRequest{Limit: 1, Cursor: page.NextCursor})
|
|
workflowAPIError(t, err, "INVALID_ARGUMENT")
|
|
_, err = api.ControlWorkflow(ctx, other.ID, workerTestID(t), WorkflowControlRequest{Operation: "CANCEL", ExpectedControlRevision: other.ControlRevision, Reason: "stop unused wait"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitWorkflowSDK(t, ctx, api, other.ID, "CANCELLED")
|
|
})
|
|
t.Run("unknown_review_resolve_and_continue", func(t *testing.T) {
|
|
v := startWorkflowSDK(t, ctx, api, workflowSDKSpec(queue, handler))
|
|
w := workflowWorkerSDK(t, ctx, api, queue, handler)
|
|
defer w.Close()
|
|
a := finishWorkflowSDK(t, ctx, w, WorkerUnknown, json.RawMessage(`{"unknown":true}`))
|
|
step := waitWorkflowStepSDK(t, ctx, api, v.ID, "work", func(v WorkflowStepDetail) bool { return v.Step.State == "NEEDS_REVIEW" })
|
|
task, err := api.GetTask(ctx, a.Task.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
in := ReviewWorkflowTaskRequest{WorkflowRecoveryDecision: workflowDecision(step).WorkflowRecoveryDecision, TaskID: task.ID, ExpectedTaskRevision: task.Revision, ExecutionSettled: true, Outcome: TaskSucceeded, Result: workflowExact}
|
|
key := workerTestID(t)
|
|
reviewed, err := api.ReviewWorkflowTask(ctx, v.ID, key, in)
|
|
if err != nil || reviewed.Task == nil || reviewed.TaskAuditID == nil {
|
|
t.Fatal(err)
|
|
}
|
|
duplicate, err := api.ReviewWorkflowTask(ctx, v.ID, key, in)
|
|
if err != nil || !duplicate.Deduplicated || duplicate.AuditID != reviewed.AuditID {
|
|
t.Fatal(err)
|
|
}
|
|
step = waitWorkflowStepSDK(t, ctx, api, v.ID, "work", func(v WorkflowStepDetail) bool { return v.Revision > reviewed.Instance.Revision })
|
|
resolved, err := api.ResolveWorkflowStep(ctx, v.ID, workerTestID(t), ResolveWorkflowStepRequest{WorkflowStepDecision: workflowDecision(step), ExecutionSettled: true, Outcome: "SUCCEEDED", Result: workflowExact})
|
|
if err != nil || resolved.Step.State != "SUCCEEDED" {
|
|
t.Fatal(err)
|
|
}
|
|
audit, err := api.GetWorkflowOperation(ctx, v.ID, resolved.AuditID)
|
|
if err != nil || audit.Action != "workflow.step.resolve" || audit.EvidenceRef != "evidence:workflow-sdk" {
|
|
t.Fatal(err)
|
|
}
|
|
continueWorkflowSDK(t, ctx, api, v.ID)
|
|
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
|
|
attempt, err := api.GetTaskAttempt(ctx, task.ID, a.AttemptID)
|
|
if err != nil || attempt.State != "UNKNOWN" {
|
|
t.Fatal("old evidence rewritten", err)
|
|
}
|
|
})
|
|
t.Run("retry_task_and_rearm_signal", func(t *testing.T) {
|
|
spec := workflowSDKSpec(queue, handler)
|
|
task := spec.Body.Steps[0]
|
|
wait := WorkflowStep{Key: "receipt", Kind: "SIGNAL", Signal: &WorkflowSignal{Name: "done", PayloadVersion: "v1", Correlation: WorkflowValue{Kind: "LITERAL", Literal: json.RawMessage(`"shared"`)}, PayloadType: spec.InputType, TimeoutMS: 60000}}
|
|
typ := WorkflowType{Kind: "OBJECT", Fields: map[string]WorkflowType{"left": spec.InputType, "right": spec.InputType}}
|
|
spec.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "pair", Kind: "PARALLEL", Parallel: &WorkflowParallel{Branches: []WorkflowParallelBranch{{Name: "left", Body: WorkflowBlock{Steps: []WorkflowStep{task}, OutputType: spec.InputType, Output: WorkflowRef("work")}}, {Name: "right", Body: WorkflowBlock{Steps: []WorkflowStep{wait}, OutputType: spec.InputType, Output: WorkflowRef("receipt")}}}}}}, OutputType: typ, Output: WorkflowRef("pair")}
|
|
v := startWorkflowSDK(t, ctx, api, spec)
|
|
w := workflowWorkerSDK(t, ctx, api, queue, handler)
|
|
defer w.Close()
|
|
finishWorkflowSDK(t, ctx, w, WorkerFailed, json.RawMessage(`{"known_failure":true}`))
|
|
waitWorkflowStepSDK(t, ctx, api, v.ID, "work", func(v WorkflowStepDetail) bool { return v.Step.State == "FAILED" })
|
|
old, err := api.GetWorkflowStep(ctx, v.ID, "receipt")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key := workerTestID(t)
|
|
rearm := RearmWorkflowWaitRequest{WorkflowStepDecision: workflowDecision(old), WaitVerified: true}
|
|
restored, err := api.RearmWorkflowWait(ctx, v.ID, key, rearm)
|
|
if err != nil || restored.Step.ActivationSequence != 2 {
|
|
t.Fatal("new wait generation", err)
|
|
}
|
|
same, err := api.RearmWorkflowWait(ctx, v.ID, key, rearm)
|
|
if err != nil || !same.Deduplicated || same.AuditID != restored.AuditID {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := api.GetWorkflowStep(ctx, v.ID, "receipt")
|
|
if err != nil || current.Wait.DeadlineAt != old.Wait.DeadlineAt {
|
|
t.Fatal("wait extended", err)
|
|
}
|
|
|
|
priorWait, err := api.GetWorkflowActivation(ctx, v.ID, "receipt", *old.Step.ActivationID)
|
|
if err != nil || priorWait.Activation.Sequence != 1 || priorWait.Step.ActivationSequence != 2 || priorWait.Wait.DeadlineAt != old.Wait.DeadlineAt || priorWait.Wait.ID != old.Wait.ID {
|
|
t.Fatal("wait history retargeted", err)
|
|
}
|
|
assertTwoActivationPagesSDK(t, ctx, api, v.ID, "receipt", *old.Step.ActivationID, *restored.Step.ActivationID)
|
|
stale, err := api.SendWorkflowSignal(ctx, v.ID, workerTestID(t), SendWorkflowSignalRequest{Name: "done", PayloadVersion: "v1", TargetStepKey: "receipt", TargetActivationSequence: 1, CorrelationKey: "shared", Payload: workflowExact, Reason: "late prior generation"})
|
|
if err != nil || stale.Signal.State != "REJECTED" {
|
|
t.Fatal("old signal not fenced", err)
|
|
}
|
|
step, err := api.GetWorkflowStep(ctx, v.ID, "work")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
retryKey := workerTestID(t)
|
|
retry := RetryWorkflowStepRequest{WorkflowStepDecision: workflowDecision(step), SafeToRetry: true}
|
|
child, err := api.RetryWorkflowStep(ctx, v.ID, retryKey, retry)
|
|
if err != nil || child.Step.ActivationSequence != 2 || child.Task.Retry.MaxAttempts != 1 {
|
|
t.Fatal("task budget restored incorrectly", err)
|
|
}
|
|
repeated, err := api.RetryWorkflowStep(ctx, v.ID, retryKey, retry)
|
|
if err != nil || !repeated.Deduplicated || repeated.Task.ID != child.Task.ID {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertTwoActivationPagesSDK(t, ctx, api, v.ID, "work", *step.Step.ActivationID, *child.Step.ActivationID)
|
|
oldTask, err := api.GetWorkflowActivation(ctx, v.ID, "work", *step.Step.ActivationID)
|
|
if err != nil || oldTask.Activation.TaskID == nil || *oldTask.Activation.TaskID != *child.Task.RetryOf || oldTask.Activation.State != "FAILED" {
|
|
t.Fatal("old task activation", err)
|
|
}
|
|
assertWorkflowSourceSDK(t, ctx, api, v, child.Task.ID, 2, 0, *child.Task.RetryOf)
|
|
assertWorkflowSourceSDK(t, ctx, api, v, *child.Task.RetryOf, 1, 0, "")
|
|
continueWorkflowSDK(t, ctx, api, v.ID)
|
|
a := finishWorkflowSDK(t, ctx, w, WorkerSucceeded, workflowExact)
|
|
if a.Task.ID != child.Task.ID {
|
|
t.Fatal("wrong retry task")
|
|
}
|
|
_, err = api.SendWorkflowSignal(ctx, v.ID, workerTestID(t), SendWorkflowSignalRequest{Name: "done", PayloadVersion: "v1", TargetStepKey: "receipt", TargetActivationSequence: 2, CorrelationKey: "shared", Payload: workflowExact, Reason: "verified new wait"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
|
|
})
|
|
t.Run("review_pending_then_continue_original_poll", func(t *testing.T) {
|
|
spec := workflowSDKSpec(queue, handler)
|
|
task := *spec.Body.Steps[0].Task
|
|
task.ResultType = WorkflowType{Kind: "STRING", Enum: []string{"PENDING", "DONE"}}
|
|
spec.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "query", Kind: "POLL", Poll: &WorkflowPoll{Task: task, MaxRounds: 3, TimeoutMS: 60000, IntervalMS: 1000, Pending: []string{"PENDING"}, Succeeded: []string{"DONE"}}}}, OutputType: task.ResultType, Output: WorkflowRef("query")}
|
|
v := startWorkflowSDK(t, ctx, api, spec)
|
|
w := workflowWorkerSDK(t, ctx, api, queue, handler)
|
|
defer w.Close()
|
|
finishWorkflowSDK(t, ctx, w, WorkerUnknown, json.RawMessage(`"unknown"`))
|
|
step := waitWorkflowStepSDK(t, ctx, api, v.ID, "query", func(v WorkflowStepDetail) bool { return v.Step.State == "NEEDS_REVIEW" })
|
|
source, err := api.GetTask(ctx, step.Poll.LastTaskID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertWorkflowSourceSDK(t, ctx, api, v, source.ID, 1, 1, "")
|
|
reviewed, err := api.ReviewWorkflowTask(ctx, v.ID, workerTestID(t), ReviewWorkflowTaskRequest{WorkflowRecoveryDecision: workflowDecision(step).WorkflowRecoveryDecision, TaskID: source.ID, ExpectedTaskRevision: source.Revision, ExecutionSettled: true, Outcome: TaskSucceeded, Result: json.RawMessage(`"PENDING"`)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
step = waitWorkflowStepSDK(t, ctx, api, v.ID, "query", func(v WorkflowStepDetail) bool { return v.Revision > reviewed.Instance.Revision })
|
|
key := workerTestID(t)
|
|
in := ContinueWorkflowPollRequest{WorkflowStepDecision: workflowDecision(step), PendingConfirmed: true}
|
|
out, err := api.ContinueWorkflowPoll(ctx, v.ID, key, in)
|
|
if err != nil || out.Step.ActivationSequence != 1 {
|
|
t.Fatal(err)
|
|
}
|
|
same, err := api.ContinueWorkflowPoll(ctx, v.ID, key, in)
|
|
if err != nil || !same.Deduplicated || same.AuditID != out.AuditID {
|
|
t.Fatal(err)
|
|
}
|
|
continueWorkflowSDK(t, ctx, api, v.ID)
|
|
last := finishWorkflowSDK(t, ctx, w, WorkerSucceeded, json.RawMessage(`"DONE"`))
|
|
assertWorkflowSourceSDK(t, ctx, api, v, last.Task.ID, 1, 2, "")
|
|
assertWorkflowSourceSDK(t, ctx, api, v, source.ID, 1, 1, "")
|
|
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
|
|
final, err := api.GetWorkflowStep(ctx, v.ID, "query")
|
|
if err != nil || final.Poll.UsedRounds != 2 || final.Poll.DeadlineAt != step.Poll.DeadlineAt {
|
|
t.Fatal("poll budget changed", err)
|
|
}
|
|
roundPage, err := api.ListWorkflowPollRounds(ctx, v.ID, "query", *final.Step.ActivationID, HistoryPageRequest{Limit: 1})
|
|
if err != nil || len(roundPage.Items) != 1 || roundPage.Items[0].Round != 2 || roundPage.Items[0].TaskID != last.Task.ID || roundPage.NextCursor == "" {
|
|
t.Fatal("round page", err)
|
|
}
|
|
priorRound, err := api.ListWorkflowPollRounds(ctx, v.ID, "query", *final.Step.ActivationID, HistoryPageRequest{Limit: 1, Cursor: roundPage.NextCursor})
|
|
if err != nil || len(priorRound.Items) != 1 || priorRound.Items[0].Round != 1 || priorRound.Items[0].TaskID != source.ID || priorRound.NextCursor != "" {
|
|
t.Fatal("prior round", err)
|
|
}
|
|
|
|
})
|
|
t.Run("poll_recovery_source_keeps_business_round", func(t *testing.T) {
|
|
spec := workflowSDKSpec(queue, handler)
|
|
task := *spec.Body.Steps[0].Task
|
|
task.ResultType = WorkflowType{Kind: "STRING", Enum: []string{"PENDING", "DONE"}}
|
|
spec.Body = WorkflowBlock{Steps: []WorkflowStep{{Key: "query", Kind: "POLL", Poll: &WorkflowPoll{Task: task, MaxRounds: 3, TimeoutMS: 60000, IntervalMS: 1000, Pending: []string{"PENDING"}, Succeeded: []string{"DONE"}}}}, OutputType: task.ResultType, Output: WorkflowRef("query")}
|
|
v := startWorkflowSDK(t, ctx, api, spec)
|
|
w := workflowWorkerSDK(t, ctx, api, queue, handler)
|
|
defer w.Close()
|
|
old := finishWorkflowSDK(t, ctx, w, WorkerFailed, json.RawMessage(`"failure"`))
|
|
step := waitWorkflowStepSDK(t, ctx, api, v.ID, "query", func(v WorkflowStepDetail) bool { return v.Step.State == "NEEDS_REVIEW" })
|
|
child, err := api.RetryWorkflowStep(ctx, v.ID, workerTestID(t), RetryWorkflowStepRequest{WorkflowStepDecision: workflowDecision(step), SafeToRetry: true})
|
|
if err != nil || child.Task == nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertWorkflowSourceSDK(t, ctx, api, v, child.Task.ID, 2, 1, old.Task.ID)
|
|
assertWorkflowSourceSDK(t, ctx, api, v, old.Task.ID, 1, 1, "")
|
|
continueWorkflowSDK(t, ctx, api, v.ID)
|
|
finishWorkflowSDK(t, ctx, w, WorkerSucceeded, json.RawMessage(`"DONE"`))
|
|
waitWorkflowSDK(t, ctx, api, v.ID, "SUCCEEDED")
|
|
|
|
assertTwoActivationPagesSDK(t, ctx, api, v.ID, "query", *step.Step.ActivationID, *child.Step.ActivationID)
|
|
for _, item := range []struct{ activation, task string }{{*step.Step.ActivationID, old.Task.ID}, {*child.Step.ActivationID, child.Task.ID}} {
|
|
rounds, err := api.ListWorkflowPollRounds(ctx, v.ID, "query", item.activation, HistoryPageRequest{})
|
|
if err != nil || len(rounds.Items) != 1 || rounds.Items[0].Round != 1 || rounds.Items[0].TaskID != item.task {
|
|
t.Fatal("recovery same business round", err)
|
|
}
|
|
}
|
|
assertWorkflowSourceSDK(t, ctx, api, v, child.Task.ID, 2, 1, old.Task.ID)
|
|
})
|
|
if !t.Failed() {
|
|
t.Log("public workflow start/query/control/signal, all six recovery commands, audit and real Worker execution verified without server imports")
|
|
}
|
|
}
|
|
|
|
func assertTwoActivationPagesSDK(t *testing.T, ctx context.Context, api *Client, id, key, old, current string) {
|
|
t.Helper()
|
|
first, err := api.ListWorkflowActivations(ctx, id, key, HistoryPageRequest{Limit: 1})
|
|
if err != nil || len(first.Items) != 1 || first.Items[0].ID != current || first.Items[0].Sequence != 2 || first.NextCursor == "" {
|
|
t.Fatal("current activation page", err)
|
|
}
|
|
second, err := api.ListWorkflowActivations(ctx, id, key, HistoryPageRequest{Limit: 1, Cursor: first.NextCursor})
|
|
if err != nil || len(second.Items) != 1 || second.Items[0].ID != old || second.Items[0].Sequence != 1 || second.NextCursor != "" {
|
|
t.Fatal("old activation page", err)
|
|
}
|
|
_, err = api.ListWorkflowSignals(ctx, id, HistoryPageRequest{Cursor: first.NextCursor})
|
|
workflowAPIError(t, err, "INVALID_ARGUMENT")
|
|
}
|