fix: bind workflow samples to distinct task execution proofs
This commit is contained in:
87
sdk/go/cmd/orbit-workflow-load/handler.go
Normal file
87
sdk/go/cmd/orbit-workflow-load/handler.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// invocation 固定首次本地调用及其尝试身份。
|
||||
type invocation struct {
|
||||
sequence int // sequence 输入中的实例序号。
|
||||
attemptID string // attemptID 首次调用的尝试身份。
|
||||
calls int // calls 同一任务实际调用次数。
|
||||
}
|
||||
|
||||
// invocationLog 按任务身份记录,不能用同一分支的两次调用抵充两个分支。
|
||||
type invocationLog struct {
|
||||
mu sync.Mutex // mu 保护并发Worker回调。
|
||||
limit int // limit 最多保存两倍计划实例数的任务身份。
|
||||
calls map[string]*invocation // calls 固定身份到首次尝试和计数。
|
||||
invalid int // invalid 不符合输入约束的调用。
|
||||
overflow int // overflow 超过身份预算的调用。
|
||||
}
|
||||
|
||||
func newInvocationLog(n int) *invocationLog {
|
||||
return &invocationLog{limit: 2 * n, calls: map[string]*invocation{}}
|
||||
}
|
||||
func (l *invocationLog) handler(run context.Context, c config, runID string) orbit.WorkerHandler {
|
||||
return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
p, ok := decodePayload(e.Task.Payload)
|
||||
if !ok || p.Sequence < 0 || p.Sequence >= c.Requests || !matchesTask(e.Task, c, runID, p.Sequence) || !validID(e.AttemptID) || e.Epoch != 1 || e.ConcurrencyToken != 0 {
|
||||
l.mu.Lock()
|
||||
l.invalid++
|
||||
l.mu.Unlock()
|
||||
return nil, &orbit.WorkerFailure{Code: "WORKFLOW_INPUT_MISMATCH"}
|
||||
}
|
||||
l.mu.Lock()
|
||||
if previous := l.calls[e.Task.ID]; previous != nil {
|
||||
previous.calls++
|
||||
l.mu.Unlock()
|
||||
return nil, &orbit.WorkerFailure{Code: "DUPLICATE_INVOCATION"}
|
||||
}
|
||||
if len(l.calls) >= l.limit {
|
||||
l.overflow++
|
||||
l.mu.Unlock()
|
||||
return nil, &orbit.WorkerFailure{Code: "INVOCATION_LIMIT"}
|
||||
}
|
||||
l.calls[e.Task.ID] = &invocation{sequence: p.Sequence, attemptID: e.AttemptID, calls: 1}
|
||||
l.mu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-run.Done():
|
||||
return nil, run.Err()
|
||||
default:
|
||||
}
|
||||
return e.Task.Payload, nil
|
||||
}}
|
||||
}
|
||||
func (l *invocationLog) summarize(r *workflowReport, proofs []verifiedTask) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
known := map[string]verifiedTask{}
|
||||
for _, p := range proofs {
|
||||
if _, exists := known[p.taskID]; exists {
|
||||
r.Errors["DUPLICATE_VERIFIED_TASK"]++
|
||||
}
|
||||
known[p.taskID] = p
|
||||
call := l.calls[p.taskID]
|
||||
if call == nil || call.sequence != p.sequence || call.attemptID != p.attemptID {
|
||||
r.Errors["LOCAL_PROOF_MISMATCH"]++
|
||||
}
|
||||
}
|
||||
for id, call := range l.calls {
|
||||
r.HandlerCalls += call.calls
|
||||
if call.calls > 1 {
|
||||
r.DuplicateInvocations += call.calls - 1
|
||||
}
|
||||
p, ok := known[id]
|
||||
if !ok || p.sequence != call.sequence || p.attemptID != call.attemptID {
|
||||
r.UnconfirmedInvocations += call.calls
|
||||
}
|
||||
}
|
||||
r.Errors["INVALID_HANDLER_INPUT"] += l.invalid
|
||||
r.Errors["HANDLER_IDENTITY_LIMIT"] += l.overflow
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -18,23 +17,6 @@ type workflowPayload struct {
|
||||
Sequence int `json:"sequence"` // Sequence 从0开始的实例序号,用于发现错配和重复调用。
|
||||
}
|
||||
|
||||
func decodePayload(raw json.RawMessage) (workflowPayload, bool) {
|
||||
var wire struct {
|
||||
RunID *string `json:"run_id"`
|
||||
Sequence *int `json:"sequence"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&wire) != nil || wire.RunID == nil || wire.Sequence == nil {
|
||||
return workflowPayload{}, false
|
||||
}
|
||||
var trailing any
|
||||
if decoder.Decode(&trailing) != io.EOF {
|
||||
return workflowPayload{}, false
|
||||
}
|
||||
return workflowPayload{RunID: *wire.RunID, Sequence: *wire.Sequence}, true
|
||||
}
|
||||
|
||||
func handlerName(runID string) string { return "load.workflow." + runID }
|
||||
func definitionKey(runID string) string { return "orbit-workflow-load-" + runID }
|
||||
func requestKey(runID string, sequence int) string {
|
||||
@@ -67,38 +49,7 @@ type workflowVerification struct {
|
||||
sequence int
|
||||
duration time.Duration
|
||||
code string
|
||||
}
|
||||
|
||||
// invocationLog 有界保存每个流程的实际处理器调用,两个并行分支各应调用一次。
|
||||
type invocationLog struct {
|
||||
mu sync.Mutex
|
||||
calls []int
|
||||
invalid int
|
||||
}
|
||||
|
||||
func newInvocationLog(n int) *invocationLog { return &invocationLog{calls: make([]int, n)} }
|
||||
|
||||
func (log *invocationLog) handler(ctx context.Context, cfg config, runID string) orbit.WorkerHandler {
|
||||
return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(work context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
payload, valid := decodePayload(execution.Task.Payload)
|
||||
if !valid || payload.RunID != runID || payload.Sequence < 0 || payload.Sequence >= cfg.Requests || execution.Task.Handler != handlerName(runID) || execution.Task.HandlerVersion != "v1" || execution.Task.PayloadVersion != "v1" || execution.Task.QueueID != cfg.QueueID {
|
||||
log.mu.Lock()
|
||||
log.invalid++
|
||||
log.mu.Unlock()
|
||||
return nil, &orbit.WorkerFailure{Code: "WORKFLOW_INPUT_MISMATCH"}
|
||||
}
|
||||
log.mu.Lock()
|
||||
log.calls[payload.Sequence]++
|
||||
log.mu.Unlock()
|
||||
select {
|
||||
case <-work.Done():
|
||||
return nil, work.Err()
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
return execution.Task.Payload, nil
|
||||
}}
|
||||
tasks []verifiedTask
|
||||
}
|
||||
|
||||
func workflowSpec(queue, handler string, timeout time.Duration) orbit.WorkflowSpecification {
|
||||
@@ -128,16 +79,12 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
|
||||
return r.finish()
|
||||
}
|
||||
r.Queue = &queue
|
||||
if queue.ID != cfg.QueueID || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
|
||||
if queue.ID != cfg.QueueID || !validID(queue.NamespaceID) || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
|
||||
r.Errors["QUEUE_UNAVAILABLE"]++
|
||||
return r.finish()
|
||||
}
|
||||
handler := handlerName(runID)
|
||||
typicalTimeout := cfg.RunTimeout
|
||||
if typicalTimeout > 30*time.Second {
|
||||
typicalTimeout = 30 * time.Second
|
||||
}
|
||||
spec := workflowSpec(cfg.QueueID, handler, typicalTimeout)
|
||||
spec := workflowSpec(cfg.QueueID, handler, cfg.RunTimeout)
|
||||
requestCtx, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
definition, err := api.CreateWorkflowDefinition(requestCtx, definitionKey(runID), orbit.CreateWorkflowDefinitionRequest{Name: definitionKey(runID), Description: "bounded parallel workflow verification"})
|
||||
stop()
|
||||
@@ -146,7 +93,7 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
|
||||
return r.finish()
|
||||
}
|
||||
r.DefinitionID = definition.Definition.ID
|
||||
if !validID(r.DefinitionID) {
|
||||
if !validID(r.DefinitionID) || definition.Deduplicated || definition.Definition.NamespaceID != queue.NamespaceID || definition.Definition.Name != definitionKey(runID) {
|
||||
r.Errors["INVALID_DEFINITION_RECEIPT"]++
|
||||
return r.finish()
|
||||
}
|
||||
@@ -158,7 +105,7 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
|
||||
return r.finish()
|
||||
}
|
||||
r.VersionID = version.Version.ID
|
||||
if !validID(r.VersionID) || version.Version.Version != "v1" || version.Version.DefinitionID != r.DefinitionID {
|
||||
if !validID(r.VersionID) || version.Version.Version != "v1" || version.Version.DefinitionID != r.DefinitionID || version.Deduplicated || version.Version.NamespaceID != queue.NamespaceID || !sameSpecification(version.Version.Specification, spec) {
|
||||
r.Errors["INVALID_VERSION_RECEIPT"]++
|
||||
return r.finish()
|
||||
}
|
||||
@@ -185,13 +132,21 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
|
||||
}(worker)
|
||||
}
|
||||
|
||||
admissions := submitWorkflows(ctx, cfg, api, runID, r.DefinitionID)
|
||||
admissions := submitWorkflows(ctx, cfg, api, runID, r.DefinitionID, r.VersionID, queue.NamespaceID)
|
||||
r.Attempted = len(admissions)
|
||||
r.NotSubmitted = cfg.Requests - r.Attempted
|
||||
var accepted []workflowAdmission
|
||||
var admissionDurations []time.Duration
|
||||
seenInstances := map[string]bool{}
|
||||
for _, a := range admissions {
|
||||
admissionDurations = append(admissionDurations, a.duration)
|
||||
if a.code == "" && seenInstances[a.instance.ID] {
|
||||
a.code = "DUPLICATE_INSTANCE"
|
||||
}
|
||||
if a.code == "" {
|
||||
seenInstances[a.instance.ID] = true
|
||||
}
|
||||
r.Instances = append(r.Instances, instanceReport{Sequence: a.sequence, Key: requestKey(runID, a.sequence), ID: a.instance.ID, Code: a.code})
|
||||
if a.code != "" {
|
||||
r.AdmissionErrors++
|
||||
r.Errors["ADMISSION_"+a.code]++
|
||||
@@ -204,12 +159,20 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
|
||||
r.AdmissionFinishedAt = time.Now().UTC()
|
||||
verified := verifyWorkflows(ctx, cfg, api, runID, r.DefinitionID, accepted)
|
||||
var completions []time.Duration
|
||||
var proven []verifiedTask
|
||||
for _, v := range verified {
|
||||
if v.code != "" {
|
||||
r.Errors[v.code]++
|
||||
continue
|
||||
}
|
||||
r.VerifiedSuccess++
|
||||
proven = append(proven, v.tasks...)
|
||||
for i := range r.Instances {
|
||||
if r.Instances[i].Sequence == v.sequence {
|
||||
r.Instances[i].Verified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
completions = append(completions, v.duration)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
@@ -227,23 +190,12 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
|
||||
r.Reconnects += stats.Reconnects
|
||||
r.Renewals += stats.RenewalsConfirmed
|
||||
}
|
||||
log.mu.Lock()
|
||||
for _, calls := range log.calls {
|
||||
r.HandlerCalls += calls
|
||||
if calls > 2 {
|
||||
r.DuplicateInvocations += calls - 2
|
||||
}
|
||||
if calls != 2 {
|
||||
r.Errors["HANDLER_CALL_MISMATCH"]++
|
||||
}
|
||||
}
|
||||
r.Errors["INVALID_HANDLER_INPUT"] += log.invalid
|
||||
log.mu.Unlock()
|
||||
log.summarize(&r, proven)
|
||||
r.Completion = distributionOf(completions)
|
||||
return r.finish()
|
||||
}
|
||||
|
||||
func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string) []workflowAdmission {
|
||||
func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, definitionID, versionID, namespace string) []workflowAdmission {
|
||||
jobs := make(chan int)
|
||||
out := make(chan workflowAdmission, cfg.Requests)
|
||||
var wg sync.WaitGroup
|
||||
@@ -263,7 +215,7 @@ func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, de
|
||||
a := workflowAdmission{sequence: sequence, instance: result.Instance, duration: time.Since(started)}
|
||||
if err != nil {
|
||||
a.code = classify(err)
|
||||
} else if result.Deduplicated || !validID(result.Instance.ID) || result.Instance.DefinitionID != definitionID || result.Instance.Version != "v1" || result.Instance.BusinessKey != businessKey(runID, sequence) {
|
||||
} else if result.Deduplicated || !matchesInstance(result.Instance, cfg, runID, sequence, definitionID, versionID, namespace) {
|
||||
a.code = "INVALID_RECEIPT"
|
||||
}
|
||||
out <- a
|
||||
@@ -301,8 +253,8 @@ func verifyWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, de
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for admission := range jobs {
|
||||
duration, code := verifyWorkflow(ctx, cfg, api, runID, definitionID, admission)
|
||||
out <- workflowVerification{sequence: admission.sequence, duration: duration, code: code}
|
||||
duration, tasks, code := verifyWorkflow(ctx, cfg, api, runID, definitionID, admission)
|
||||
out <- workflowVerification{sequence: admission.sequence, duration: duration, tasks: tasks, code: code}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -315,110 +267,61 @@ func verifyWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, de
|
||||
return results
|
||||
}
|
||||
|
||||
func verifyWorkflow(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string, admission workflowAdmission) (time.Duration, string) {
|
||||
func verifyWorkflow(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string, admission workflowAdmission) (time.Duration, []verifiedTask, string) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return 0, "VERIFY_" + classify(ctx.Err())
|
||||
return 0, nil, "VERIFY_" + classify(ctx.Err())
|
||||
}
|
||||
call, stop := context.WithTimeout(ctx, cfg.Timeout)
|
||||
instance, err := api.GetWorkflow(call, admission.instance.ID)
|
||||
stop()
|
||||
if err != nil {
|
||||
return 0, "READ_" + classify(err)
|
||||
return 0, nil, "READ_" + classify(err)
|
||||
}
|
||||
if instance.ID != admission.instance.ID || instance.DefinitionID != definitionID || instance.Version != "v1" || instance.BusinessKey != businessKey(runID, admission.sequence) {
|
||||
return 0, "WORKFLOW_MISMATCH"
|
||||
if instance.ID != admission.instance.ID || !matchesInstance(instance, cfg, runID, admission.sequence, definitionID, admission.instance.DefinitionVersionID, admission.instance.NamespaceID) || !instance.CreatedAt.Equal(admission.instance.CreatedAt) || !instance.DeadlineAt.Equal(admission.instance.DeadlineAt) {
|
||||
return 0, nil, "WORKFLOW_MISMATCH"
|
||||
}
|
||||
switch instance.State {
|
||||
case "SUCCEEDED":
|
||||
if instance.CompletedAt == nil || instance.CreatedAt.IsZero() || instance.CompletedAt.Before(instance.CreatedAt) {
|
||||
return 0, "COMPLETION_MISMATCH"
|
||||
return 0, nil, "COMPLETION_MISMATCH"
|
||||
}
|
||||
call, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
steps, e := api.GetWorkflowSteps(call, instance.ID)
|
||||
stop()
|
||||
if e != nil {
|
||||
return 0, "STEPS_" + classify(e)
|
||||
return 0, nil, "STEPS_" + classify(e)
|
||||
}
|
||||
if code := verifySteps(ctx, cfg, api, instance, runID, admission.sequence, steps); code != "" {
|
||||
return 0, code
|
||||
tasks, code := verifySteps(ctx, cfg, api, instance, runID, admission.sequence, steps)
|
||||
if code != "" {
|
||||
return 0, nil, code
|
||||
}
|
||||
if !matchesOutput(instance.Output, runID, admission.sequence) {
|
||||
return 0, "OUTPUT_MISMATCH"
|
||||
return 0, nil, "OUTPUT_MISMATCH"
|
||||
}
|
||||
return instance.CompletedAt.Sub(instance.CreatedAt), ""
|
||||
return instance.CompletedAt.Sub(instance.CreatedAt), tasks, ""
|
||||
case "NEEDS_REVIEW", "FAILED", "CANCELLED":
|
||||
return 0, "WORKFLOW_" + instance.State
|
||||
return 0, nil, "WORKFLOW_" + instance.State
|
||||
case "RUNNING", "WAITING", "PAUSED", "PENDING":
|
||||
default:
|
||||
return 0, "UNKNOWN_WORKFLOW_STATE"
|
||||
return 0, nil, "UNKNOWN_WORKFLOW_STATE"
|
||||
}
|
||||
timer := time.NewTimer(cfg.PollInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return 0, "VERIFY_" + classify(ctx.Err())
|
||||
return 0, nil, "VERIFY_" + classify(ctx.Err())
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifySteps(ctx context.Context, cfg config, api workflowAPI, instance orbit.WorkflowInstance, runID string, sequence int, steps orbit.WorkflowSteps) string {
|
||||
var taskSteps []orbit.WorkflowStepStatus
|
||||
for _, step := range steps.Steps {
|
||||
if step.Kind == "TASK" {
|
||||
taskSteps = append(taskSteps, step)
|
||||
}
|
||||
}
|
||||
if len(taskSteps) != 2 {
|
||||
return "STEP_COUNT_MISMATCH"
|
||||
}
|
||||
for _, step := range taskSteps {
|
||||
if step.State != "SUCCEEDED" || step.TaskID == nil || !validID(*step.TaskID) || step.ActivationSequence != 1 || step.ActivationID == nil {
|
||||
return "STEP_MISMATCH"
|
||||
}
|
||||
call, stop := context.WithTimeout(ctx, cfg.Timeout)
|
||||
task, err := api.GetTask(call, *step.TaskID)
|
||||
stop()
|
||||
if err != nil {
|
||||
return "TASK_" + classify(err)
|
||||
}
|
||||
payload, valid := decodePayload(task.Result)
|
||||
if task.ID != *step.TaskID || task.State != orbit.TaskSucceeded || task.EffectUncertain || task.AttemptCount != 1 || !valid || payload != (workflowPayload{RunID: runID, Sequence: sequence}) {
|
||||
return "TASK_MISMATCH"
|
||||
}
|
||||
call, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
|
||||
stop()
|
||||
if err != nil {
|
||||
return "HISTORY_" + classify(err)
|
||||
}
|
||||
if len(history.Items) != 1 || history.NextCursor != "" || history.Items[0].TaskID != task.ID || history.Items[0].State != orbit.AttemptSucceeded {
|
||||
return "ATTEMPT_MISMATCH"
|
||||
}
|
||||
call, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
source, err := api.GetTaskSource(call, task.ID)
|
||||
stop()
|
||||
if err != nil || source.Kind != "WORKFLOW" || source.Workflow == nil || source.Workflow.WorkflowID != instance.ID || source.Workflow.DefinitionID != instance.DefinitionID || source.Workflow.DefinitionVersionID != instance.DefinitionVersionID || source.Workflow.Version != instance.Version || source.Workflow.StepID != step.ID || source.Workflow.ActivationID != *step.ActivationID || source.Workflow.ActivationSequence != 1 {
|
||||
return "SOURCE_MISMATCH"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
func sameSpecification(a, b orbit.WorkflowSpecification) bool {
|
||||
left, e := json.Marshal(a)
|
||||
right, e2 := json.Marshal(b)
|
||||
return e == nil && e2 == nil && bytes.Equal(left, right)
|
||||
}
|
||||
|
||||
func matchesOutput(raw json.RawMessage, runID string, sequence int) bool {
|
||||
var wire struct {
|
||||
Left workflowPayload `json:"left"`
|
||||
Right workflowPayload `json:"right"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&wire) != nil {
|
||||
return false
|
||||
}
|
||||
var trailing any
|
||||
if decoder.Decode(&trailing) != io.EOF {
|
||||
return false
|
||||
}
|
||||
return wire.Left == (workflowPayload{RunID: runID, Sequence: sequence}) && wire.Right == wire.Left
|
||||
func matchesInstance(i orbit.WorkflowInstance, c config, runID string, sequence int, definitionID, versionID, namespace string) bool {
|
||||
p, ok := decodePayload(i.Input)
|
||||
return validID(i.ID) && i.NamespaceID == namespace && i.DefinitionID == definitionID && i.DefinitionVersionID == versionID && i.Version == "v1" && i.BusinessKey == businessKey(runID, sequence) && ok && p == (workflowPayload{RunID: runID, Sequence: sequence}) && !i.CreatedAt.IsZero() && i.DeadlineAt.Equal(i.CreatedAt.Add(time.Duration(c.RunTimeout.Milliseconds())*time.Millisecond)) && !i.Paused && i.CancelRequestedAt == nil && i.StopKind == nil && i.StopRequestedAt == nil
|
||||
}
|
||||
|
||||
64
sdk/go/cmd/orbit-workflow-load/payload.go
Normal file
64
sdk/go/cmd/orbit-workflow-load/payload.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
)
|
||||
|
||||
// exactFields 验证精确字段集合;原文重复键、大小写别名和尾随JSON均拒绝。
|
||||
func exactFields(raw json.RawMessage, names ...string) (map[string]json.RawMessage, bool) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
token, err := decoder.Token()
|
||||
if err != nil || token != json.Delim('{') {
|
||||
return nil, false
|
||||
}
|
||||
fields := make(map[string]json.RawMessage, len(names))
|
||||
for decoder.More() {
|
||||
token, err = decoder.Token()
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
key, ok := token.(string)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
allowed := false
|
||||
for _, name := range names {
|
||||
allowed = allowed || name == key
|
||||
}
|
||||
if !allowed {
|
||||
return nil, false
|
||||
}
|
||||
if _, exists := fields[key]; exists {
|
||||
return nil, false
|
||||
}
|
||||
var value json.RawMessage
|
||||
if decoder.Decode(&value) != nil || bytes.Equal(bytes.TrimSpace(value), []byte("null")) {
|
||||
return nil, false
|
||||
}
|
||||
fields[key] = value
|
||||
}
|
||||
if _, err = decoder.Token(); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
var tail any
|
||||
return fields, len(fields) == len(names) && decoder.Decode(&tail) == io.EOF
|
||||
}
|
||||
func decodePayload(raw json.RawMessage) (workflowPayload, bool) {
|
||||
fields, ok := exactFields(raw, "run_id", "sequence")
|
||||
var out workflowPayload
|
||||
if !ok || json.Unmarshal(fields["run_id"], &out.RunID) != nil || json.Unmarshal(fields["sequence"], &out.Sequence) != nil {
|
||||
return out, false
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
func matchesOutput(raw json.RawMessage, runID string, sequence int) bool {
|
||||
fields, ok := exactFields(raw, "left", "right")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
left, ok := decodePayload(fields["left"])
|
||||
right, ok2 := decodePayload(fields["right"])
|
||||
return ok && ok2 && left == (workflowPayload{RunID: runID, Sequence: sequence}) && right == left
|
||||
}
|
||||
@@ -18,44 +18,55 @@ type distribution struct {
|
||||
Max *float64 `json:"max_ms"` // Max 纳入样本的最大毫秒数。
|
||||
}
|
||||
|
||||
// instanceReport 保存每个启动请求的原身份,便于未知结果按原键核对。
|
||||
type instanceReport struct {
|
||||
Sequence int `json:"sequence"` // Sequence 从零开始的请求序号。
|
||||
Key string `json:"key"` // Key 启动幂等键,同时作为业务键。
|
||||
ID string `json:"id,omitempty"` // ID 已取得的实例身份;为空不证明未提交。
|
||||
Code string `json:"admission_error,omitempty"` // Code 受理错误分类,不包含原异常正文。
|
||||
Verified bool `json:"verified"` // Verified 是否通过持久结果核验。
|
||||
}
|
||||
|
||||
// workflowReport 是一次有界流程测量结果,不是容量承诺或生产高可用验收。
|
||||
type workflowReport struct {
|
||||
RunID string `json:"run_id"` // RunID 本次定义、实例和处理器身份的隔离前缀。
|
||||
GoVersion string `json:"go_version"` // GoVersion 实际编译工具链版本。
|
||||
GOOS string `json:"goos"` // GOOS 负载进程操作系统。
|
||||
GOARCH string `json:"goarch"` // GOARCH 负载进程架构。
|
||||
StartedAt time.Time `json:"started_at"` // StartedAt 从预检开始的本机时间。
|
||||
AdmissionFinishedAt time.Time `json:"admission_finished_at"` // AdmissionFinishedAt 实例受理组完成时间。
|
||||
FinishedAt time.Time `json:"finished_at"` // FinishedAt 核验及Worker收尾完成时间。
|
||||
Requests int `json:"requests"` // Requests 计划启动的流程实例数。
|
||||
Concurrency int `json:"concurrency"` // Concurrency 启动和核验请求的最大并发。
|
||||
Workers int `json:"workers"` // Workers 本进程独立WSS Worker数。
|
||||
Capacity int `json:"capacity_per_worker"` // Capacity 每个Worker的执行槽上限。
|
||||
RunTimeoutMS int64 `json:"run_timeout_ms"` // RunTimeoutMS 整次测量的截止时间。
|
||||
PollIntervalMS int64 `json:"poll_interval_ms"` // PollIntervalMS 流程状态轮询间隔。
|
||||
HTTPTimeoutMS int64 `json:"http_timeout_ms"` // HTTPTimeoutMS 单次HTTP交换截止时间。
|
||||
Queue *orbit.Queue `json:"queue,omitempty"` // Queue 预检时的队列快照,不修改配置。
|
||||
DefinitionID string `json:"definition_id,omitempty"` // DefinitionID 本次新建的稳定定义身份。
|
||||
VersionID string `json:"version_id,omitempty"` // VersionID 本次发布的不可变版本身份。
|
||||
Attempted int `json:"attempted"` // Attempted 实际发起的启动调用数。
|
||||
NotSubmitted int `json:"not_submitted"` // NotSubmitted 截止前未发起的实例数。
|
||||
Accepted int `json:"accepted"` // Accepted 收到并核验唯一启动回执的实例数。
|
||||
AdmissionErrors int `json:"admission_errors"` // AdmissionErrors 明确拒绝或回执不合法的实例数。
|
||||
VerifiedSuccess int `json:"verified_success"` // VerifiedSuccess 流程、步骤、任务、来源和结果均核验通过数。
|
||||
Unverified int `json:"unverified"` // Unverified 已受理但没有完成全部核验的实例数。
|
||||
HandlerCalls int `json:"handler_calls"` // HandlerCalls 有效处理器调用总数,两个分支各一次。
|
||||
DuplicateInvocations int `json:"duplicate_invocations"` // DuplicateInvocations 超过两个分支预期的额外调用数。
|
||||
WorkerStarted uint64 `json:"worker_started"` // WorkerStarted SDK本进程领取并启动的任务数。
|
||||
Reconnects uint64 `json:"worker_reconnects"` // Reconnects SDK重连次数。
|
||||
Renewals uint64 `json:"worker_renewals_confirmed"` // Renewals SDK确认的租约续期次数。
|
||||
Admission distribution `json:"admission"` // Admission 启动请求本机往返耗时。
|
||||
Completion distribution `json:"completion"` // Completion 数据库实例创建至完成的服务端耗时。
|
||||
Errors map[string]int `json:"errors"` // Errors 稳定错误分类,不写入凭据、SQL或请求正文。
|
||||
Complete bool `json:"complete"` // Complete 仅全部实例和两倍处理器调用逐项通过核验时为真。
|
||||
Instances []instanceReport `json:"instances"` // Instances 已尝试启动的请求身份,包含失败或未知结果。
|
||||
UnconfirmedInvocations int `json:"unconfirmed_invocations"` // UnconfirmedInvocations 尚未关联到已核验分支任务/尝试的调用数。
|
||||
RunID string `json:"run_id"` // RunID 本次定义、实例和处理器身份的隔离前缀。
|
||||
GoVersion string `json:"go_version"` // GoVersion 实际编译工具链版本。
|
||||
GOOS string `json:"goos"` // GOOS 负载进程操作系统。
|
||||
GOARCH string `json:"goarch"` // GOARCH 负载进程架构。
|
||||
StartedAt time.Time `json:"started_at"` // StartedAt 从预检开始的本机时间。
|
||||
AdmissionFinishedAt time.Time `json:"admission_finished_at"` // AdmissionFinishedAt 实例受理组完成时间。
|
||||
FinishedAt time.Time `json:"finished_at"` // FinishedAt 核验及Worker收尾完成时间。
|
||||
Requests int `json:"requests"` // Requests 计划启动的流程实例数。
|
||||
Concurrency int `json:"concurrency"` // Concurrency 启动和核验请求的最大并发。
|
||||
Workers int `json:"workers"` // Workers 本进程独立WSS Worker数。
|
||||
Capacity int `json:"capacity_per_worker"` // Capacity 每个Worker的执行槽上限。
|
||||
RunTimeoutMS int64 `json:"run_timeout_ms"` // RunTimeoutMS 整次测量的截止时间。
|
||||
PollIntervalMS int64 `json:"poll_interval_ms"` // PollIntervalMS 流程状态轮询间隔。
|
||||
HTTPTimeoutMS int64 `json:"http_timeout_ms"` // HTTPTimeoutMS 单次HTTP交换截止时间。
|
||||
Queue *orbit.Queue `json:"queue,omitempty"` // Queue 预检时的队列快照,不修改配置。
|
||||
DefinitionID string `json:"definition_id,omitempty"` // DefinitionID 本次新建的稳定定义身份。
|
||||
VersionID string `json:"version_id,omitempty"` // VersionID 本次发布的不可变版本身份。
|
||||
Attempted int `json:"attempted"` // Attempted 实际发起的启动调用数。
|
||||
NotSubmitted int `json:"not_submitted"` // NotSubmitted 截止前未发起的实例数。
|
||||
Accepted int `json:"accepted"` // Accepted 收到并核验唯一启动回执的实例数。
|
||||
AdmissionErrors int `json:"admission_errors"` // AdmissionErrors 明确拒绝或回执不合法的实例数。
|
||||
VerifiedSuccess int `json:"verified_success"` // VerifiedSuccess 流程、步骤、任务、来源和结果均核验通过数。
|
||||
Unverified int `json:"unverified"` // Unverified 已受理但没有完成全部核验的实例数。
|
||||
HandlerCalls int `json:"handler_calls"` // HandlerCalls 有效处理器调用总数,两个分支各一次。
|
||||
DuplicateInvocations int `json:"duplicate_invocations"` // DuplicateInvocations 同一任务的额外调用数,不能抵充另一分支。
|
||||
WorkerStarted uint64 `json:"worker_started"` // WorkerStarted SDK本进程领取并启动的任务数。
|
||||
Reconnects uint64 `json:"worker_reconnects"` // Reconnects SDK重连次数。
|
||||
Renewals uint64 `json:"worker_renewals_confirmed"` // Renewals SDK确认的租约续期次数。
|
||||
Admission distribution `json:"admission"` // Admission 启动请求本机往返耗时。
|
||||
Completion distribution `json:"completion"` // Completion 数据库实例创建至完成的服务端耗时。
|
||||
Errors map[string]int `json:"errors"` // Errors 稳定错误分类,不写入凭据、SQL或请求正文。
|
||||
Complete bool `json:"complete"` // Complete 仅全部实例和两倍处理器调用逐项通过核验时为真。
|
||||
}
|
||||
|
||||
func newReport(cfg config, runID string) workflowReport {
|
||||
return workflowReport{RunID: runID, GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, StartedAt: time.Now().UTC(), Requests: cfg.Requests, NotSubmitted: cfg.Requests, Concurrency: cfg.Concurrency, Workers: cfg.Workers, Capacity: cfg.Capacity, RunTimeoutMS: cfg.RunTimeout.Milliseconds(), PollIntervalMS: cfg.PollInterval.Milliseconds(), HTTPTimeoutMS: cfg.Timeout.Milliseconds(), Errors: map[string]int{}}
|
||||
return workflowReport{Instances: []instanceReport{}, RunID: runID, GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, StartedAt: time.Now().UTC(), Requests: cfg.Requests, NotSubmitted: cfg.Requests, Concurrency: cfg.Concurrency, Workers: cfg.Workers, Capacity: cfg.Capacity, RunTimeoutMS: cfg.RunTimeout.Milliseconds(), PollIntervalMS: cfg.PollInterval.Milliseconds(), HTTPTimeoutMS: cfg.Timeout.Milliseconds(), Errors: map[string]int{}}
|
||||
}
|
||||
|
||||
func (r workflowReport) finish() workflowReport {
|
||||
@@ -66,7 +77,7 @@ func (r workflowReport) finish() workflowReport {
|
||||
}
|
||||
r.FinishedAt = time.Now().UTC()
|
||||
r.Unverified = r.Accepted - r.VerifiedSuccess
|
||||
r.Complete = r.Attempted == r.Requests && r.NotSubmitted == 0 && r.AdmissionErrors == 0 && r.Accepted == r.Requests && r.VerifiedSuccess == r.Requests && r.HandlerCalls == 2*r.Requests && r.WorkerStarted == uint64(2*r.Requests) && r.DuplicateInvocations == 0 && len(r.Errors) == 0
|
||||
r.Complete = r.Attempted == r.Requests && r.NotSubmitted == 0 && r.AdmissionErrors == 0 && r.Accepted == r.Requests && r.VerifiedSuccess == r.Requests && r.HandlerCalls == 2*r.Requests && r.WorkerStarted == uint64(2*r.Requests) && r.DuplicateInvocations == 0 && r.UnconfirmedInvocations == 0 && len(r.Errors) == 0
|
||||
return r
|
||||
}
|
||||
|
||||
|
||||
90
sdk/go/cmd/orbit-workflow-load/verify.go
Normal file
90
sdk/go/cmd/orbit-workflow-load/verify.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
// verifiedTask 保存已核验分支的任务和尝试,最终与本地调用逐个关联。
|
||||
type verifiedTask struct {
|
||||
sequence int // sequence 实例请求序号。
|
||||
taskID string // taskID 分支唯一任务身份。
|
||||
attemptID string // attemptID 唯一成功尝试身份。
|
||||
}
|
||||
|
||||
func verifySteps(ctx context.Context, cfg config, api workflowAPI, instance orbit.WorkflowInstance, runID string, sequence int, steps orbit.WorkflowSteps) ([]verifiedTask, string) {
|
||||
if steps.InstanceID != instance.ID || steps.Revision != instance.Revision || len(steps.Steps) != 3 {
|
||||
return nil, "STEP_COUNT_OR_PARENT_MISMATCH"
|
||||
}
|
||||
keys, ids, tasks, activations := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
for _, s := range steps.Steps {
|
||||
kind := map[string]string{"pair": "PARALLEL", "left": "TASK", "right": "TASK"}[s.Key]
|
||||
if kind == "" || s.Kind != kind || s.State != "SUCCEEDED" || !validID(s.ID) || s.Revision == 0 || ids[s.ID] || keys[s.Key] || s.ActivationSequence != 1 || s.ActivationID == nil || !validID(*s.ActivationID) || activations[*s.ActivationID] {
|
||||
return nil, "STEP_MISMATCH"
|
||||
}
|
||||
keys[s.Key] = true
|
||||
ids[s.ID] = true
|
||||
activations[*s.ActivationID] = true
|
||||
if kind == "PARALLEL" {
|
||||
if s.TaskID != nil {
|
||||
return nil, "JOIN_TASK_MISMATCH"
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.TaskID == nil || !validID(*s.TaskID) || tasks[*s.TaskID] {
|
||||
return nil, "DUPLICATE_OR_INVALID_TASK"
|
||||
}
|
||||
tasks[*s.TaskID] = true
|
||||
}
|
||||
proofs := make([]verifiedTask, 0, 2)
|
||||
for _, step := range steps.Steps {
|
||||
if step.Kind != "TASK" {
|
||||
continue
|
||||
}
|
||||
call, stop := context.WithTimeout(ctx, cfg.Timeout)
|
||||
task, err := api.GetTask(call, *step.TaskID)
|
||||
stop()
|
||||
if err != nil {
|
||||
return nil, "TASK_" + classify(err)
|
||||
}
|
||||
result, ok := decodePayload(task.Result)
|
||||
if task.ID != *step.TaskID || !matchesTask(task, cfg, runID, sequence) || task.NamespaceID != instance.NamespaceID || task.State != orbit.TaskSucceeded || task.EffectUncertain || task.AttemptCount != 1 || task.CompletedAt == nil || task.CancelRequestedAt != nil || !ok || result != (workflowPayload{RunID: runID, Sequence: sequence}) {
|
||||
return nil, "TASK_MISMATCH"
|
||||
}
|
||||
if task.CreatedAt.Before(instance.CreatedAt) || !task.AvailableAt.Equal(task.CreatedAt) || !task.DeadlineAt.Equal(instance.DeadlineAt) || task.CompletedAt.Before(task.CreatedAt) || instance.CompletedAt == nil || task.CompletedAt.After(*instance.CompletedAt) {
|
||||
return nil, "TASK_TIMESTAMP_MISMATCH"
|
||||
}
|
||||
call, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
|
||||
stop()
|
||||
if err != nil {
|
||||
return nil, "HISTORY_" + classify(err)
|
||||
}
|
||||
if len(history.Items) != 1 || history.NextCursor != "" {
|
||||
return nil, "ATTEMPT_COUNT_MISMATCH"
|
||||
}
|
||||
a := history.Items[0]
|
||||
if !validID(a.ID) || a.TaskID != task.ID || a.State != orbit.AttemptSucceeded || a.Epoch != 1 || a.AuthorizedAt == nil || a.EndedAt == nil || a.ErrorCode != "" || a.Retryable || a.ConcurrencyToken != 0 {
|
||||
return nil, "ATTEMPT_MISMATCH"
|
||||
}
|
||||
if a.CreatedAt.Before(task.AvailableAt) || a.AuthorizedAt.Before(a.CreatedAt) || a.EndedAt.Before(*a.AuthorizedAt) || !a.EndedAt.Equal(*task.CompletedAt) {
|
||||
return nil, "ATTEMPT_TIMESTAMP_MISMATCH"
|
||||
}
|
||||
call, stop = context.WithTimeout(ctx, cfg.Timeout)
|
||||
source, err := api.GetTaskSource(call, task.ID)
|
||||
stop()
|
||||
if err != nil {
|
||||
return nil, "SOURCE_" + classify(err)
|
||||
}
|
||||
s := source.Workflow
|
||||
if source.TaskID != task.ID || source.Kind != "WORKFLOW" || source.RetryOf != nil || source.Schedule != nil || s == nil || s.WorkflowID != instance.ID || s.DefinitionID != instance.DefinitionID || s.DefinitionVersionID != instance.DefinitionVersionID || s.Version != instance.Version || s.StepID != step.ID || s.StepKey != step.Key || s.StepKind != "TASK" || s.ActivationID != *step.ActivationID || s.ActivationSequence != 1 || s.PollRound != 0 || s.PollRoundID != nil || s.RetryOf != nil {
|
||||
return nil, "SOURCE_MISMATCH"
|
||||
}
|
||||
proofs = append(proofs, verifiedTask{sequence: sequence, taskID: task.ID, attemptID: a.ID})
|
||||
}
|
||||
return proofs, ""
|
||||
}
|
||||
func matchesTask(t orbit.Task, c config, runID string, sequence int) bool {
|
||||
input, ok := decodePayload(t.Payload)
|
||||
return validID(t.ID) && t.QueueID == c.QueueID && t.Handler == handlerName(runID) && t.HandlerVersion == "v1" && t.PayloadVersion == "v1" && t.BusinessKey == "" && t.ConcurrencyKey == "" && t.Priority == 0 && t.RetryOf == nil && t.StartDeadlineAt == nil && t.RecoverySafety == orbit.RecoveryReconcileOnly && t.Retry == (orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(c.RunTimeout.Milliseconds())}) && ok && input == (workflowPayload{RunID: runID, Sequence: sequence})
|
||||
}
|
||||
210
sdk/go/cmd/orbit-workflow-load/verify_test.go
Normal file
210
sdk/go/cmd/orbit-workflow-load/verify_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testID(n int) string { return fmt.Sprintf("%032x", n) }
|
||||
func testConfig() config {
|
||||
return config{QueueID: testID(1), Requests: 1, Concurrency: 1, Workers: 1, Capacity: 2, Timeout: time.Second, RunTimeout: 90 * time.Second, PollInterval: 100 * time.Millisecond}
|
||||
}
|
||||
|
||||
type proofAPI struct {
|
||||
workflowAPI
|
||||
instance orbit.WorkflowInstance
|
||||
steps orbit.WorkflowSteps
|
||||
tasks map[string]orbit.Task
|
||||
attempts map[string]orbit.AttemptPage
|
||||
sources map[string]orbit.TaskSource
|
||||
}
|
||||
|
||||
func (a *proofAPI) GetWorkflow(context.Context, string) (orbit.WorkflowInstance, error) {
|
||||
return a.instance, nil
|
||||
}
|
||||
func (a *proofAPI) GetWorkflowSteps(context.Context, string) (orbit.WorkflowSteps, error) {
|
||||
return a.steps, nil
|
||||
}
|
||||
func (a *proofAPI) GetTask(_ context.Context, id string) (orbit.Task, error) { return a.tasks[id], nil }
|
||||
func (a *proofAPI) ListTaskAttempts(_ context.Context, id string, _ orbit.HistoryPageRequest) (orbit.AttemptPage, error) {
|
||||
return a.attempts[id], nil
|
||||
}
|
||||
func (a *proofAPI) GetTaskSource(_ context.Context, id string) (orbit.TaskSource, error) {
|
||||
return a.sources[id], nil
|
||||
}
|
||||
func (a *proofAPI) StartWorkflow(context.Context, string, orbit.StartWorkflowRequest) (orbit.WorkflowStartResult, error) {
|
||||
return orbit.WorkflowStartResult{Instance: a.instance}, nil
|
||||
}
|
||||
func proofFixture() (*proofAPI, workflowAdmission) {
|
||||
c := testConfig()
|
||||
start := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
|
||||
end := start.Add(3 * time.Second)
|
||||
raw := json.RawMessage(`{"run_id":"run","sequence":0}`)
|
||||
instance := orbit.WorkflowInstance{ID: testID(2), NamespaceID: testID(3), DefinitionID: testID(4), DefinitionVersionID: testID(5), Version: "v1", State: "SUCCEEDED", Revision: 4, Input: raw, Output: json.RawMessage(`{"left":{"run_id":"run","sequence":0},"right":{"run_id":"run","sequence":0}}`), BusinessKey: businessKey("run", 0), DeadlineAt: start.Add(c.RunTimeout), CreatedAt: start, CompletedAt: &end}
|
||||
api := &proofAPI{instance: instance, steps: orbit.WorkflowSteps{InstanceID: instance.ID, Revision: instance.Revision}, tasks: map[string]orbit.Task{}, attempts: map[string]orbit.AttemptPage{}, sources: map[string]orbit.TaskSource{}}
|
||||
for i, key := range []string{"pair", "left", "right"} {
|
||||
activation := testID(10 + i)
|
||||
step := orbit.WorkflowStepStatus{ID: testID(20 + i), Key: key, Kind: "TASK", State: "SUCCEEDED", Revision: 3, ActivationID: &activation, ActivationSequence: 1}
|
||||
if i == 0 {
|
||||
step.Kind = "PARALLEL"
|
||||
api.steps.Steps = append(api.steps.Steps, step)
|
||||
continue
|
||||
}
|
||||
taskID := testID(30 + i)
|
||||
step.TaskID = &taskID
|
||||
api.steps.Steps = append(api.steps.Steps, step)
|
||||
authorized, completed := start.Add(time.Second), start.Add(2*time.Second)
|
||||
task := orbit.Task{ID: taskID, NamespaceID: instance.NamespaceID, QueueID: c.QueueID, Handler: handlerName("run"), HandlerVersion: "v1", PayloadVersion: "v1", State: orbit.TaskSucceeded, Payload: raw, Result: raw, RecoverySafety: orbit.RecoveryReconcileOnly, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(c.RunTimeout.Milliseconds())}, AttemptCount: 1, CreatedAt: start, AvailableAt: start, DeadlineAt: instance.DeadlineAt, CompletedAt: &completed}
|
||||
api.tasks[taskID] = task
|
||||
api.attempts[taskID] = orbit.AttemptPage{Items: []orbit.AttemptSummary{{ID: testID(40 + i), TaskID: taskID, State: orbit.AttemptSucceeded, Epoch: 1, CreatedAt: start, AuthorizedAt: &authorized, EndedAt: &completed}}}
|
||||
api.sources[taskID] = orbit.TaskSource{TaskID: taskID, Kind: "WORKFLOW", Workflow: &orbit.TaskWorkflowSource{WorkflowID: instance.ID, DefinitionID: instance.DefinitionID, DefinitionVersionID: instance.DefinitionVersionID, Version: "v1", StepID: step.ID, StepKey: key, StepKind: "TASK", ActivationID: activation, ActivationSequence: 1}}
|
||||
}
|
||||
return api, workflowAdmission{sequence: 0, instance: instance}
|
||||
}
|
||||
func TestWorkflowProofComplete(t *testing.T) {
|
||||
api, admission := proofFixture()
|
||||
cfg := testConfig()
|
||||
duration, proofs, code := verifyWorkflow(context.Background(), cfg, api, "run", admission.instance.DefinitionID, admission)
|
||||
if code != "" || duration != 3*time.Second || len(proofs) != 2 || proofs[0].taskID == proofs[1].taskID {
|
||||
t.Fatalf("valid proof rejected %s %#v", code, proofs)
|
||||
}
|
||||
log := newInvocationLog(1)
|
||||
handler := log.handler(context.Background(), cfg, "run")
|
||||
var wg sync.WaitGroup
|
||||
for _, p := range proofs {
|
||||
wg.Add(1)
|
||||
go func(p verifiedTask) {
|
||||
defer wg.Done()
|
||||
_, err := handler.Handle(context.Background(), orbit.WorkerExecution{Task: api.tasks[p.taskID], AttemptID: p.attemptID, Epoch: 1})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}(p)
|
||||
}
|
||||
wg.Wait()
|
||||
r := newReport(cfg, "run")
|
||||
log.summarize(&r, proofs)
|
||||
if r.HandlerCalls != 2 || r.DuplicateInvocations != 0 || r.UnconfirmedInvocations != 0 || len(r.finish().Errors) != 0 {
|
||||
t.Fatalf("call proof mismatch %#v", r)
|
||||
}
|
||||
}
|
||||
func TestWorkflowProofRejectsWrongIdentitiesAndHistory(t *testing.T) {
|
||||
for name, change := range map[string]func(*proofAPI){
|
||||
"version substitution": func(a *proofAPI) { a.instance.DefinitionVersionID = testID(80) },
|
||||
"changed input": func(a *proofAPI) { a.instance.Input = json.RawMessage(`{"run_id":"run"}`) },
|
||||
"wrong steps owner": func(a *proofAPI) { a.steps.InstanceID = testID(80) },
|
||||
"missing join": func(a *proofAPI) { a.steps.Steps = a.steps.Steps[1:] },
|
||||
"duplicate branch": func(a *proofAPI) { a.steps.Steps[2] = a.steps.Steps[1] },
|
||||
"wrong branch name": func(a *proofAPI) { a.steps.Steps[2].Key = "third" },
|
||||
"reused task": func(a *proofAPI) { a.steps.Steps[2].TaskID = a.steps.Steps[1].TaskID },
|
||||
"reused activation": func(a *proofAPI) { a.steps.Steps[2].ActivationID = a.steps.Steps[1].ActivationID },
|
||||
"wrong task policy": func(a *proofAPI) { v := a.tasks[testID(31)]; v.Retry.MaxAttempts = 2; a.tasks[v.ID] = v },
|
||||
"wrong task queue": func(a *proofAPI) { v := a.tasks[testID(31)]; v.QueueID = testID(80); a.tasks[v.ID] = v },
|
||||
"missing result sequence": func(a *proofAPI) {
|
||||
v := a.tasks[testID(31)]
|
||||
v.Result = json.RawMessage(`{"run_id":"run"}`)
|
||||
a.tasks[v.ID] = v
|
||||
},
|
||||
"uncertain task": func(a *proofAPI) { v := a.tasks[testID(31)]; v.EffectUncertain = true; a.tasks[v.ID] = v },
|
||||
"wrong task deadline": func(a *proofAPI) {
|
||||
v := a.tasks[testID(31)]
|
||||
v.DeadlineAt = v.DeadlineAt.Add(time.Second)
|
||||
a.tasks[v.ID] = v
|
||||
},
|
||||
"extra attempt": func(a *proofAPI) {
|
||||
v := a.attempts[testID(31)]
|
||||
v.Items = append(v.Items, v.Items[0])
|
||||
a.attempts[testID(31)] = v
|
||||
},
|
||||
"missing authorization": func(a *proofAPI) { a.attempts[testID(31)].Items[0].AuthorizedAt = nil },
|
||||
"wrong epoch": func(a *proofAPI) { a.attempts[testID(31)].Items[0].Epoch = 2 },
|
||||
"wrong attempt time": func(a *proofAPI) {
|
||||
v := a.instance.CreatedAt.Add(-time.Second)
|
||||
a.attempts[testID(31)].Items[0].EndedAt = &v
|
||||
},
|
||||
"wrong source task": func(a *proofAPI) { v := a.sources[testID(31)]; v.TaskID = testID(80); a.sources[testID(31)] = v },
|
||||
"wrong source branch": func(a *proofAPI) { a.sources[testID(31)].Workflow.StepKey = "right" },
|
||||
"wrong source kind": func(a *proofAPI) { a.sources[testID(31)].Workflow.StepKind = "POLL" },
|
||||
"wrong output zero": func(a *proofAPI) {
|
||||
a.instance.Output = json.RawMessage(`{"left":{"run_id":"run"},"right":{"run_id":"run"}}`)
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
a, admission := proofFixture()
|
||||
change(a)
|
||||
_, proofs, code := verifyWorkflow(context.Background(), testConfig(), a, "run", admission.instance.DefinitionID, admission)
|
||||
if code == "" || len(proofs) != 0 {
|
||||
t.Fatalf("bad evidence counted %s %#v", code, proofs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestStartBindsPublishedVersion(t *testing.T) {
|
||||
a, _ := proofFixture()
|
||||
c := testConfig()
|
||||
results := submitWorkflows(context.Background(), c, a, "run", a.instance.DefinitionID, a.instance.DefinitionVersionID, a.instance.NamespaceID)
|
||||
if len(results) != 1 || results[0].code != "" {
|
||||
t.Fatalf("valid start rejected %#v", results)
|
||||
}
|
||||
results = submitWorkflows(context.Background(), c, a, "run", a.instance.DefinitionID, testID(80), a.instance.NamespaceID)
|
||||
if len(results) != 1 || results[0].code != "INVALID_RECEIPT" {
|
||||
t.Fatal("unpublished version accepted")
|
||||
}
|
||||
}
|
||||
func TestDuplicateInvocationCannotCountAsSecondBranch(t *testing.T) {
|
||||
a, _ := proofFixture()
|
||||
c := testConfig()
|
||||
log := newInvocationLog(1)
|
||||
h := log.handler(context.Background(), c, "run")
|
||||
proof := verifiedTask{sequence: 0, taskID: testID(31), attemptID: testID(41)}
|
||||
for i := 0; i < 2; i++ {
|
||||
_, _ = h.Handle(context.Background(), orbit.WorkerExecution{Task: a.tasks[proof.taskID], AttemptID: proof.attemptID, Epoch: 1})
|
||||
}
|
||||
r := newReport(c, "run")
|
||||
log.summarize(&r, []verifiedTask{proof, {sequence: 0, taskID: testID(32), attemptID: testID(42)}})
|
||||
if r.HandlerCalls != 2 || r.DuplicateInvocations != 1 || r.Errors["LOCAL_PROOF_MISMATCH"] != 1 {
|
||||
t.Fatalf("duplicate filled missing branch %#v", r)
|
||||
}
|
||||
r = newReport(c, "run")
|
||||
log.summarize(&r, nil)
|
||||
if r.UnconfirmedInvocations != 2 {
|
||||
t.Fatal("unverified calls disappeared")
|
||||
}
|
||||
}
|
||||
func TestStrictOutputKeepsSequenceZeroRequired(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"left":{"run_id":"run"},"right":{"run_id":"run","sequence":0}}`,
|
||||
`{"left":{"run_id":"run","sequence":null},"right":{"run_id":"run","sequence":0}}`,
|
||||
`{"left":{"run_id":"run","sequence":0,"sequence":0},"right":{"run_id":"run","sequence":0}}`,
|
||||
`{"left":{"run_id":"run","Sequence":0},"right":{"run_id":"run","sequence":0}}`,
|
||||
`{"left":{"run_id":"run","sequence":0},"right":{"run_id":"run","sequence":0},"right":{"run_id":"run","sequence":0}}`,
|
||||
} {
|
||||
if matchesOutput(json.RawMessage(raw), "run", 0) {
|
||||
t.Fatalf("invalid zero payload accepted %s", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestSpecificationRetainsConfiguredTimeout(t *testing.T) {
|
||||
c := testConfig()
|
||||
spec := workflowSpec(c.QueueID, handlerName("run"), c.RunTimeout)
|
||||
if spec.TimeoutMS != 90000 || spec.Body.Steps[0].Parallel.Branches[0].Body.Steps[0].Task.TimeoutMS != 90000 {
|
||||
t.Fatal("run timeout silently capped")
|
||||
}
|
||||
var stored orbit.WorkflowSpecification
|
||||
raw, _ := json.Marshal(spec)
|
||||
if err := json.Unmarshal(raw, &stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sameSpecification(stored, spec) {
|
||||
t.Fatal("serialized specification rejected")
|
||||
}
|
||||
stored.TimeoutMS = 30000
|
||||
if sameSpecification(stored, spec) {
|
||||
t.Fatal("changed published spec accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user