k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: bind workflow samples to distinct task execution proofs

Browse Source
This commit is contained in:
k2safe
2026-09-20 14:10:35 +08:00
parent 29734596a3
commit 8c9c6062ef
17 changed files with 1783 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
sdk/go/cmd/orbit-workflow-load/measure.go
View File

@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"sync"
"time"
@@ -18,23 +17,6 @@ type workflowPayload struct {
Sequence int `json:"sequence"` // Sequence 从0开始的实例序号，用于发现错配和重复调用。
}
func decodePayload(raw json.RawMessage) (workflowPayload, bool) {
var wire struct {
RunID *string `json:"run_id"`
Sequence *int `json:"sequence"`
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
if decoder.Decode(&wire) != nil || wire.RunID == nil || wire.Sequence == nil {
return workflowPayload{}, false
}
var trailing any
if decoder.Decode(&trailing) != io.EOF {
return workflowPayload{}, false
}
return workflowPayload{RunID: *wire.RunID, Sequence: *wire.Sequence}, true
}
func handlerName(runID string) string { return "load.workflow." + runID }
func definitionKey(runID string) string { return "orbit-workflow-load-" + runID }
func requestKey(runID string, sequence int) string {
@@ -67,38 +49,7 @@ type workflowVerification struct {
sequence int
duration time.Duration
code string
}
// invocationLog 有界保存每个流程的实际处理器调用，两个并行分支各应调用一次。
type invocationLog struct {
mu sync.Mutex
calls []int
invalid int
}
func newInvocationLog(n int) *invocationLog { return &invocationLog{calls: make([]int, n)} }
func (log *invocationLog) handler(ctx context.Context, cfg config, runID string) orbit.WorkerHandler {
return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(work context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
payload, valid := decodePayload(execution.Task.Payload)
if !valid || payload.RunID != runID || payload.Sequence < 0 || payload.Sequence >= cfg.Requests || execution.Task.Handler != handlerName(runID) || execution.Task.HandlerVersion != "v1" || execution.Task.PayloadVersion != "v1" || execution.Task.QueueID != cfg.QueueID {
log.mu.Lock()
log.invalid++
log.mu.Unlock()
return nil, &orbit.WorkerFailure{Code: "WORKFLOW_INPUT_MISMATCH"}
}
log.mu.Lock()
log.calls[payload.Sequence]++
log.mu.Unlock()
select {
case <-work.Done():
return nil, work.Err()
case <-ctx.Done():
return nil, ctx.Err()
default:
}
return execution.Task.Payload, nil
}}
tasks []verifiedTask
}
func workflowSpec(queue, handler string, timeout time.Duration) orbit.WorkflowSpecification {
@@ -128,16 +79,12 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
return r.finish()
}
r.Queue = &queue
if queue.ID != cfg.QueueID || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
if queue.ID != cfg.QueueID || !validID(queue.NamespaceID) || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
r.Errors["QUEUE_UNAVAILABLE"]++
return r.finish()
}
handler := handlerName(runID)
typicalTimeout := cfg.RunTimeout
if typicalTimeout > 30*time.Second {
typicalTimeout = 30 * time.Second
}
spec := workflowSpec(cfg.QueueID, handler, typicalTimeout)
spec := workflowSpec(cfg.QueueID, handler, cfg.RunTimeout)
requestCtx, stop = context.WithTimeout(ctx, cfg.Timeout)
definition, err := api.CreateWorkflowDefinition(requestCtx, definitionKey(runID), orbit.CreateWorkflowDefinitionRequest{Name: definitionKey(runID), Description: "bounded parallel workflow verification"})
stop()
@@ -146,7 +93,7 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
return r.finish()
}
r.DefinitionID = definition.Definition.ID
if !validID(r.DefinitionID) {
if !validID(r.DefinitionID) || definition.Deduplicated || definition.Definition.NamespaceID != queue.NamespaceID || definition.Definition.Name != definitionKey(runID) {
r.Errors["INVALID_DEFINITION_RECEIPT"]++
return r.finish()
}
@@ -158,7 +105,7 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
return r.finish()
}
r.VersionID = version.Version.ID
if !validID(r.VersionID) || version.Version.Version != "v1" || version.Version.DefinitionID != r.DefinitionID {
if !validID(r.VersionID) || version.Version.Version != "v1" || version.Version.DefinitionID != r.DefinitionID || version.Deduplicated || version.Version.NamespaceID != queue.NamespaceID || !sameSpecification(version.Version.Specification, spec) {
r.Errors["INVALID_VERSION_RECEIPT"]++
return r.finish()
}
@@ -185,13 +132,21 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
}(worker)
}
admissions := submitWorkflows(ctx, cfg, api, runID, r.DefinitionID)
admissions := submitWorkflows(ctx, cfg, api, runID, r.DefinitionID, r.VersionID, queue.NamespaceID)
r.Attempted = len(admissions)
r.NotSubmitted = cfg.Requests - r.Attempted
var accepted []workflowAdmission
var admissionDurations []time.Duration
seenInstances := map[string]bool{}
for _, a := range admissions {
admissionDurations = append(admissionDurations, a.duration)
if a.code == "" && seenInstances[a.instance.ID] {
a.code = "DUPLICATE_INSTANCE"
}
if a.code == "" {
seenInstances[a.instance.ID] = true
}
r.Instances = append(r.Instances, instanceReport{Sequence: a.sequence, Key: requestKey(runID, a.sequence), ID: a.instance.ID, Code: a.code})
if a.code != "" {
r.AdmissionErrors++
r.Errors["ADMISSION_"+a.code]++
@@ -204,12 +159,20 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
r.AdmissionFinishedAt = time.Now().UTC()
verified := verifyWorkflows(ctx, cfg, api, runID, r.DefinitionID, accepted)
var completions []time.Duration
var proven []verifiedTask
for _, v := range verified {
if v.code != "" {
r.Errors[v.code]++
continue
}
r.VerifiedSuccess++
proven = append(proven, v.tasks...)
for i := range r.Instances {
if r.Instances[i].Sequence == v.sequence {
r.Instances[i].Verified = true
break
}
}
completions = append(completions, v.duration)
}
if ctx.Err() != nil {
@@ -227,23 +190,12 @@ func measureClient(parent context.Context, cfg config, api *orbit.Client, runID
r.Reconnects += stats.Reconnects
r.Renewals += stats.RenewalsConfirmed
}
log.mu.Lock()
for _, calls := range log.calls {
r.HandlerCalls += calls
if calls > 2 {
r.DuplicateInvocations += calls - 2
}
if calls != 2 {
r.Errors["HANDLER_CALL_MISMATCH"]++
}
}
r.Errors["INVALID_HANDLER_INPUT"] += log.invalid
log.mu.Unlock()
log.summarize(&r, proven)
r.Completion = distributionOf(completions)
return r.finish()
}
func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string) []workflowAdmission {
func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, definitionID, versionID, namespace string) []workflowAdmission {
jobs := make(chan int)
out := make(chan workflowAdmission, cfg.Requests)
var wg sync.WaitGroup
@@ -263,7 +215,7 @@ func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, de
a := workflowAdmission{sequence: sequence, instance: result.Instance, duration: time.Since(started)}
if err != nil {
a.code = classify(err)
} else if result.Deduplicated || !validID(result.Instance.ID) || result.Instance.DefinitionID != definitionID || result.Instance.Version != "v1" || result.Instance.BusinessKey != businessKey(runID, sequence) {
} else if result.Deduplicated || !matchesInstance(result.Instance, cfg, runID, sequence, definitionID, versionID, namespace) {
a.code = "INVALID_RECEIPT"
}
out <- a
@@ -301,8 +253,8 @@ func verifyWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, de
go func() {
defer wg.Done()
for admission := range jobs {
duration, code := verifyWorkflow(ctx, cfg, api, runID, definitionID, admission)
out <- workflowVerification{sequence: admission.sequence, duration: duration, code: code}
duration, tasks, code := verifyWorkflow(ctx, cfg, api, runID, definitionID, admission)
out <- workflowVerification{sequence: admission.sequence, duration: duration, tasks: tasks, code: code}
}
}()
}
@@ -315,110 +267,61 @@ func verifyWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, de
return results
}
func verifyWorkflow(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string, admission workflowAdmission) (time.Duration, string) {
func verifyWorkflow(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string, admission workflowAdmission) (time.Duration, []verifiedTask, string) {
for {
if ctx.Err() != nil {
return 0, "VERIFY_" + classify(ctx.Err())
return 0, nil, "VERIFY_" + classify(ctx.Err())
}
call, stop := context.WithTimeout(ctx, cfg.Timeout)
instance, err := api.GetWorkflow(call, admission.instance.ID)
stop()
if err != nil {
return 0, "READ_" + classify(err)
return 0, nil, "READ_" + classify(err)
}
if instance.ID != admission.instance.ID || instance.DefinitionID != definitionID || instance.Version != "v1" || instance.BusinessKey != businessKey(runID, admission.sequence) {
return 0, "WORKFLOW_MISMATCH"
if instance.ID != admission.instance.ID || !matchesInstance(instance, cfg, runID, admission.sequence, definitionID, admission.instance.DefinitionVersionID, admission.instance.NamespaceID) || !instance.CreatedAt.Equal(admission.instance.CreatedAt) || !instance.DeadlineAt.Equal(admission.instance.DeadlineAt) {
return 0, nil, "WORKFLOW_MISMATCH"
}
switch instance.State {
case "SUCCEEDED":
if instance.CompletedAt == nil || instance.CreatedAt.IsZero() || instance.CompletedAt.Before(instance.CreatedAt) {
return 0, "COMPLETION_MISMATCH"
return 0, nil, "COMPLETION_MISMATCH"
}
call, stop = context.WithTimeout(ctx, cfg.Timeout)
steps, e := api.GetWorkflowSteps(call, instance.ID)
stop()
if e != nil {
return 0, "STEPS_" + classify(e)
return 0, nil, "STEPS_" + classify(e)
}
if code := verifySteps(ctx, cfg, api, instance, runID, admission.sequence, steps); code != "" {
return 0, code
tasks, code := verifySteps(ctx, cfg, api, instance, runID, admission.sequence, steps)
if code != "" {
return 0, nil, code
}
if !matchesOutput(instance.Output, runID, admission.sequence) {
return 0, "OUTPUT_MISMATCH"
return 0, nil, "OUTPUT_MISMATCH"
}
return instance.CompletedAt.Sub(instance.CreatedAt), ""
return instance.CompletedAt.Sub(instance.CreatedAt), tasks, ""
case "NEEDS_REVIEW", "FAILED", "CANCELLED":
return 0, "WORKFLOW_" + instance.State
return 0, nil, "WORKFLOW_" + instance.State
case "RUNNING", "WAITING", "PAUSED", "PENDING":
default:
return 0, "UNKNOWN_WORKFLOW_STATE"
return 0, nil, "UNKNOWN_WORKFLOW_STATE"
}
timer := time.NewTimer(cfg.PollInterval)
select {
case <-ctx.Done():
timer.Stop()
return 0, "VERIFY_" + classify(ctx.Err())
return 0, nil, "VERIFY_" + classify(ctx.Err())
case <-timer.C:
}
}
}
func verifySteps(ctx context.Context, cfg config, api workflowAPI, instance orbit.WorkflowInstance, runID string, sequence int, steps orbit.WorkflowSteps) string {
var taskSteps []orbit.WorkflowStepStatus
for _, step := range steps.Steps {
if step.Kind == "TASK" {
taskSteps = append(taskSteps, step)
}
}
if len(taskSteps) != 2 {
return "STEP_COUNT_MISMATCH"
}
for _, step := range taskSteps {
if step.State != "SUCCEEDED" || step.TaskID == nil || !validID(*step.TaskID) || step.ActivationSequence != 1 || step.ActivationID == nil {
return "STEP_MISMATCH"
}
call, stop := context.WithTimeout(ctx, cfg.Timeout)
task, err := api.GetTask(call, *step.TaskID)
stop()
if err != nil {
return "TASK_" + classify(err)
}
payload, valid := decodePayload(task.Result)
if task.ID != *step.TaskID || task.State != orbit.TaskSucceeded || task.EffectUncertain || task.AttemptCount != 1 || !valid || payload != (workflowPayload{RunID: runID, Sequence: sequence}) {
return "TASK_MISMATCH"
}
call, stop = context.WithTimeout(ctx, cfg.Timeout)
history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
stop()
if err != nil {
return "HISTORY_" + classify(err)
}
if len(history.Items) != 1 || history.NextCursor != "" || history.Items[0].TaskID != task.ID || history.Items[0].State != orbit.AttemptSucceeded {
return "ATTEMPT_MISMATCH"
}
call, stop = context.WithTimeout(ctx, cfg.Timeout)
source, err := api.GetTaskSource(call, task.ID)
stop()
if err != nil || source.Kind != "WORKFLOW" || source.Workflow == nil || source.Workflow.WorkflowID != instance.ID || source.Workflow.DefinitionID != instance.DefinitionID || source.Workflow.DefinitionVersionID != instance.DefinitionVersionID || source.Workflow.Version != instance.Version || source.Workflow.StepID != step.ID || source.Workflow.ActivationID != *step.ActivationID || source.Workflow.ActivationSequence != 1 {
return "SOURCE_MISMATCH"
}
}
return ""
func sameSpecification(a, b orbit.WorkflowSpecification) bool {
left, e := json.Marshal(a)
right, e2 := json.Marshal(b)
return e == nil && e2 == nil && bytes.Equal(left, right)
}
func matchesOutput(raw json.RawMessage, runID string, sequence int) bool {
var wire struct {
Left workflowPayload `json:"left"`
Right workflowPayload `json:"right"`
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
if decoder.Decode(&wire) != nil {
return false
}
var trailing any
if decoder.Decode(&trailing) != io.EOF {
return false
}
return wire.Left == (workflowPayload{RunID: runID, Sequence: sequence}) && wire.Right == wire.Left
func matchesInstance(i orbit.WorkflowInstance, c config, runID string, sequence int, definitionID, versionID, namespace string) bool {
p, ok := decodePayload(i.Input)
return validID(i.ID) && i.NamespaceID == namespace && i.DefinitionID == definitionID && i.DefinitionVersionID == versionID && i.Version == "v1" && i.BusinessKey == businessKey(runID, sequence) && ok && p == (workflowPayload{RunID: runID, Sequence: sequence}) && !i.CreatedAt.IsZero() && i.DeadlineAt.Equal(i.CreatedAt.Add(time.Duration(c.RunTimeout.Milliseconds())*time.Millisecond)) && !i.Paused && i.CancelRequestedAt == nil && i.StopKind == nil && i.StopRequestedAt == nil
}