orbit/sdk/go/cmd/orbit-workflow-load/measure.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"

	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
)

// workflowPayload 是两个并行分支共享的固定输入；不含凭据或外部地址。
type workflowPayload struct {
	RunID    string `json:"run_id"`   // RunID 隔离本次测量，不作为业务幂等保证。
	Sequence int    `json:"sequence"` // Sequence 从0开始的实例序号，用于发现错配和重复调用。
}

func decodePayload(raw json.RawMessage) (workflowPayload, bool) {
	var wire struct {
		RunID    *string `json:"run_id"`
		Sequence *int    `json:"sequence"`
	}
	decoder := json.NewDecoder(bytes.NewReader(raw))
	decoder.DisallowUnknownFields()
	if decoder.Decode(&wire) != nil || wire.RunID == nil || wire.Sequence == nil {
		return workflowPayload{}, false
	}
	var trailing any
	if decoder.Decode(&trailing) != io.EOF {
		return workflowPayload{}, false
	}
	return workflowPayload{RunID: *wire.RunID, Sequence: *wire.Sequence}, true
}

func handlerName(runID string) string   { return "load.workflow." + runID }
func definitionKey(runID string) string { return "orbit-workflow-load-" + runID }
func requestKey(runID string, sequence int) string {
	return fmt.Sprintf("orbit-workflow-%s-%06d", runID, sequence)
}
func businessKey(runID string, sequence int) string {
	return fmt.Sprintf("orbit-workflow-%s-%06d", runID, sequence)
}

type workflowAPI interface {
	GetQueue(context.Context, string) (orbit.Queue, error)
	CreateWorkflowDefinition(context.Context, string, orbit.CreateWorkflowDefinitionRequest) (orbit.WorkflowDefinitionResult, error)
	PublishWorkflowVersion(context.Context, string, string, orbit.PublishWorkflowVersionRequest) (orbit.WorkflowVersionResult, error)
	StartWorkflow(context.Context, string, orbit.StartWorkflowRequest) (orbit.WorkflowStartResult, error)
	GetWorkflow(context.Context, string) (orbit.WorkflowInstance, error)
	GetWorkflowSteps(context.Context, string) (orbit.WorkflowSteps, error)
	GetTask(context.Context, string) (orbit.Task, error)
	ListTaskAttempts(context.Context, string, orbit.HistoryPageRequest) (orbit.AttemptPage, error)
	GetTaskSource(context.Context, string) (orbit.TaskSource, error)
}

type workflowAdmission struct {
	sequence int
	instance orbit.WorkflowInstance
	duration time.Duration
	code     string
}

type workflowVerification struct {
	sequence int
	duration time.Duration
	code     string
}

// invocationLog 有界保存每个流程的实际处理器调用，两个并行分支各应调用一次。
type invocationLog struct {
	mu      sync.Mutex
	calls   []int
	invalid int
}

func newInvocationLog(n int) *invocationLog { return &invocationLog{calls: make([]int, n)} }

func (log *invocationLog) handler(ctx context.Context, cfg config, runID string) orbit.WorkerHandler {
	return orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: handlerName(runID), HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(work context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
		payload, valid := decodePayload(execution.Task.Payload)
		if !valid || payload.RunID != runID || payload.Sequence < 0 || payload.Sequence >= cfg.Requests || execution.Task.Handler != handlerName(runID) || execution.Task.HandlerVersion != "v1" || execution.Task.PayloadVersion != "v1" || execution.Task.QueueID != cfg.QueueID {
			log.mu.Lock()
			log.invalid++
			log.mu.Unlock()
			return nil, &orbit.WorkerFailure{Code: "WORKFLOW_INPUT_MISMATCH"}
		}
		log.mu.Lock()
		log.calls[payload.Sequence]++
		log.mu.Unlock()
		select {
		case <-work.Done():
			return nil, work.Err()
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		return execution.Task.Payload, nil
	}}
}

func workflowSpec(queue, handler string, timeout time.Duration) orbit.WorkflowSpecification {
	typ := orbit.WorkflowType{Kind: "OBJECT", Fields: map[string]orbit.WorkflowType{"run_id": {Kind: "STRING"}, "sequence": {Kind: "NUMBER"}}}
	task := orbit.WorkflowTask{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", InputType: typ, Input: orbit.WorkflowRef("input"), ResultType: typ, TimeoutMS: uint64(timeout.Milliseconds()), Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: uint64(timeout.Milliseconds())}, RecoverySafety: orbit.RecoveryReconcileOnly}
	parallel := orbit.WorkflowParallel{Branches: []orbit.WorkflowParallelBranch{
		{Name: "left", Body: orbit.WorkflowBlock{Steps: []orbit.WorkflowStep{{Key: "left", Kind: "TASK", Task: &task}}, OutputType: typ, Output: orbit.WorkflowRef("left")}},
		{Name: "right", Body: orbit.WorkflowBlock{Steps: []orbit.WorkflowStep{{Key: "right", Kind: "TASK", Task: &task}}, OutputType: typ, Output: orbit.WorkflowRef("right")}},
	}}
	output := orbit.WorkflowType{Kind: "OBJECT", Fields: map[string]orbit.WorkflowType{"left": typ, "right": typ}}
	return orbit.WorkflowSpecification{SchemaVersion: 1, InputType: typ, TimeoutMS: uint64(timeout.Milliseconds()), FailurePolicy: "REVIEW", Body: orbit.WorkflowBlock{Steps: []orbit.WorkflowStep{{Key: "pair", Kind: "PARALLEL", Parallel: &parallel}}, OutputType: output, Output: orbit.WorkflowRef("pair")}}
}

func measure(parent context.Context, cfg config, api *orbit.Client, runID string) workflowReport {
	return measureClient(parent, cfg, api, runID)
}

func measureClient(parent context.Context, cfg config, api *orbit.Client, runID string) workflowReport {
	r := newReport(cfg, runID)
	ctx, cancel := context.WithTimeout(parent, cfg.RunTimeout)
	defer cancel()
	requestCtx, stop := context.WithTimeout(ctx, cfg.Timeout)
	queue, err := api.GetQueue(requestCtx, cfg.QueueID)
	stop()
	if err != nil {
		r.Errors["PREFLIGHT_"+classify(err)]++
		return r.finish()
	}
	r.Queue = &queue
	if queue.ID != cfg.QueueID || queue.Paused || queue.MaxConcurrency == 0 || queue.RatePerSecond == 0 {
		r.Errors["QUEUE_UNAVAILABLE"]++
		return r.finish()
	}
	handler := handlerName(runID)
	typicalTimeout := cfg.RunTimeout
	if typicalTimeout > 30*time.Second {
		typicalTimeout = 30 * time.Second
	}
	spec := workflowSpec(cfg.QueueID, handler, typicalTimeout)
	requestCtx, stop = context.WithTimeout(ctx, cfg.Timeout)
	definition, err := api.CreateWorkflowDefinition(requestCtx, definitionKey(runID), orbit.CreateWorkflowDefinitionRequest{Name: definitionKey(runID), Description: "bounded parallel workflow verification"})
	stop()
	if err != nil {
		r.Errors["CREATE_"+classify(err)]++
		return r.finish()
	}
	r.DefinitionID = definition.Definition.ID
	if !validID(r.DefinitionID) {
		r.Errors["INVALID_DEFINITION_RECEIPT"]++
		return r.finish()
	}
	requestCtx, stop = context.WithTimeout(ctx, cfg.Timeout)
	version, err := api.PublishWorkflowVersion(requestCtx, r.DefinitionID, runID, orbit.PublishWorkflowVersionRequest{Version: "v1", Specification: spec, Reason: "bounded parallel workflow verification"})
	stop()
	if err != nil {
		r.Errors["PUBLISH_"+classify(err)]++
		return r.finish()
	}
	r.VersionID = version.Version.ID
	if !validID(r.VersionID) || version.Version.Version != "v1" || version.Version.DefinitionID != r.DefinitionID {
		r.Errors["INVALID_VERSION_RECEIPT"]++
		return r.finish()
	}

	log := newInvocationLog(cfg.Requests)
	workers := make([]*orbit.Worker, 0, cfg.Workers)
	for i := 0; i < cfg.Workers; i++ {
		worker, e := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: cfg.QueueID, Capacity: uint32(cfg.Capacity), ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{log.handler(ctx, cfg, runID)}})
		if e != nil {
			r.Errors["WORKER_CONFIG"]++
			return r.finish()
		}
		workers = append(workers, worker)
	}
	workerCtx, stopWorkers := context.WithCancel(ctx)
	done := make(chan error, len(workers))
	for _, worker := range workers {
		go func(w *orbit.Worker) {
			e := w.Run(workerCtx)
			if workerCtx.Err() == nil {
				cancel()
			}
			done <- e
		}(worker)
	}

	admissions := submitWorkflows(ctx, cfg, api, runID, r.DefinitionID)
	r.Attempted = len(admissions)
	r.NotSubmitted = cfg.Requests - r.Attempted
	var accepted []workflowAdmission
	var admissionDurations []time.Duration
	for _, a := range admissions {
		admissionDurations = append(admissionDurations, a.duration)
		if a.code != "" {
			r.AdmissionErrors++
			r.Errors["ADMISSION_"+a.code]++
			continue
		}
		accepted = append(accepted, a)
	}
	r.Accepted = len(accepted)
	r.Admission = distributionOf(admissionDurations)
	r.AdmissionFinishedAt = time.Now().UTC()
	verified := verifyWorkflows(ctx, cfg, api, runID, r.DefinitionID, accepted)
	var completions []time.Duration
	for _, v := range verified {
		if v.code != "" {
			r.Errors[v.code]++
			continue
		}
		r.VerifiedSuccess++
		completions = append(completions, v.duration)
	}
	if ctx.Err() != nil {
		r.Errors["RUN_"+classify(ctx.Err())]++
	}
	stopWorkers()
	for range workers {
		if e := <-done; e != nil {
			r.Errors["WORKER_STOP"]++
		}
	}
	for _, worker := range workers {
		stats := worker.Stats()
		r.WorkerStarted += stats.Started
		r.Reconnects += stats.Reconnects
		r.Renewals += stats.RenewalsConfirmed
	}
	log.mu.Lock()
	for _, calls := range log.calls {
		r.HandlerCalls += calls
		if calls > 2 {
			r.DuplicateInvocations += calls - 2
		}
		if calls != 2 {
			r.Errors["HANDLER_CALL_MISMATCH"]++
		}
	}
	r.Errors["INVALID_HANDLER_INPUT"] += log.invalid
	log.mu.Unlock()
	r.Completion = distributionOf(completions)
	return r.finish()
}

func submitWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string) []workflowAdmission {
	jobs := make(chan int)
	out := make(chan workflowAdmission, cfg.Requests)
	var wg sync.WaitGroup
	for i := 0; i < cfg.Concurrency; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for sequence := range jobs {
				if ctx.Err() != nil {
					continue
				}
				payload, _ := json.Marshal(workflowPayload{RunID: runID, Sequence: sequence})
				started := time.Now()
				call, stop := context.WithTimeout(ctx, cfg.Timeout)
				result, err := api.StartWorkflow(call, requestKey(runID, sequence), orbit.StartWorkflowRequest{DefinitionID: definitionID, Version: "v1", Input: payload, BusinessKey: businessKey(runID, sequence)})
				stop()
				a := workflowAdmission{sequence: sequence, instance: result.Instance, duration: time.Since(started)}
				if err != nil {
					a.code = classify(err)
				} else if result.Deduplicated || !validID(result.Instance.ID) || result.Instance.DefinitionID != definitionID || result.Instance.Version != "v1" || result.Instance.BusinessKey != businessKey(runID, sequence) {
					a.code = "INVALID_RECEIPT"
				}
				out <- a
			}
		}()
	}
feed:
	for sequence := 0; sequence < cfg.Requests; sequence++ {
		select {
		case <-ctx.Done():
			break feed
		case jobs <- sequence:
		}
	}
	close(jobs)
	wg.Wait()
	close(out)
	results := make([]workflowAdmission, 0, cfg.Requests)
	for result := range out {
		results = append(results, result)
	}
	return results
}

func verifyWorkflows(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string, admissions []workflowAdmission) []workflowVerification {
	jobs := make(chan workflowAdmission, len(admissions))
	for _, admission := range admissions {
		jobs <- admission
	}
	close(jobs)
	out := make(chan workflowVerification, len(admissions))
	var wg sync.WaitGroup
	for i := 0; i < cfg.Concurrency; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for admission := range jobs {
				duration, code := verifyWorkflow(ctx, cfg, api, runID, definitionID, admission)
				out <- workflowVerification{sequence: admission.sequence, duration: duration, code: code}
			}
		}()
	}
	wg.Wait()
	close(out)
	results := make([]workflowVerification, 0, len(admissions))
	for result := range out {
		results = append(results, result)
	}
	return results
}

func verifyWorkflow(ctx context.Context, cfg config, api workflowAPI, runID, definitionID string, admission workflowAdmission) (time.Duration, string) {
	for {
		if ctx.Err() != nil {
			return 0, "VERIFY_" + classify(ctx.Err())
		}
		call, stop := context.WithTimeout(ctx, cfg.Timeout)
		instance, err := api.GetWorkflow(call, admission.instance.ID)
		stop()
		if err != nil {
			return 0, "READ_" + classify(err)
		}
		if instance.ID != admission.instance.ID || instance.DefinitionID != definitionID || instance.Version != "v1" || instance.BusinessKey != businessKey(runID, admission.sequence) {
			return 0, "WORKFLOW_MISMATCH"
		}
		switch instance.State {
		case "SUCCEEDED":
			if instance.CompletedAt == nil || instance.CreatedAt.IsZero() || instance.CompletedAt.Before(instance.CreatedAt) {
				return 0, "COMPLETION_MISMATCH"
			}
			call, stop = context.WithTimeout(ctx, cfg.Timeout)
			steps, e := api.GetWorkflowSteps(call, instance.ID)
			stop()
			if e != nil {
				return 0, "STEPS_" + classify(e)
			}
			if code := verifySteps(ctx, cfg, api, instance, runID, admission.sequence, steps); code != "" {
				return 0, code
			}
			if !matchesOutput(instance.Output, runID, admission.sequence) {
				return 0, "OUTPUT_MISMATCH"
			}
			return instance.CompletedAt.Sub(instance.CreatedAt), ""
		case "NEEDS_REVIEW", "FAILED", "CANCELLED":
			return 0, "WORKFLOW_" + instance.State
		case "RUNNING", "WAITING", "PAUSED", "PENDING":
		default:
			return 0, "UNKNOWN_WORKFLOW_STATE"
		}
		timer := time.NewTimer(cfg.PollInterval)
		select {
		case <-ctx.Done():
			timer.Stop()
			return 0, "VERIFY_" + classify(ctx.Err())
		case <-timer.C:
		}
	}
}

func verifySteps(ctx context.Context, cfg config, api workflowAPI, instance orbit.WorkflowInstance, runID string, sequence int, steps orbit.WorkflowSteps) string {
	var taskSteps []orbit.WorkflowStepStatus
	for _, step := range steps.Steps {
		if step.Kind == "TASK" {
			taskSteps = append(taskSteps, step)
		}
	}
	if len(taskSteps) != 2 {
		return "STEP_COUNT_MISMATCH"
	}
	for _, step := range taskSteps {
		if step.State != "SUCCEEDED" || step.TaskID == nil || !validID(*step.TaskID) || step.ActivationSequence != 1 || step.ActivationID == nil {
			return "STEP_MISMATCH"
		}
		call, stop := context.WithTimeout(ctx, cfg.Timeout)
		task, err := api.GetTask(call, *step.TaskID)
		stop()
		if err != nil {
			return "TASK_" + classify(err)
		}
		payload, valid := decodePayload(task.Result)
		if task.ID != *step.TaskID || task.State != orbit.TaskSucceeded || task.EffectUncertain || task.AttemptCount != 1 || !valid || payload != (workflowPayload{RunID: runID, Sequence: sequence}) {
			return "TASK_MISMATCH"
		}
		call, stop = context.WithTimeout(ctx, cfg.Timeout)
		history, err := api.ListTaskAttempts(call, task.ID, orbit.HistoryPageRequest{Limit: 2})
		stop()
		if err != nil {
			return "HISTORY_" + classify(err)
		}
		if len(history.Items) != 1 || history.NextCursor != "" || history.Items[0].TaskID != task.ID || history.Items[0].State != orbit.AttemptSucceeded {
			return "ATTEMPT_MISMATCH"
		}
		call, stop = context.WithTimeout(ctx, cfg.Timeout)
		source, err := api.GetTaskSource(call, task.ID)
		stop()
		if err != nil || source.Kind != "WORKFLOW" || source.Workflow == nil || source.Workflow.WorkflowID != instance.ID || source.Workflow.DefinitionID != instance.DefinitionID || source.Workflow.DefinitionVersionID != instance.DefinitionVersionID || source.Workflow.Version != instance.Version || source.Workflow.StepID != step.ID || source.Workflow.ActivationID != *step.ActivationID || source.Workflow.ActivationSequence != 1 {
			return "SOURCE_MISMATCH"
		}
	}
	return ""
}

func matchesOutput(raw json.RawMessage, runID string, sequence int) bool {
	var wire struct {
		Left  workflowPayload `json:"left"`
		Right workflowPayload `json:"right"`
	}
	decoder := json.NewDecoder(bytes.NewReader(raw))
	decoder.DisallowUnknownFields()
	if decoder.Decode(&wire) != nil {
		return false
	}
	var trailing any
	if decoder.Decode(&trailing) != io.EOF {
		return false
	}
	return wire.Left == (workflowPayload{RunID: runID, Sequence: sequence}) && wire.Right == wire.Left
}