feat: add durable schedule management and multi-node crash recovery

2026-09-17 16:28:58 +08:00
parent d65cf96ea7
commit 6805506450
43 changed files with 2952 additions and 29 deletions
--- a/sdk/go/examples/schedules/main.go
+++ b/sdk/go/examples/schedules/main.go
@@ -0,0 +1,149 @@
+// schedules 演示持久化周期任务：创建、恢复、Worker 执行、触发历史和暂停。
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+func run() error {
+	token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
+	if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("read private credentials: %w", err)
+		}
+		var credentials struct {
+			Token   string `json:"token"`
+			QueueID string `json:"queue_id"`
+		}
+		if json.Unmarshal(raw, &credentials) != nil {
+			return errors.New("invalid credentials file")
+		}
+		if token == "" {
+			token = credentials.Token
+		}
+		if queue == "" {
+			queue = credentials.QueueID
+		}
+	}
+	client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
+	if err != nil {
+		return err
+	}
+
+	operation, err := orbit.NewWorkerMessageID()
+	if err != nil {
+		return err
+	}
+	handler := "object.snapshot." + operation
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+	defer cancel()
+	worker, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		// 实际副作用应按 Task.ID 或业务操作键去重；此示例只返回输入快照。
+		return e.Task.Payload, nil
+	}}}})
+	if err != nil {
+		return err
+	}
+	start := time.Now().UTC().Truncate(time.Microsecond).Add(1500 * time.Millisecond)
+	end := start.Add(10 * time.Second)
+	created, err := client.CreateSchedule(ctx, "create-"+operation, orbit.CreateScheduleRequest{Name: handler, Paused: true, Definition: orbit.ScheduleDefinition{
+		Rule:   orbit.ScheduleRule{Kind: "FIXED_RATE", Timezone: "UTC", StartAt: start, EndAt: &end, IntervalMS: 1000},
+		Policy: orbit.SchedulePolicy{Misfire: "COALESCE", Overlap: "SERIAL", CatchupLimit: 2, CatchupWindowMS: 30000, GraceMS: 1000},
+		Task:   orbit.SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"source_ref":"objects/sample","sequence":9007199254740993}`)},
+	}})
+	if err != nil {
+		return err
+	}
+	if _, err := client.ResumeSchedule(ctx, created.Schedule.ID, "resume-"+operation, orbit.ControlScheduleRequest{ExpectedRevision: created.Schedule.Revision, Reason: "start bounded example"}); err != nil {
+		return err
+	}
+	workerCtx, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- worker.Run(workerCtx) }()
+	waited := false
+	defer func() {
+		stop()
+		if !waited {
+			<-done
+		}
+	}()
+	completed := map[string]bool{}
+	for len(completed) < 2 {
+		page, err := client.ListScheduleOccurrences(ctx, created.Schedule.ID, orbit.SchedulePageRequest{})
+		if err != nil {
+			return err
+		}
+		for _, item := range page.Items {
+			if item.TaskID == nil || completed[*item.TaskID] {
+				continue
+			}
+			task, err := client.GetTask(ctx, *item.TaskID)
+			if err != nil {
+				return err
+			}
+			if task.State == orbit.TaskSucceeded {
+				completed[task.ID] = true
+				fmt.Printf("schedule=%s occurrence=%s task=%s version=%d state=%s\n", created.Schedule.ID, item.ID, task.ID, item.ScheduleVersion, task.State)
+			}
+			if task.State == orbit.TaskFailed || task.State == orbit.TaskNeedsReview {
+				return fmt.Errorf("scheduled task requires attention: %s", task.State)
+			}
+		}
+		if len(completed) >= 2 {
+			break
+		}
+		select {
+		case err := <-done:
+			waited = true
+			return fmt.Errorf("Worker stopped before completion: %v", err)
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(100 * time.Millisecond):
+		}
+	}
+	// 扫描也改变计划版本；遇到 CAS 冲突读取新状态，再以新操作身份重试暂停。
+	for {
+		current, err := client.GetSchedule(ctx, created.Schedule.ID)
+		if err != nil {
+			return err
+		}
+		key, err := orbit.NewWorkerMessageID()
+		if err != nil {
+			return err
+		}
+		paused, err := client.PauseSchedule(ctx, current.ID, key, orbit.ControlScheduleRequest{ExpectedRevision: current.Revision, Reason: "example finished"})
+		if err == nil {
+			fmt.Printf("paused=%t completed_tasks=%d current_version=%d\n", paused.Schedule.Paused, len(completed), paused.Schedule.CurrentVersion)
+			break
+		}
+		var apiErr *orbit.APIError
+		if !errors.As(err, &apiErr) || apiErr.Code != "STATE_CONFLICT" {
+			return err
+		}
+	}
+	stop()
+	err = <-done
+	waited = true
+	if err != nil {
+		return err
+	}
+	fmt.Printf("worker_drained=true running=%d\n", worker.Stats().Running)
+	return nil
+}