feat: serialize keyed tasks across queues with durable fencing tokens
This commit is contained in:
169
sdk/go/examples/keyed-concurrency/main.go
Normal file
169
sdk/go/examples/keyed-concurrency/main.go
Normal file
@@ -0,0 +1,169 @@
|
||||
// keyed-concurrency 演示跨队列资源互斥及资源端单调序号校验。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
var errStale = errors.New("stale resource token")
|
||||
|
||||
// resource 仅用于本地示范。生产必须在资源存储事务内持久化最高序号和操作身份,
|
||||
// 并与实际写入原子提交;Go mutex 无法保护其他进程或外部服务。
|
||||
type resource struct {
|
||||
mu sync.Mutex
|
||||
namespace, key string
|
||||
highest uint64
|
||||
operations map[string]bool
|
||||
}
|
||||
|
||||
func (r *resource) apply(namespace, key string, token uint64, operation string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if namespace != r.namespace || key != r.key || token == 0 || token < r.highest {
|
||||
return errStale
|
||||
}
|
||||
r.highest = token
|
||||
// 同一操作的重报不产生第二次模拟写入。
|
||||
r.operations[operation] = true
|
||||
return nil
|
||||
}
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
token := os.Getenv("ORBIT_TOKEN")
|
||||
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" && token == "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var c struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err = json.Unmarshal(raw, &c); err != nil {
|
||||
return err
|
||||
}
|
||||
token = c.Token
|
||||
}
|
||||
api, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
op, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := "keyed." + op
|
||||
var queues []string
|
||||
var tasks []orbit.Task
|
||||
for i := 0; i < 2; i++ {
|
||||
q, err := api.CreateQueue(ctx, fmt.Sprintf("queue-%s-%d", op, i), orbit.CreateQueueRequest{Name: fmt.Sprintf("%s.%d", name, i), MaxConcurrency: 4, RatePerSecond: 100})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
queues = append(queues, q.Queue.ID)
|
||||
in := orbit.SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"objects/shared"}`), ConcurrencyKey: "object/" + op, TaskTimeoutMS: 20000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}}
|
||||
out, err := api.SubmitTask(ctx, fmt.Sprintf("operation-%s-%d", op, i), in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tasks = append(tasks, out.Task)
|
||||
}
|
||||
r := &resource{namespace: tasks[0].NamespaceID, key: tasks[0].ConcurrencyKey, operations: map[string]bool{}}
|
||||
handler := orbit.WorkerHandler{Capability: orbit.WorkerCapability{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(ctx context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 本示例每条任务表示一个独立操作,Task.ID 在安全重试期间保持不变。
|
||||
if err := r.apply(e.Task.NamespaceID, e.Task.ConcurrencyKey, e.ConcurrencyToken, e.Task.ID); err != nil {
|
||||
return nil, &orbit.WorkerFailure{Code: "STALE_RESOURCE_TOKEN"}
|
||||
}
|
||||
return json.RawMessage(`{"updated":true}`), nil
|
||||
}}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 2)
|
||||
started := 0
|
||||
defer func() {
|
||||
stop()
|
||||
for i := 0; i < started; i++ {
|
||||
<-done
|
||||
}
|
||||
}()
|
||||
for _, queue := range queues {
|
||||
w, err := orbit.NewWorker(api, orbit.WorkerOptions{QueueID: queue, Capacity: 4, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{handler}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started++
|
||||
go func() { done <- w.Run(run) }()
|
||||
}
|
||||
var low, high uint64
|
||||
for _, task := range tasks {
|
||||
for {
|
||||
current, err := api.GetTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.State == orbit.TaskSucceeded {
|
||||
break
|
||||
}
|
||||
if current.State == orbit.TaskFailed || current.State == orbit.TaskCancelled || current.State == orbit.TaskNeedsReview {
|
||||
return fmt.Errorf("unexpected task state: %s", current.State)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(80 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
history, err := api.ListTaskAttempts(ctx, task.ID, orbit.HistoryPageRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(history.Items) != 1 || history.Items[0].ConcurrencyToken == 0 {
|
||||
return errors.New("missing resource token")
|
||||
}
|
||||
v := history.Items[0].ConcurrencyToken
|
||||
if low == 0 || v < low {
|
||||
low = v
|
||||
}
|
||||
if v > high {
|
||||
high = v
|
||||
}
|
||||
}
|
||||
if low == high {
|
||||
return errors.New("resource token reused")
|
||||
}
|
||||
if !errors.Is(r.apply(r.namespace, r.key, low, "late-write"), errStale) {
|
||||
return errors.New("old resource token accepted")
|
||||
}
|
||||
r.mu.Lock()
|
||||
count := len(r.operations)
|
||||
r.mu.Unlock()
|
||||
if count != 2 {
|
||||
return errors.New("unexpected resource writes")
|
||||
}
|
||||
stop()
|
||||
for started > 0 {
|
||||
err := <-done
|
||||
started--
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fmt.Printf("queues=2 completed=2 tokens_increasing=true\nresource_writes=%d stale_write_rejected=true worker_drained=true\n", count)
|
||||
return nil
|
||||
}
|
||||
@@ -13,18 +13,19 @@ type HistoryPageRequest struct {
|
||||
|
||||
// AttemptSummary 是执行尝试的查询快照,不提供当前执行授权。
|
||||
type AttemptSummary struct {
|
||||
ID string `json:"id"` // ID 尝试标识,不是执行秘密。
|
||||
TaskID string `json:"task_id"` // TaskID 所属任务;查询始终限定在该任务下。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 单任务执行代次,JSON 为十进制字符串。
|
||||
State AttemptState `json:"state"` // State 尝试状态,与整个任务状态分开。
|
||||
Revision uint64 `json:"revision,string"` // Revision 当前记录版本,可因续租/恢复变化。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 数据库记录的租约期限;查询不能授权业务执行。
|
||||
AuthorizedAt *time.Time `json:"authorized_at,omitempty"` // AuthorizedAt 开始批准时间;空表示未批准,不证明副作用已经发生。
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"` // EndedAt 尝试结束时间;仍在执行时为空。
|
||||
Retryable bool `json:"retryable,omitempty"` // Retryable 该明确失败报告是否声明可安全重试;不代表平台必定安排重试。
|
||||
ErrorCode string `json:"error_code,omitempty"` // ErrorCode 稳定错误分类;无错误时为空。
|
||||
CreatedAt time.Time `json:"created_at"` // CreatedAt 创建时间,UTC 微秒精度。
|
||||
UpdatedAt time.Time `json:"updated_at"` // UpdatedAt 最新变更时间,不能代替版本校验。
|
||||
ConcurrencyToken uint64 `json:"concurrency_token,omitempty,string"` // ConcurrencyToken 资源键的单调许可序号;无资源键时为零,不能代替当前授权。
|
||||
ID string `json:"id"` // ID 尝试标识,不是执行秘密。
|
||||
TaskID string `json:"task_id"` // TaskID 所属任务;查询始终限定在该任务下。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 单任务执行代次,JSON 为十进制字符串。
|
||||
State AttemptState `json:"state"` // State 尝试状态,与整个任务状态分开。
|
||||
Revision uint64 `json:"revision,string"` // Revision 当前记录版本,可因续租/恢复变化。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 数据库记录的租约期限;查询不能授权业务执行。
|
||||
AuthorizedAt *time.Time `json:"authorized_at,omitempty"` // AuthorizedAt 开始批准时间;空表示未批准,不证明副作用已经发生。
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"` // EndedAt 尝试结束时间;仍在执行时为空。
|
||||
Retryable bool `json:"retryable,omitempty"` // Retryable 该明确失败报告是否声明可安全重试;不代表平台必定安排重试。
|
||||
ErrorCode string `json:"error_code,omitempty"` // ErrorCode 稳定错误分类;无错误时为空。
|
||||
CreatedAt time.Time `json:"created_at"` // CreatedAt 创建时间,UTC 微秒精度。
|
||||
UpdatedAt time.Time `json:"updated_at"` // UpdatedAt 最新变更时间,不能代替版本校验。
|
||||
}
|
||||
|
||||
// Attempt 增加当前归属与已保存结果,不返回令牌、摘要或原始请求秘密。
|
||||
|
||||
217
sdk/go/keyed_concurrency_integration_test.go
Normal file
217
sdk/go/keyed_concurrency_integration_test.go
Normal file
@@ -0,0 +1,217 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIntegrationWorkerRuntimeKeyedConcurrency(t *testing.T) {
|
||||
api, _, token := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
name := "keyed." + workerTestID(t)
|
||||
gateway := newWorkerFaultGateway(t, api.baseURL)
|
||||
client, err := NewClient(gateway.server.URL, Options{Token: token, HTTPClient: gateway.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var active, calls atomic.Int32
|
||||
var mu sync.Mutex
|
||||
tokens := map[string]uint64{}
|
||||
var last uint64
|
||||
handler := runtimeHandler(name, func(ctx context.Context, e WorkerExecution) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
if active.Add(1) != 1 {
|
||||
t.Error("same resource handlers overlapped")
|
||||
}
|
||||
defer active.Add(-1)
|
||||
mu.Lock()
|
||||
if e.Task.ConcurrencyKey != name || e.ConcurrencyToken <= last {
|
||||
t.Error("key token was lost, duplicated or reset")
|
||||
}
|
||||
last = e.ConcurrencyToken
|
||||
tokens[e.Task.ID] = e.ConcurrencyToken
|
||||
mu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
return e.Task.Payload, nil
|
||||
})
|
||||
var tasks []Task
|
||||
var workers []*Worker
|
||||
for i := 0; i < 2; i++ {
|
||||
q, err := api.CreateQueue(ctx, workerTestID(t), CreateQueueRequest{Name: name + workerTestID(t), MaxConcurrency: 4, RatePerSecond: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for j := 0; j < 2; j++ {
|
||||
out, err := api.SubmitTask(ctx, workerTestID(t), SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`), ConcurrencyKey: name})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks = append(tasks, out.Task)
|
||||
}
|
||||
w, err := NewWorker(client, WorkerOptions{QueueID: q.Queue.ID, Capacity: 4, Handlers: []WorkerHandler{handler}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workers = append(workers, w)
|
||||
}
|
||||
// 在真实 Start 提交后丢 ACK;恢复必须沿用原 key/token,不能重跑处理器。
|
||||
gateway.drop.Store("task.start")
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 2)
|
||||
for _, w := range workers {
|
||||
go func() { done <- w.Run(run) }()
|
||||
}
|
||||
defer func() {
|
||||
stop()
|
||||
for range workers {
|
||||
if err := <-done; err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, task := range tasks {
|
||||
finished := waitRuntimeState(t, ctx, api, task.ID, TaskSucceeded)
|
||||
if finished.AttemptCount != 1 || finished.ConcurrencyKey != name {
|
||||
t.Fatal("keyed task lost identity")
|
||||
}
|
||||
history, err := api.ListTaskAttempts(ctx, task.ID, HistoryPageRequest{})
|
||||
if err != nil || len(history.Items) != 1 {
|
||||
t.Fatal("missing keyed history", err)
|
||||
}
|
||||
mu.Lock()
|
||||
want := tokens[task.ID]
|
||||
mu.Unlock()
|
||||
if want == 0 || history.Items[0].ConcurrencyToken != want {
|
||||
t.Fatal("history token differs from handler")
|
||||
}
|
||||
detail, err := api.GetTaskAttempt(ctx, task.ID, history.Items[0].ID)
|
||||
if err != nil || detail.ConcurrencyToken != want {
|
||||
t.Fatal("detail token differs from handler", err)
|
||||
}
|
||||
}
|
||||
gateway.expectDrop(t, "task.start")
|
||||
if calls.Load() != 4 || active.Load() != 0 {
|
||||
t.Fatal("incorrect handler count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerProcessKeyedConcurrency(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
_, _, token := integrationWorkerClient(t)
|
||||
api := processClient(t, a, token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
name := "key-crash." + workerTestID(t)
|
||||
var registrations []WorkerRegistration
|
||||
var tasks []Task
|
||||
for i := 0; i < 2; i++ {
|
||||
q, err := api.CreateQueue(ctx, workerTestID(t), CreateQueueRequest{Name: name + workerTestID(t), MaxConcurrency: 2, RatePerSecond: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := NewWorkerRegistration(q.Queue.ID, 1, []WorkerCapability{{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.ConcurrencyKeys = true
|
||||
registrations = append(registrations, r)
|
||||
out, err := api.SubmitTask(ctx, workerTestID(t), SubmitTaskRequest{QueueID: q.Queue.ID, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), ConcurrencyKey: name})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks = append(tasks, out.Task)
|
||||
}
|
||||
dial := func(api *Client, r WorkerRegistration) *WorkerConnection {
|
||||
t.Helper()
|
||||
w, err := api.DialWorker(ctx, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { w.Close() })
|
||||
return w
|
||||
}
|
||||
capacity := func(w *WorkerConnection) {
|
||||
t.Helper()
|
||||
if _, err := w.Capacity(ctx, workerTestID(t), WorkerCapacity{Sequence: 1, Capacity: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
w1 := dial(api, registrations[0])
|
||||
capacity(w1)
|
||||
first, err := w1.NextAssignment(ctx)
|
||||
if err != nil || first.ConcurrencyToken == 0 {
|
||||
t.Fatal("first key not granted", err)
|
||||
}
|
||||
started, err := w1.Start(ctx, workerTestID(t), first.Attempt())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, active := started.Lease.ExecutionDeadline(); !active {
|
||||
t.Fatal("no current execution authorization")
|
||||
}
|
||||
a.kill(t)
|
||||
b := startOrbitProcess(t)
|
||||
api = processClient(t, b, token)
|
||||
if a.nodeID == b.nodeID {
|
||||
t.Fatal("node id reused")
|
||||
}
|
||||
w1 = dial(api, registrations[0])
|
||||
lease, err := w1.Resume(ctx, workerTestID(t), first.Attempt())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, active := lease.ExecutionDeadline(); !active {
|
||||
t.Fatal("original attempt not resumed")
|
||||
}
|
||||
w2 := dial(api, registrations[1])
|
||||
capacity(w2)
|
||||
brief, stop := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
_, err = w2.NextAssignment(brief)
|
||||
stop()
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal("restart lost key exclusion", err)
|
||||
}
|
||||
waiting, err := api.GetTask(ctx, tasks[1].ID)
|
||||
if err != nil || waiting.AttemptCount != 0 {
|
||||
t.Fatal("blocked key consumed attempt", err)
|
||||
}
|
||||
history, err := api.GetTaskAttempt(ctx, tasks[0].ID, first.AttemptID)
|
||||
if err != nil || history.ConcurrencyToken != first.ConcurrencyToken {
|
||||
t.Fatal("restart changed key token", err)
|
||||
}
|
||||
finishID := workerTestID(t)
|
||||
finish := WorkerFinish{Ref: first.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{}`)}
|
||||
if _, err = w1.Finish(ctx, finishID, finish); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := w2.NextAssignment(ctx)
|
||||
if err != nil || second.ConcurrencyToken <= first.ConcurrencyToken {
|
||||
t.Fatal("restart reset sequence", err)
|
||||
}
|
||||
if _, err = w2.Start(ctx, workerTestID(t), second.Attempt()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 第二次许可已经开始,旧完成回执的重放不能释放它。
|
||||
if out, err := w1.Finish(ctx, finishID, finish); err != nil || !out.Deduplicated {
|
||||
t.Fatal("duplicate finish changed ownership", err)
|
||||
}
|
||||
if _, err = w2.Renew(ctx, workerTestID(t), second.Attempt()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w2.Finish(ctx, workerTestID(t), WorkerFinish{Ref: second.Attempt(), Outcome: WorkerSucceeded, Result: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, task := range tasks {
|
||||
waitRuntimeState(t, ctx, api, task.ID, TaskSucceeded)
|
||||
}
|
||||
}
|
||||
52
sdk/go/keyed_concurrency_test.go
Normal file
52
sdk/go/keyed_concurrency_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
func TestWorkerConcurrencyKeyWireContract(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
support bool
|
||||
key string
|
||||
token uint64
|
||||
valid bool
|
||||
}{
|
||||
{"legacy", false, "", 0, true},
|
||||
{"new_unkeyed", true, "", 0, true},
|
||||
{"large_counter", true, "object/资源", ^uint64(0) - 1, true},
|
||||
{"not_negotiated", false, "object/a", 2, false},
|
||||
{"missing_counter", true, "object/a", 0, false},
|
||||
{"unexpected_counter", true, "", 2, false},
|
||||
{"leading_space", true, " object/a", 2, false},
|
||||
{"control", true, "object/\n", 2, false},
|
||||
{"too_long", true, strings.Repeat("字", 192), 2, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := workerTestRegistration(t)
|
||||
r.ConcurrencyKeys = tc.support
|
||||
a := workerTestAssignment(r)
|
||||
a.Task.ConcurrencyKey, a.ConcurrencyToken = tc.key, tc.token
|
||||
_, w := workerMock(t, r, func(ctx context.Context, c *websocket.Conn) {
|
||||
workerTestWrite(t, ctx, c, workerFrame{Type: "task.assign", Body: workerTestBody(t, a)})
|
||||
<-ctx.Done()
|
||||
})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
got, err := w.NextAssignment(ctx)
|
||||
if tc.valid {
|
||||
if err != nil || got.ConcurrencyToken != tc.token || got.Task.ConcurrencyKey != tc.key {
|
||||
t.Fatalf("key/token did not survive wire: %v", err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrWorkerProtocol) {
|
||||
t.Fatalf("invalid assignment accepted: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,8 @@ type SubmitTaskRequest struct {
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
// BusinessKey 可选关联键,不提供业务效果幂等保证。
|
||||
BusinessKey string `json:"business_key,omitempty"`
|
||||
// ConcurrencyKey 显式互斥资源键,同命名空间跨队列最多一个有效尝试;不代替业务幂等。
|
||||
ConcurrencyKey string `json:"concurrency_key,omitempty"`
|
||||
// AvailableAt 最早执行时间;为空表示受理后即可进入派发范围。
|
||||
AvailableAt *time.Time `json:"available_at,omitempty"`
|
||||
// StartDeadlineAt 最晚开始时间;为空表示使用任务总截止时间。
|
||||
@@ -58,6 +60,8 @@ type Task struct {
|
||||
EffectUncertain bool `json:"effect_uncertain,omitempty"`
|
||||
// BusinessKey 可选业务关联键,不自动提供业务效果幂等。
|
||||
BusinessKey string `json:"business_key,omitempty"`
|
||||
// ConcurrencyKey 显式互斥资源键,同命名空间跨队列最多一个有效尝试;不代替业务幂等。
|
||||
ConcurrencyKey string `json:"concurrency_key,omitempty"`
|
||||
// AvailableAt 最早可执行时间,UTC。
|
||||
AvailableAt time.Time `json:"available_at"`
|
||||
// StartDeadlineAt 最晚开始时间;为空时使用任务总截止时间。
|
||||
@@ -126,6 +130,8 @@ type TaskSummary struct {
|
||||
EffectUncertain bool `json:"effect_uncertain,omitempty"`
|
||||
// BusinessKey 可选业务关联键,不自动提供业务效果幂等。
|
||||
BusinessKey string `json:"business_key,omitempty"`
|
||||
// ConcurrencyKey 显式互斥资源键,同命名空间跨队列最多一个有效尝试;不代替业务幂等。
|
||||
ConcurrencyKey string `json:"concurrency_key,omitempty"`
|
||||
// AvailableAt 最早可执行时间,UTC。
|
||||
AvailableAt time.Time `json:"available_at"`
|
||||
// DeadlineAt 任务总截止时间,UTC。
|
||||
|
||||
@@ -399,6 +399,9 @@ func (w *WorkerConnection) command(ctx context.Context, op, id string, ref Worke
|
||||
}
|
||||
|
||||
func (w *WorkerConnection) validAssignment(a WorkerAssignment) bool {
|
||||
if (a.Task.ConcurrencyKey == "") != (a.ConcurrencyToken == 0) || (a.Task.ConcurrencyKey != "" && (!w.registration.ConcurrencyKeys || !validConcurrencyKey(a.Task.ConcurrencyKey))) {
|
||||
return false
|
||||
}
|
||||
if !validWorkerAttempt(a.Attempt()) || a.Task.QueueID != w.registration.QueueID || !validID(a.Task.NamespaceID) || a.Task.State != TaskLeased || a.Task.Revision == 0 || a.LeaseUntil.IsZero() || a.ServerTime.IsZero() || len(a.Task.Payload) == 0 || bytes.Equal(bytes.TrimSpace(a.Task.Payload), []byte("null")) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -202,7 +202,7 @@ func (r *workerRuntime) event(j *runtimeAttempt, kind, code string) {
|
||||
}
|
||||
func (r *workerRuntime) accept(a WorkerAssignment) error {
|
||||
if j := r.jobs[a.AttemptID]; j != nil {
|
||||
if j.assignment.Attempt() != a.Attempt() {
|
||||
if j.assignment.Attempt() != a.Attempt() || j.assignment.ConcurrencyToken != a.ConcurrencyToken || j.assignment.Task.ConcurrencyKey != a.Task.ConcurrencyKey {
|
||||
return ErrWorkerProtocol
|
||||
}
|
||||
return nil
|
||||
@@ -553,7 +553,7 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
|
||||
task := j.assignment.Task
|
||||
task.Payload = append([]byte(nil), task.Payload...)
|
||||
task.Result = append([]byte(nil), task.Result...)
|
||||
execution := WorkerExecution{Task: task, AttemptID: j.assignment.AttemptID, Epoch: j.assignment.Epoch}
|
||||
execution := WorkerExecution{ConcurrencyToken: j.assignment.ConcurrencyToken, Task: task, AttemptID: j.assignment.AttemptID, Epoch: j.assignment.Epoch}
|
||||
handler := r.worker.handlers[WorkerCapability{Handler: task.Handler, HandlerVersion: task.HandlerVersion, PayloadVersion: task.PayloadVersion}]
|
||||
r.event(j, "handler_started", "")
|
||||
go r.worker.invoke(guard.ctx, handler, execution, j.assignment.Attempt(), r.results, r.ctx.Done())
|
||||
|
||||
@@ -16,9 +16,10 @@ var ErrWorkerAlreadyRun = errors.New("Worker instance may only Run once")
|
||||
|
||||
// WorkerExecution 是交给业务处理器的输入,不包含实例或尝试秘密。
|
||||
type WorkerExecution struct {
|
||||
Task Task // Task 固定任务输入;业务副作用应使用稳定业务身份去重。
|
||||
AttemptID string // AttemptID 当前尝试,用于核查与追踪,不代替业务幂等键。
|
||||
Epoch uint64 // Epoch 执行代次;接入方资源可用它拒绝旧尝试写入。
|
||||
ConcurrencyToken uint64 // ConcurrencyToken 同一命名空间资源键的单调序号,用于资源端拒绝旧执行写入。
|
||||
Task Task // Task 固定任务输入;业务副作用应使用稳定业务身份去重。
|
||||
AttemptID string // AttemptID 当前尝试,用于核查与追踪,不代替业务幂等键。
|
||||
Epoch uint64 // Epoch 单任务执行代次;跨任务资源隔离使用 ConcurrencyToken。
|
||||
}
|
||||
|
||||
func (e WorkerExecution) String() string {
|
||||
@@ -143,6 +144,7 @@ func NewWorker(client *Client, options WorkerOptions) (*Worker, error) {
|
||||
return nil, err
|
||||
}
|
||||
r.CancellationNotifications = true
|
||||
r.ConcurrencyKeys = true
|
||||
options.Handlers = append([]WorkerHandler(nil), options.Handlers...)
|
||||
return &Worker{client: client, options: options, registration: r, handlers: handlers, events: make(chan WorkerEvent, 128)}, nil
|
||||
}
|
||||
|
||||
@@ -8,7 +8,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// WorkerCapability 精确声明一种处理器与参数版本,不进行静默版本降级。
|
||||
@@ -21,6 +24,7 @@ type WorkerCapability struct {
|
||||
// WorkerRegistration 是同一进程重连时必须复用的身份;进程重启使用新身份。
|
||||
// 包含秘密,禁止以 JSON 等形式写入日志;SDK 的 String/GoString 会隐藏秘密。
|
||||
type WorkerRegistration struct {
|
||||
ConcurrencyKeys bool `json:"concurrency_keys,omitempty"` // ConcurrencyKeys 显式支持互斥资源键和单调序号交付,重连不可变。
|
||||
CancellationNotifications bool `json:"cancellation_notifications,omitempty"` // CancellationNotifications 接受 task.cancel 提示;旧服务不支持,须先升级服务端。
|
||||
InstanceID string `json:"instance_id"` // InstanceID 本进程随机标识,32 位小写十六进制。
|
||||
InstanceSecret string `json:"instance_secret"` // InstanceSecret 32 字节随机秘密的 base64url 编码,重连保留。
|
||||
@@ -71,12 +75,13 @@ type WorkerCapacity struct {
|
||||
// WorkerAssignment 是预占通知;必须 Start 并核验当前租约后才能运行处理器。
|
||||
// 含业务输入和执行秘密,禁止序列化到普通日志。
|
||||
type WorkerAssignment struct {
|
||||
Task Task `json:"task"` // Task 任务受理快照及业务输入。
|
||||
AttemptID string `json:"attempt_id"` // AttemptID 本次尝试标识。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 任务执行代次,隔离过时尝试。
|
||||
LeaseSecret string `json:"lease_secret"` // LeaseSecret 本次尝试的秘密,业务处理器不应接收它。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 预占期限;不能独立授权执行。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 领取时数据库时间。
|
||||
ConcurrencyToken uint64 `json:"concurrency_token,omitempty,string"` // ConcurrencyToken 资源键的单调许可序号;无资源键时为零,不能代替当前授权。
|
||||
Task Task `json:"task"` // Task 任务受理快照及业务输入。
|
||||
AttemptID string `json:"attempt_id"` // AttemptID 本次尝试标识。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 任务执行代次,隔离过时尝试。
|
||||
LeaseSecret string `json:"lease_secret"` // LeaseSecret 本次尝试的秘密,业务处理器不应接收它。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 预占期限;不能独立授权执行。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 领取时数据库时间。
|
||||
}
|
||||
|
||||
// Attempt 返回控制操作所需的证明;不要把证明转交业务处理器。
|
||||
@@ -215,3 +220,15 @@ func normalizeRegistration(r WorkerRegistration) (WorkerRegistration, error) {
|
||||
func validWorkerAttempt(a WorkerAttempt) bool {
|
||||
return validID(a.TaskID) && validID(a.AttemptID) && a.Epoch > 0 && validWorkerSecret(a.LeaseSecret)
|
||||
}
|
||||
|
||||
func validConcurrencyKey(key string) bool {
|
||||
if !utf8.ValidString(key) || utf8.RuneCountInString(key) > 191 || strings.TrimSpace(key) != key {
|
||||
return false
|
||||
}
|
||||
for _, c := range key {
|
||||
if unicode.IsControl(c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user