feat: support durable task cancellation without foreign keys
This commit is contained in:
133
sdk/go/examples/cancellation/main.go
Normal file
133
sdk/go/examples/cancellation/main.go
Normal file
@@ -0,0 +1,133 @@
|
||||
// cancellation 演示运行中取消、处理器确认停止、结果查询和有界排空。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
|
||||
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read private credentials: %w", err)
|
||||
}
|
||||
var c struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &c) != nil {
|
||||
return errors.New("invalid credentials file")
|
||||
}
|
||||
if token == "" {
|
||||
token = c.Token
|
||||
}
|
||||
if queue == "" {
|
||||
queue = c.QueueID
|
||||
}
|
||||
}
|
||||
client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operation, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := "object.cancel." + operation
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
entered := make(chan struct{})
|
||||
w, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(work context.Context, e orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
close(entered)
|
||||
<-work.Done()
|
||||
if !errors.Is(context.Cause(work), orbit.ErrTaskCancellationRequested) {
|
||||
return nil, work.Err()
|
||||
}
|
||||
// 此示例没有外部副作用,因此可以确认安全停止;真实接入需完成自己的核实。
|
||||
return nil, &orbit.WorkerCancellationConfirmed{Details: json.RawMessage(`{"evidence_ref":"objects/no-external-work"}`)}
|
||||
}}}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := client.SubmitTask(ctx, "submit-"+operation, orbit.SubmitTaskRequest{QueueID: queue, Handler: name, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"objects/example"}`), TaskTimeoutMS: 20000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 15000}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
waited := false
|
||||
defer func() {
|
||||
stop()
|
||||
if !waited {
|
||||
<-done
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
waited = true
|
||||
return fmt.Errorf("Worker stopped before handler: %v", err)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
for {
|
||||
current, err := client.GetTask(ctx, task.Task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := client.CancelTask(ctx, current.ID, key, orbit.CancelTaskRequest{ExpectedRevision: current.Revision, Reason: "finish cancellation example"})
|
||||
if err == nil {
|
||||
fmt.Printf("task=%s cancel_requested=%t accepted_state=%s\n", out.Task.ID, out.Task.CancelRequestedAt != nil, out.Task.State)
|
||||
break
|
||||
}
|
||||
var api *orbit.APIError
|
||||
if !errors.As(err, &api) || api.Code != "STATE_CONFLICT" {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for {
|
||||
current, err := client.GetTask(ctx, task.Task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.State == orbit.TaskCancelled {
|
||||
fmt.Printf("final_state=%s attempts=%d\n", current.State, current.AttemptCount)
|
||||
break
|
||||
}
|
||||
if current.State == orbit.TaskNeedsReview || current.State == orbit.TaskFailed || current.State == orbit.TaskSucceeded {
|
||||
return fmt.Errorf("unexpected result: %s", current.State)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
stop()
|
||||
err = <-done
|
||||
waited = true
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("worker_drained=true running=%d\n", w.Stats().Running)
|
||||
return nil
|
||||
}
|
||||
156
sdk/go/worker_cancel_integration_test.go
Normal file
156
sdk/go/worker_cancel_integration_test.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func cancelRuntimeTask(t *testing.T, ctx context.Context, c *Client, id string) CancelTaskResult {
|
||||
t.Helper()
|
||||
for {
|
||||
current, err := c.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := c.CancelTask(ctx, id, "cancel-"+workerTestID(t), CancelTaskRequest{ExpectedRevision: current.Revision, Reason: "controlled cancellation regression"})
|
||||
if err == nil {
|
||||
return out
|
||||
}
|
||||
var api *APIError
|
||||
if !errors.As(err, &api) || api.Code != "STATE_CONFLICT" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerRuntimeCancellation(t *testing.T) {
|
||||
api, queue, token := integrationWorkerClient(t)
|
||||
for _, mode := range []string{"confirmed", "unknown", "success", "lost_notice", "before_handler", "uncooperative"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 18*time.Second)
|
||||
defer cancel()
|
||||
g := newWorkerFaultGateway(t, api.baseURL)
|
||||
client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := "cancel." + mode + "." + workerTestID(t)
|
||||
entered := make(chan struct{})
|
||||
observed := make(chan error, 1)
|
||||
release := make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
w, err := NewWorker(client, WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 2 * time.Second, Handlers: []WorkerHandler{runtimeHandler(name, func(work context.Context, e WorkerExecution) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
close(entered)
|
||||
if mode == "uncooperative" {
|
||||
<-release
|
||||
return json.RawMessage(`{"late":true}`), nil
|
||||
}
|
||||
<-work.Done()
|
||||
observed <- context.Cause(work)
|
||||
switch mode {
|
||||
case "unknown":
|
||||
return nil, work.Err()
|
||||
case "success":
|
||||
return json.RawMessage(`{"sequence":9007199254740993}`), nil
|
||||
default:
|
||||
return nil, &WorkerCancellationConfirmed{Details: json.RawMessage(`{"evidence_ref":"objects/stopped","sequence":9007199254740993}`)}
|
||||
}
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
timeout := uint64(12000)
|
||||
if mode == "uncooperative" {
|
||||
timeout = 2000
|
||||
}
|
||||
task := runtimeTask(t, ctx, api, queue, name, timeout)
|
||||
if mode == "before_handler" {
|
||||
g.dropOffline.Store(true)
|
||||
g.drop.Store("task.start")
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
defer stop()
|
||||
defer close(release)
|
||||
if mode == "before_handler" {
|
||||
g.expectDrop(t, "task.start")
|
||||
} else {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("Worker stopped: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
}
|
||||
if mode == "lost_notice" {
|
||||
g.dropCancel.Store(true)
|
||||
}
|
||||
if mode == "confirmed" {
|
||||
g.drop.Store("task.finish")
|
||||
}
|
||||
out := cancelRuntimeTask(t, ctx, api, task.ID)
|
||||
if out.Task.State != TaskRunning || out.Task.CancelRequestedAt == nil || out.Task.CompletedAt != nil {
|
||||
t.Fatal("cancel intent reported completion")
|
||||
}
|
||||
if mode == "before_handler" {
|
||||
g.offline.Store(false)
|
||||
}
|
||||
if mode == "lost_notice" {
|
||||
g.expectDrop(t, "task.cancel")
|
||||
}
|
||||
if mode != "uncooperative" && mode != "before_handler" {
|
||||
select {
|
||||
case cause := <-observed:
|
||||
if !errors.Is(cause, ErrTaskCancellationRequested) {
|
||||
t.Fatalf("wrong handler cancellation cause: %v", cause)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("handler missed cancellation")
|
||||
}
|
||||
}
|
||||
want := TaskCancelled
|
||||
if mode == "unknown" || mode == "uncooperative" {
|
||||
want = TaskNeedsReview
|
||||
}
|
||||
if mode == "success" {
|
||||
want = TaskSucceeded
|
||||
}
|
||||
finished := waitRuntimeState(t, ctx, api, task.ID, want)
|
||||
if finished.AttemptCount != 1 {
|
||||
t.Fatal("cancel request created retry")
|
||||
}
|
||||
if mode == "success" && string(finished.Result) != `{"sequence":9007199254740993}` {
|
||||
t.Fatal("cancel overwrote success")
|
||||
}
|
||||
if mode == "before_handler" && calls.Load() != 0 {
|
||||
t.Fatal("handler started after cancellation observed")
|
||||
}
|
||||
if mode == "confirmed" {
|
||||
g.expectDrop(t, "task.finish")
|
||||
}
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if mode == "uncooperative" {
|
||||
if !errors.Is(err, ErrWorkerShutdownTimeout) {
|
||||
t.Fatalf("uncooperative handler falsely drained: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Worker did not drain")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
42
sdk/go/worker_cancel_test.go
Normal file
42
sdk/go/worker_cancel_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
func TestCancellationNotificationAndExecutionBoundary(t *testing.T) {
|
||||
r := workerTestRegistration(t)
|
||||
r.CancellationNotifications = true
|
||||
_, c := workerMock(t, r, func(ctx context.Context, c *websocket.Conn) {
|
||||
body := workerTestBody(t, WorkerCancellation{TaskID: workerTestTask, AttemptID: workerTestAttempt, Epoch: 1, RequestedAt: time.Now().UTC()})
|
||||
workerTestWrite(t, ctx, c, workerFrame{Type: "task.cancel", Body: body})
|
||||
<-ctx.Done()
|
||||
})
|
||||
ctx, stop := context.WithTimeout(context.Background(), time.Second)
|
||||
defer stop()
|
||||
notice, err := c.NextCancellation(ctx)
|
||||
if err != nil || notice.AttemptID != workerTestAttempt || notice.Epoch != 1 {
|
||||
t.Fatalf("notice: %v", err)
|
||||
}
|
||||
cancelled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
if _, err := c.NextCancellation(cancelled); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("cancelled read did not end")
|
||||
}
|
||||
if c.Err() != nil {
|
||||
t.Fatal("cancelled wait closed connection")
|
||||
}
|
||||
now := time.Now()
|
||||
lease := WorkerLease{Active: true, State: "RUNNING", ServerTime: now, LeaseUntil: now.Add(time.Second), CancelRequestedAt: &now}
|
||||
if err := lease.confirm(now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := lease.ExecutionDeadline(); ok {
|
||||
t.Fatal("cancel intent still granted execution")
|
||||
}
|
||||
}
|
||||
@@ -47,16 +47,17 @@ type workerFrame struct {
|
||||
// 本类型不自动重连、不自动续租、不运行处理器。关闭连接不等于取消业务效果。
|
||||
// 最多 32 个在途控制请求、8 个未读派发;超时关闭连接以便按原消息身份核对。
|
||||
type WorkerConnection struct {
|
||||
conn *websocket.Conn
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
welcome WorkerWelcome
|
||||
registration WorkerRegistration
|
||||
assignments chan WorkerAssignment
|
||||
done chan struct{}
|
||||
mu sync.Mutex
|
||||
err error
|
||||
pending map[string]chan workerFrame
|
||||
conn *websocket.Conn
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
welcome WorkerWelcome
|
||||
registration WorkerRegistration
|
||||
assignments chan WorkerAssignment
|
||||
cancellations chan WorkerCancellation
|
||||
done chan struct{}
|
||||
mu sync.Mutex
|
||||
err error
|
||||
pending map[string]chan workerFrame
|
||||
}
|
||||
|
||||
// DialWorker 使用 Authorization 和固定子协议连接;ctx 只控制握手。
|
||||
@@ -116,7 +117,7 @@ func (c *Client) DialWorker(ctx context.Context, registration WorkerRegistration
|
||||
return nil, ErrWorkerProtocol
|
||||
}
|
||||
lifetime, cancel := context.WithCancel(context.Background())
|
||||
w := &WorkerConnection{conn: conn, ctx: lifetime, cancel: cancel, welcome: welcome, registration: r, assignments: make(chan WorkerAssignment, 8), done: make(chan struct{}), pending: make(map[string]chan workerFrame)}
|
||||
w := &WorkerConnection{conn: conn, ctx: lifetime, cancel: cancel, welcome: welcome, registration: r, assignments: make(chan WorkerAssignment, 8), cancellations: make(chan WorkerCancellation, 32), done: make(chan struct{}), pending: make(map[string]chan workerFrame)}
|
||||
ok = true
|
||||
go w.readLoop()
|
||||
return w, nil
|
||||
@@ -175,6 +176,20 @@ func (w *WorkerConnection) readLoop() {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if f.Type == "task.cancel" {
|
||||
var c WorkerCancellation
|
||||
if !w.registration.CancellationNotifications || f.ReplyTo != "" || json.Unmarshal(f.Body, &c) != nil || !validID(c.TaskID) || !validID(c.AttemptID) || c.Epoch == 0 || c.RequestedAt.IsZero() {
|
||||
w.fail(ErrWorkerProtocol)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case w.cancellations <- c:
|
||||
default:
|
||||
w.fail(ErrWorkerBackpressure)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if f.Type != "ack" && f.Type != "error" {
|
||||
w.fail(ErrWorkerProtocol)
|
||||
return
|
||||
@@ -222,6 +237,28 @@ func (w *WorkerConnection) NextAssignment(ctx context.Context) (WorkerAssignment
|
||||
}
|
||||
}
|
||||
|
||||
// NextCancellation 等待特定尝试的停止提示;需注册 CancellationNotifications 并持续读取。
|
||||
// 提示允许迟到或重复;只取消匹配 TaskID/AttemptID/Epoch 的处理器,结果仍通过 Finish 确认。
|
||||
func (w *WorkerConnection) NextCancellation(ctx context.Context) (WorkerCancellation, error) {
|
||||
if !w.registration.CancellationNotifications {
|
||||
return WorkerCancellation{}, errors.New("cancellation notifications not enabled")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return WorkerCancellation{}, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return WorkerCancellation{}, ctx.Err()
|
||||
case <-w.ctx.Done():
|
||||
return WorkerCancellation{}, w.Err()
|
||||
case c := <-w.cancellations:
|
||||
if err := w.Err(); err != nil {
|
||||
return WorkerCancellation{}, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (w *WorkerConnection) call(ctx context.Context, kind, id string, in, out any) (time.Time, error) {
|
||||
started := time.Now()
|
||||
if !validID(id) {
|
||||
@@ -335,7 +372,7 @@ func (w *WorkerConnection) Finish(ctx context.Context, id string, in WorkerFinis
|
||||
if in.Retryable && in.Outcome != WorkerFailed {
|
||||
return out, errors.New("retryable is only valid for known failure")
|
||||
}
|
||||
if (in.Outcome != WorkerSucceeded && in.Outcome != WorkerFailed && in.Outcome != WorkerUnknown) || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) || validateWorkerJSON(in.Result) != nil {
|
||||
if (in.Outcome != WorkerSucceeded && in.Outcome != WorkerFailed && in.Outcome != WorkerUnknown && in.Outcome != WorkerCancelled) || len(in.Result) > 256<<10 || bytes.Equal(bytes.TrimSpace(in.Result), []byte("null")) || validateWorkerJSON(in.Result) != nil {
|
||||
return out, errors.New("invalid Worker result outcome or JSON")
|
||||
}
|
||||
if in.Outcome == WorkerSucceeded && in.ErrorCode != "" || in.Outcome != WorkerSucceeded && !workerName(in.ErrorCode, 64) {
|
||||
|
||||
@@ -32,6 +32,9 @@ func (w *Worker) invoke(ctx context.Context, handler WorkerHandler, execution Wo
|
||||
}()
|
||||
if ctx.Err() != nil {
|
||||
out = unknownHandlerResult(ref, "LEASE_LOST")
|
||||
if errors.Is(context.Cause(ctx), ErrTaskCancellationRequested) {
|
||||
out = cancelledHandlerResult(ref, nil)
|
||||
}
|
||||
completed = true
|
||||
return
|
||||
}
|
||||
@@ -46,7 +49,10 @@ func (w *Worker) invoke(ctx context.Context, handler WorkerHandler, execution Wo
|
||||
}
|
||||
} else {
|
||||
var known *WorkerFailure
|
||||
if errors.As(err, &known) && known != nil && workerName(known.Code, 64) {
|
||||
var cancelled *WorkerCancellationConfirmed
|
||||
if errors.As(err, &cancelled) && cancelled != nil && errors.Is(context.Cause(ctx), ErrTaskCancellationRequested) {
|
||||
out = cancelledHandlerResult(ref, cancelled.Details)
|
||||
} else if errors.As(err, &known) && known != nil && workerName(known.Code, 64) {
|
||||
details := append(json.RawMessage(nil), known.Details...)
|
||||
if len(details) == 0 {
|
||||
details = json.RawMessage(`{}`)
|
||||
@@ -64,6 +70,17 @@ func (w *Worker) invoke(ctx context.Context, handler WorkerHandler, execution Wo
|
||||
}
|
||||
completed = true
|
||||
}
|
||||
|
||||
func cancelledHandlerResult(ref WorkerAttempt, details json.RawMessage) WorkerFinish {
|
||||
details = append(json.RawMessage(nil), details...)
|
||||
if len(details) == 0 {
|
||||
details = json.RawMessage(`{}`)
|
||||
}
|
||||
if !validHandlerResult(details) {
|
||||
return unknownHandlerResult(ref, "INVALID_HANDLER_RESULT")
|
||||
}
|
||||
return WorkerFinish{Ref: ref, Outcome: WorkerCancelled, ErrorCode: "CANCEL_CONFIRMED", Result: details}
|
||||
}
|
||||
func unknownHandlerResult(ref WorkerAttempt, code string) WorkerFinish {
|
||||
return WorkerFinish{Ref: ref, Outcome: WorkerUnknown, ErrorCode: code, Result: json.RawMessage(`{}`)}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,7 @@ type workerFaultGateway struct {
|
||||
drop atomic.Value
|
||||
target atomic.Value
|
||||
dropOffline atomic.Bool
|
||||
dropCancel atomic.Bool
|
||||
offline atomic.Bool
|
||||
mu sync.Mutex
|
||||
active map[*websocket.Conn]struct{}
|
||||
@@ -137,6 +138,10 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
operation := operations[f.ReplyTo]
|
||||
delete(operations, f.ReplyTo)
|
||||
mu.Unlock()
|
||||
if f.Type == "task.cancel" && g.dropCancel.CompareAndSwap(true, false) {
|
||||
g.dropped <- "task.cancel"
|
||||
return
|
||||
}
|
||||
if f.Type == "ack" && operation != "" && g.drop.CompareAndSwap(operation, "") {
|
||||
if g.dropOffline.Load() {
|
||||
g.offline.Store(true)
|
||||
|
||||
@@ -14,6 +14,7 @@ type runtimeAttempt struct {
|
||||
startID, renewID, finishID string
|
||||
guard *executionGuard
|
||||
invoked, returned, lost bool
|
||||
cancelRequested bool
|
||||
result WorkerFinish
|
||||
nextTry, nextRenew time.Time
|
||||
retryDelay time.Duration
|
||||
@@ -152,9 +153,11 @@ func (w *Worker) Run(parent context.Context) error {
|
||||
}
|
||||
}
|
||||
var assignments <-chan WorkerAssignment
|
||||
var cancellations <-chan WorkerCancellation
|
||||
var closed <-chan struct{}
|
||||
if r.conn != nil {
|
||||
assignments = r.conn.assignments
|
||||
cancellations = r.conn.cancellations
|
||||
closed = r.conn.Done()
|
||||
}
|
||||
select {
|
||||
@@ -175,6 +178,10 @@ func (w *Worker) Run(parent context.Context) error {
|
||||
j.result = done.result
|
||||
j.nextTry = time.Time{}
|
||||
}
|
||||
case c := <-cancellations:
|
||||
if j := r.jobs[c.AttemptID]; j != nil && j.assignment.Task.ID == c.TaskID && j.assignment.Epoch == c.Epoch {
|
||||
r.cancelAttempt(j)
|
||||
}
|
||||
case reply := <-r.replies:
|
||||
if err := r.receive(reply); err != nil {
|
||||
return r.failure(err)
|
||||
@@ -232,19 +239,30 @@ func (r *workerRuntime) remove(j *runtimeAttempt) {
|
||||
}
|
||||
func (r *workerRuntime) observeExpired() {
|
||||
for _, j := range r.jobs {
|
||||
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost {
|
||||
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost && !j.cancelRequested {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *workerRuntime) cancelAttempt(j *runtimeAttempt) {
|
||||
if j.cancelRequested {
|
||||
return
|
||||
}
|
||||
j.cancelRequested = true
|
||||
if j.guard != nil {
|
||||
j.guard.stop(ErrTaskCancellationRequested)
|
||||
}
|
||||
r.event(j, "cancel_requested", "CANCEL_REQUESTED")
|
||||
}
|
||||
func (r *workerRuntime) desiredCapacity() uint32 {
|
||||
if r.draining {
|
||||
return 0
|
||||
}
|
||||
ghosts := uint32(0)
|
||||
for _, j := range r.jobs {
|
||||
if j.returned || j.lost {
|
||||
if j.returned || j.lost || j.cancelRequested {
|
||||
ghosts++
|
||||
}
|
||||
}
|
||||
@@ -320,7 +338,10 @@ func (r *workerRuntime) establish(c runtimeConnected) error {
|
||||
if j == nil {
|
||||
continue
|
||||
}
|
||||
if j.guard != nil {
|
||||
if lease.CancelRequestedAt != nil {
|
||||
r.cancelAttempt(j)
|
||||
}
|
||||
if j.guard != nil && !j.cancelRequested {
|
||||
if !j.guard.update(lease) {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
@@ -398,7 +419,7 @@ func (r *workerRuntime) dispatch() error {
|
||||
}
|
||||
id = &j.finishID
|
||||
case "renew":
|
||||
if !j.invoked || j.lost || j.returned || now.Before(j.nextRenew) {
|
||||
if !j.invoked || j.lost || j.cancelRequested || j.returned || now.Before(j.nextRenew) {
|
||||
continue
|
||||
}
|
||||
id = &j.renewID
|
||||
@@ -509,7 +530,17 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
|
||||
r.remove(j)
|
||||
return nil
|
||||
}
|
||||
if reply.result.Lease.CancelRequestedAt != nil {
|
||||
r.cancelAttempt(j)
|
||||
}
|
||||
if command.kind == "start" {
|
||||
if j.cancelRequested {
|
||||
// 开始确认已收到,但本进程未调用处理器;可以明确证明本次没有副作用。
|
||||
j.invoked, j.returned = true, true
|
||||
j.result = cancelledHandlerResult(j.assignment.Attempt(), nil)
|
||||
j.nextTry = time.Time{}
|
||||
return nil
|
||||
}
|
||||
guard, ok := newExecutionGuard(r.ctx, reply.result.Lease)
|
||||
if !ok {
|
||||
r.event(j, "lease_lost", "START_NOT_AUTHORIZED")
|
||||
@@ -529,14 +560,14 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
|
||||
} else {
|
||||
r.worker.renewals.Add(1)
|
||||
j.renewID = ""
|
||||
if !j.guard.update(reply.result.Lease) {
|
||||
if !j.cancelRequested && !j.guard.update(reply.result.Lease) {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
}
|
||||
j.nextTry = time.Time{}
|
||||
j.retryDelay = 0
|
||||
if !j.lost {
|
||||
if !j.lost && !j.cancelRequested {
|
||||
j.nextRenew = time.Now().Add(max(20*time.Millisecond, time.Until(j.guard.until())/3))
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
)
|
||||
|
||||
var ErrExecutionLeaseLost = errors.New("Worker execution lease expired or lost")
|
||||
var ErrTaskCancellationRequested = errors.New("Orbit task cancellation requested")
|
||||
var ErrWorkerShutdownTimeout = errors.New("Worker shutdown grace period expired")
|
||||
var ErrWorkerAlreadyRun = errors.New("Worker instance may only Run once")
|
||||
|
||||
@@ -41,6 +42,16 @@ type WorkerFailure struct {
|
||||
|
||||
func (e *WorkerFailure) Error() string { return "Worker handler reported a known failure" }
|
||||
|
||||
// WorkerCancellationConfirmed 仅在收到平台取消后,确认已停止且本次尝试没有未结清的外部效果时返回。
|
||||
// 普通 ctx.Err()、网络超时或无法确认外部效果仍是 UNKNOWN;它不证明先前尝试的效果已核实。
|
||||
type WorkerCancellationConfirmed struct {
|
||||
Details json.RawMessage // Details 可选证据引用,非 null JSON,最多 256 KiB;空值使用空对象。
|
||||
}
|
||||
|
||||
func (e *WorkerCancellationConfirmed) Error() string {
|
||||
return "Worker handler confirmed safe cancellation"
|
||||
}
|
||||
|
||||
// WorkerOptions 配置自动执行器。一个实例绑定一个队列,只能 Run 一次。
|
||||
type WorkerOptions struct {
|
||||
QueueID string // QueueID 凭据有消费权限的队列。
|
||||
@@ -131,6 +142,7 @@ func NewWorker(client *Client, options WorkerOptions) (*Worker, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.CancellationNotifications = true
|
||||
options.Handlers = append([]WorkerHandler(nil), options.Handlers...)
|
||||
return &Worker{client: client, options: options, registration: r, handlers: handlers, events: make(chan WorkerEvent, 128)}, nil
|
||||
}
|
||||
|
||||
@@ -21,11 +21,12 @@ type WorkerCapability struct {
|
||||
// WorkerRegistration 是同一进程重连时必须复用的身份;进程重启使用新身份。
|
||||
// 包含秘密,禁止以 JSON 等形式写入日志;SDK 的 String/GoString 会隐藏秘密。
|
||||
type WorkerRegistration struct {
|
||||
InstanceID string `json:"instance_id"` // InstanceID 本进程随机标识,32 位小写十六进制。
|
||||
InstanceSecret string `json:"instance_secret"` // InstanceSecret 32 字节随机秘密的 base64url 编码,重连保留。
|
||||
QueueID string `json:"queue_id"` // QueueID 绑定队列,重连不可更换。
|
||||
Capabilities []WorkerCapability `json:"capabilities"` // Capabilities 1–32 种精确能力,重连集合固定。
|
||||
Capacity uint32 `json:"capacity"` // Capacity 总执行槽数量,1–256;不是空闲槽的增量。
|
||||
CancellationNotifications bool `json:"cancellation_notifications,omitempty"` // CancellationNotifications 接受 task.cancel 提示;旧服务不支持,须先升级服务端。
|
||||
InstanceID string `json:"instance_id"` // InstanceID 本进程随机标识,32 位小写十六进制。
|
||||
InstanceSecret string `json:"instance_secret"` // InstanceSecret 32 字节随机秘密的 base64url 编码,重连保留。
|
||||
QueueID string `json:"queue_id"` // QueueID 绑定队列,重连不可更换。
|
||||
Capabilities []WorkerCapability `json:"capabilities"` // Capabilities 1–32 种精确能力,重连集合固定。
|
||||
Capacity uint32 `json:"capacity"` // Capacity 总执行槽数量,1–256;不是空闲槽的增量。
|
||||
}
|
||||
|
||||
// NewWorkerRegistration 生成进程身份;同进程只创建一次,并在重连时复用返回值。
|
||||
@@ -107,6 +108,7 @@ const (
|
||||
WorkerSucceeded WorkerOutcome = "SUCCEEDED"
|
||||
WorkerFailed WorkerOutcome = "FAILED"
|
||||
WorkerUnknown WorkerOutcome = "UNKNOWN"
|
||||
WorkerCancelled WorkerOutcome = "CANCELLED" // 已停止且确认本次尝试无遗留效果,必须已有平台取消意图。
|
||||
)
|
||||
|
||||
// WorkerReceipt 是不可变历史回执,不能作为当前执行权。
|
||||
@@ -121,17 +123,26 @@ type WorkerReceipt struct {
|
||||
|
||||
// WorkerLease 是本次响应重新核验的授权;必须调用 ExecutionDeadline 判断可否执行。
|
||||
type WorkerLease struct {
|
||||
Active bool `json:"active"` // Active 当前会话是否仍拥有有效尝试;LEASED 仅代表预占。
|
||||
State string `json:"state"` // State 当前尝试状态,只有 RUNNING 可执行。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 当前服务端期限,不直接和客户端墙钟比较。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 本次核验的数据库时钟。
|
||||
deadline time.Time // deadline 扣除全请求耗时和安全余量后的本地单调期限,不参与 JSON。
|
||||
CancelRequestedAt *time.Time `json:"cancel_requested_at,omitempty"` // CancelRequestedAt 非空表示应停止业务处理;仍可在原租约内确认结果。
|
||||
Active bool `json:"active"` // Active 当前会话是否仍拥有有效尝试;LEASED 仅代表预占。
|
||||
State string `json:"state"` // State 当前尝试状态,只有 RUNNING 可执行。
|
||||
LeaseUntil time.Time `json:"lease_until"` // LeaseUntil 当前服务端期限,不直接和客户端墙钟比较。
|
||||
ServerTime time.Time `json:"server_time"` // ServerTime 本次核验的数据库时钟。
|
||||
deadline time.Time // deadline 扣除全请求耗时和安全余量后的本地单调期限,不参与 JSON。
|
||||
}
|
||||
|
||||
// ExecutionDeadline 返回保守的本地执行截止时间。false 时不能运行或继续处理器。
|
||||
// 仅 Start/Renew/Resume 的 SDK 响应具有本地期限;自行 JSON 解码的值不会授权执行。
|
||||
func (l WorkerLease) ExecutionDeadline() (time.Time, bool) {
|
||||
return l.deadline, l.Active && l.State == "RUNNING" && !l.deadline.IsZero() && time.Now().Before(l.deadline)
|
||||
return l.deadline, l.Active && l.State == "RUNNING" && l.CancelRequestedAt == nil && !l.deadline.IsZero() && time.Now().Before(l.deadline)
|
||||
}
|
||||
|
||||
// WorkerCancellation 通知特定尝试停止推进,不能表示副作用已撤销或任务已终结。
|
||||
type WorkerCancellation struct {
|
||||
TaskID string `json:"task_id"` // TaskID 任务标识。
|
||||
AttemptID string `json:"attempt_id"` // AttemptID 目标尝试。
|
||||
Epoch uint64 `json:"epoch,string"` // Epoch 原执行代次,过时通知不得影响新尝试。
|
||||
RequestedAt time.Time `json:"requested_at"` // RequestedAt 首次取消意图的数据库时间。
|
||||
}
|
||||
|
||||
// WorkerCommandResult 把历史回执与当前权限分开;Deduplicated 不表示可再次执行。
|
||||
|
||||
Reference in New Issue
Block a user