feat: support durable task cancellation without foreign keys

This commit is contained in:
2026-09-17 17:03:23 +08:00
parent 6805506450
commit 3ce7d2c104
48 changed files with 1642 additions and 164 deletions

View File

@@ -78,7 +78,11 @@ func registerTasks(mux *http.ServeMux, s *service.Service) {
return nil, 0, err
}
out, err := s.CancelTask(r.Context(), t, core.ID(strings.TrimSuffix(command, ":cancel")), key, in)
return out, 200, err
status := 200
if out.Task.State == pulse.TaskRunning {
status = 202
}
return out, status, err
}))
mux.HandleFunc("POST /v1/queues", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
var in service.QueueInput

View File

@@ -248,6 +248,7 @@ func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
draining := welcome.Draining
eligible := false
leases := map[core.ID]time.Time{}
cancelSent := map[core.ID]bool{}
ticker := time.NewTicker(h.poll)
defer ticker.Stop()
for {
@@ -332,6 +333,36 @@ func (h *WorkerHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
case <-ticker.C:
// 取消与领取容量、排空无关;通知丢失后仍可由 Resume/Renew 核对持久化意图。
if input.CancellationNotifications && len(leases) > 0 {
call, stop := context.WithTimeout(ctx, 8*time.Second)
items, err := h.service.Cancellations(call, token, binding)
stop()
if err != nil {
if fault.Code(err) == "STALE_SESSION" || fault.Code(err) == "UNAUTHENTICATED" || fault.Code(err) == "PERMISSION_DENIED" {
send(workerFrame{Type: "error", Error: publicWorkerError(err)}, true)
return
}
} else {
active := make(map[core.ID]bool, len(items))
for _, item := range items {
active[item.AttemptID] = true
if cancelSent[item.AttemptID] || len(outgoing) >= 4 {
continue
}
raw, err := jsonwire.Marshal(item)
if err != nil || !send(workerFrame{Type: "task.cancel", Body: raw}, false) {
return
}
cancelSent[item.AttemptID] = true
}
for id := range cancelSent {
if !active[id] {
delete(cancelSent, id)
}
}
}
}
// 本地记录仅减少空轮询;SQL 仍原子验证总容量、队列额度和有效会话。
now := time.Now()
for attempt, until := range leases {

View File

@@ -97,6 +97,8 @@ func TestWorkerPublishedContract(t *testing.T) {
secret := strings.Repeat("A", 43)
ref := worker.AttemptRef{TaskID: id, AttemptID: id, Epoch: 1, LeaseSecret: secret}
hello := worker.ConnectInput{InstanceID: id, InstanceSecret: secret, QueueID: id, Capacity: 1, Capabilities: []worker.Capability{{Handler: "text.normalize", HandlerVersion: "v1", PayloadVersion: "v1"}}}
helloWithCancellation := hello
helloWithCancellation.CancellationNotifications = true
cases := []struct {
kind string
body any
@@ -108,6 +110,8 @@ func TestWorkerPublishedContract(t *testing.T) {
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "FAILED", ErrorCode: "OBJECT_REJECTED", Result: json.RawMessage(`{}`)}},
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "UNKNOWN", ErrorCode: "LEASE_LOST", Result: json.RawMessage(`{}`)}},
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "FAILED", Retryable: true, ErrorCode: "TEMPORARY_REJECTION", Result: json.RawMessage(`{}`)}},
{"task.finish", worker.FinishInput{Ref: ref, Outcome: "CANCELLED", ErrorCode: "CANCEL_CONFIRMED", Result: json.RawMessage(`{}`)}},
{"worker.hello", helloWithCancellation},
}
var valid [][]byte
for _, tc := range cases {
@@ -147,6 +151,7 @@ func TestWorkerPublishedContract(t *testing.T) {
{"failure_missing_code", strings.Replace(string(valid[6]), `,"error_code":"OBJECT_REJECTED"`, ``, 1)},
{"success_with_code", strings.Replace(string(valid[6]), `"FAILED"`, `"SUCCEEDED"`, 1)},
{"unknown_retry_flag", strings.Replace(string(valid[8]), `"FAILED"`, `"UNKNOWN"`, 1)},
{"cancelled_retry_flag", strings.Replace(string(valid[8]), `"FAILED"`, `"CANCELLED"`, 1)},
}
for _, tc := range invalid {
t.Run(tc.name, func(t *testing.T) {
@@ -180,7 +185,7 @@ func TestWorkerPublishedContract(t *testing.T) {
now := time.Date(2026, 9, 17, 8, 0, 0, 0, time.UTC)
welcome := worker.Welcome{SessionID: id, Epoch: 1, ServerTime: now, ExpiresAt: now.Add(45 * time.Second), LeaseDurationMS: 30000}
lease := worker.LeaseStatus{Active: true, State: pulse.AttemptRunning, LeaseUntil: now.Add(30 * time.Second), ServerTime: now}
lease := worker.LeaseStatus{Active: true, State: pulse.AttemptRunning, LeaseUntil: now.Add(30 * time.Second), ServerTime: now, CancelRequestedAt: &now}
receipt := worker.Receipt{Operation: "start", TaskState: pulse.TaskRunning, AttemptState: pulse.AttemptRunning, TaskRevision: 3, LeaseUntil: lease.LeaseUntil, Disposition: "APPLIED"}
task := service.TaskView{ID: id, NamespaceID: id, QueueID: id, State: pulse.TaskLeased, Revision: 2, Handler: "text.normalize", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), AvailableAt: now, DeadlineAt: now.Add(time.Hour), RecoverySafety: pulse.RecoveryReconcileOnly, Retry: pulse.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 30000}, AttemptCount: 1, CreatedAt: now, UpdatedAt: now}
assignment := worker.Assignment{Task: task, AttemptID: id, Epoch: 1, LeaseSecret: secret, LeaseUntil: lease.LeaseUntil, ServerTime: now}
@@ -190,10 +195,11 @@ func TestWorkerPublishedContract(t *testing.T) {
}{
{"worker.welcome", welcome}, {"ack", welcome}, {"ack", lease},
{"ack", worker.CommandResult{Receipt: receipt, Lease: lease, Deduplicated: true}}, {"task.assign", assignment},
{"task.cancel", worker.Cancellation{TaskID: id, AttemptID: id, Epoch: 1, RequestedAt: now}},
} {
raw, _ := jsonwire.Marshal(tc.body)
f := workerFrame{Version: "1", Type: tc.kind, MessageID: id, SessionID: id, ReplyTo: id, Body: raw}
if tc.kind == "task.assign" {
if tc.kind == "task.assign" || tc.kind == "task.cancel" {
f.ReplyTo = ""
}
frame, _ := jsonwire.Marshal(f)