feat: support durable task cancellation without foreign keys

This commit is contained in:
2026-09-17 17:03:23 +08:00
parent 6805506450
commit 3ce7d2c104
48 changed files with 1642 additions and 164 deletions

View File

@@ -14,6 +14,7 @@ type runtimeAttempt struct {
startID, renewID, finishID string
guard *executionGuard
invoked, returned, lost bool
cancelRequested bool
result WorkerFinish
nextTry, nextRenew time.Time
retryDelay time.Duration
@@ -152,9 +153,11 @@ func (w *Worker) Run(parent context.Context) error {
}
}
var assignments <-chan WorkerAssignment
var cancellations <-chan WorkerCancellation
var closed <-chan struct{}
if r.conn != nil {
assignments = r.conn.assignments
cancellations = r.conn.cancellations
closed = r.conn.Done()
}
select {
@@ -175,6 +178,10 @@ func (w *Worker) Run(parent context.Context) error {
j.result = done.result
j.nextTry = time.Time{}
}
case c := <-cancellations:
if j := r.jobs[c.AttemptID]; j != nil && j.assignment.Task.ID == c.TaskID && j.assignment.Epoch == c.Epoch {
r.cancelAttempt(j)
}
case reply := <-r.replies:
if err := r.receive(reply); err != nil {
return r.failure(err)
@@ -232,19 +239,30 @@ func (r *workerRuntime) remove(j *runtimeAttempt) {
}
func (r *workerRuntime) observeExpired() {
for _, j := range r.jobs {
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost {
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost && !j.cancelRequested {
j.lost = true
r.event(j, "lease_lost", "LEASE_LOST")
}
}
}
func (r *workerRuntime) cancelAttempt(j *runtimeAttempt) {
if j.cancelRequested {
return
}
j.cancelRequested = true
if j.guard != nil {
j.guard.stop(ErrTaskCancellationRequested)
}
r.event(j, "cancel_requested", "CANCEL_REQUESTED")
}
func (r *workerRuntime) desiredCapacity() uint32 {
if r.draining {
return 0
}
ghosts := uint32(0)
for _, j := range r.jobs {
if j.returned || j.lost {
if j.returned || j.lost || j.cancelRequested {
ghosts++
}
}
@@ -320,7 +338,10 @@ func (r *workerRuntime) establish(c runtimeConnected) error {
if j == nil {
continue
}
if j.guard != nil {
if lease.CancelRequestedAt != nil {
r.cancelAttempt(j)
}
if j.guard != nil && !j.cancelRequested {
if !j.guard.update(lease) {
j.lost = true
r.event(j, "lease_lost", "LEASE_LOST")
@@ -398,7 +419,7 @@ func (r *workerRuntime) dispatch() error {
}
id = &j.finishID
case "renew":
if !j.invoked || j.lost || j.returned || now.Before(j.nextRenew) {
if !j.invoked || j.lost || j.cancelRequested || j.returned || now.Before(j.nextRenew) {
continue
}
id = &j.renewID
@@ -509,7 +530,17 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
r.remove(j)
return nil
}
if reply.result.Lease.CancelRequestedAt != nil {
r.cancelAttempt(j)
}
if command.kind == "start" {
if j.cancelRequested {
// 开始确认已收到,但本进程未调用处理器;可以明确证明本次没有副作用。
j.invoked, j.returned = true, true
j.result = cancelledHandlerResult(j.assignment.Attempt(), nil)
j.nextTry = time.Time{}
return nil
}
guard, ok := newExecutionGuard(r.ctx, reply.result.Lease)
if !ok {
r.event(j, "lease_lost", "START_NOT_AUTHORIZED")
@@ -529,14 +560,14 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
} else {
r.worker.renewals.Add(1)
j.renewID = ""
if !j.guard.update(reply.result.Lease) {
if !j.cancelRequested && !j.guard.update(reply.result.Lease) {
j.lost = true
r.event(j, "lease_lost", "LEASE_LOST")
}
}
j.nextTry = time.Time{}
j.retryDelay = 0
if !j.lost {
if !j.lost && !j.cancelRequested {
j.nextRenew = time.Now().Add(max(20*time.Millisecond, time.Until(j.guard.until())/3))
}
return nil