feat: support durable task cancellation without foreign keys
This commit is contained in:
@@ -14,6 +14,7 @@ type runtimeAttempt struct {
|
||||
startID, renewID, finishID string
|
||||
guard *executionGuard
|
||||
invoked, returned, lost bool
|
||||
cancelRequested bool
|
||||
result WorkerFinish
|
||||
nextTry, nextRenew time.Time
|
||||
retryDelay time.Duration
|
||||
@@ -152,9 +153,11 @@ func (w *Worker) Run(parent context.Context) error {
|
||||
}
|
||||
}
|
||||
var assignments <-chan WorkerAssignment
|
||||
var cancellations <-chan WorkerCancellation
|
||||
var closed <-chan struct{}
|
||||
if r.conn != nil {
|
||||
assignments = r.conn.assignments
|
||||
cancellations = r.conn.cancellations
|
||||
closed = r.conn.Done()
|
||||
}
|
||||
select {
|
||||
@@ -175,6 +178,10 @@ func (w *Worker) Run(parent context.Context) error {
|
||||
j.result = done.result
|
||||
j.nextTry = time.Time{}
|
||||
}
|
||||
case c := <-cancellations:
|
||||
if j := r.jobs[c.AttemptID]; j != nil && j.assignment.Task.ID == c.TaskID && j.assignment.Epoch == c.Epoch {
|
||||
r.cancelAttempt(j)
|
||||
}
|
||||
case reply := <-r.replies:
|
||||
if err := r.receive(reply); err != nil {
|
||||
return r.failure(err)
|
||||
@@ -232,19 +239,30 @@ func (r *workerRuntime) remove(j *runtimeAttempt) {
|
||||
}
|
||||
func (r *workerRuntime) observeExpired() {
|
||||
for _, j := range r.jobs {
|
||||
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost {
|
||||
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost && !j.cancelRequested {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *workerRuntime) cancelAttempt(j *runtimeAttempt) {
|
||||
if j.cancelRequested {
|
||||
return
|
||||
}
|
||||
j.cancelRequested = true
|
||||
if j.guard != nil {
|
||||
j.guard.stop(ErrTaskCancellationRequested)
|
||||
}
|
||||
r.event(j, "cancel_requested", "CANCEL_REQUESTED")
|
||||
}
|
||||
func (r *workerRuntime) desiredCapacity() uint32 {
|
||||
if r.draining {
|
||||
return 0
|
||||
}
|
||||
ghosts := uint32(0)
|
||||
for _, j := range r.jobs {
|
||||
if j.returned || j.lost {
|
||||
if j.returned || j.lost || j.cancelRequested {
|
||||
ghosts++
|
||||
}
|
||||
}
|
||||
@@ -320,7 +338,10 @@ func (r *workerRuntime) establish(c runtimeConnected) error {
|
||||
if j == nil {
|
||||
continue
|
||||
}
|
||||
if j.guard != nil {
|
||||
if lease.CancelRequestedAt != nil {
|
||||
r.cancelAttempt(j)
|
||||
}
|
||||
if j.guard != nil && !j.cancelRequested {
|
||||
if !j.guard.update(lease) {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
@@ -398,7 +419,7 @@ func (r *workerRuntime) dispatch() error {
|
||||
}
|
||||
id = &j.finishID
|
||||
case "renew":
|
||||
if !j.invoked || j.lost || j.returned || now.Before(j.nextRenew) {
|
||||
if !j.invoked || j.lost || j.cancelRequested || j.returned || now.Before(j.nextRenew) {
|
||||
continue
|
||||
}
|
||||
id = &j.renewID
|
||||
@@ -509,7 +530,17 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
|
||||
r.remove(j)
|
||||
return nil
|
||||
}
|
||||
if reply.result.Lease.CancelRequestedAt != nil {
|
||||
r.cancelAttempt(j)
|
||||
}
|
||||
if command.kind == "start" {
|
||||
if j.cancelRequested {
|
||||
// 开始确认已收到,但本进程未调用处理器;可以明确证明本次没有副作用。
|
||||
j.invoked, j.returned = true, true
|
||||
j.result = cancelledHandlerResult(j.assignment.Attempt(), nil)
|
||||
j.nextTry = time.Time{}
|
||||
return nil
|
||||
}
|
||||
guard, ok := newExecutionGuard(r.ctx, reply.result.Lease)
|
||||
if !ok {
|
||||
r.event(j, "lease_lost", "START_NOT_AUTHORIZED")
|
||||
@@ -529,14 +560,14 @@ func (r *workerRuntime) receive(reply runtimeReply) error {
|
||||
} else {
|
||||
r.worker.renewals.Add(1)
|
||||
j.renewID = ""
|
||||
if !j.guard.update(reply.result.Lease) {
|
||||
if !j.cancelRequested && !j.guard.update(reply.result.Lease) {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
}
|
||||
j.nextTry = time.Time{}
|
||||
j.retryDelay = 0
|
||||
if !j.lost {
|
||||
if !j.lost && !j.cancelRequested {
|
||||
j.nextRenew = time.Now().Add(max(20*time.Millisecond, time.Until(j.guard.until())/3))
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user