feat: enforce durable queue resource limits and backpressure
This commit is contained in:
@@ -81,8 +81,13 @@ func queueLimit(ctx context.Context, tx *sql.Tx, ns, queue core.ID, max uint32)
|
||||
var used uint32
|
||||
var configured uint32
|
||||
err = tx.QueryRowContext(ctx, `SELECT id,used_permits,max_permits FROM pulse_concurrency_limits WHERE namespace_id=? AND scope_key='queue' AND key_digest=? FOR UPDATE`, ns, digest).Scan(&id, &used, &configured)
|
||||
if err == nil && configured != max {
|
||||
err = errors.New("queue limit configuration requires coordinated update")
|
||||
// 降额时保留已有许可;队列上限控制新的领取,存储上限至少容纳在途数。
|
||||
target := max
|
||||
if used > target {
|
||||
target = used
|
||||
}
|
||||
if err == nil && configured != target {
|
||||
_, err = tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET max_permits=?,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, target, ns, id)
|
||||
}
|
||||
return id, used, err
|
||||
}
|
||||
@@ -123,6 +128,14 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
|
||||
if used >= max {
|
||||
return nil
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
canClaim, err := queueCanClaim(ctx, tx, token.NamespaceID, state.queue, now)
|
||||
if err != nil || !canClaim {
|
||||
return err
|
||||
}
|
||||
args := []any{token.NamespaceID, state.queue}
|
||||
matches := make([]string, 0, len(state.capabilities))
|
||||
for _, c := range state.capabilities {
|
||||
@@ -465,6 +478,9 @@ func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token,
|
||||
if task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now) {
|
||||
return worker.LeaseLost
|
||||
}
|
||||
if err := consumeQueueStart(ctx, tx, token.NamespaceID, task.view.QueueID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
authorized = &now
|
||||
} else if a.State != pulse.AttemptRunning || authorized == nil {
|
||||
return fault.Conflict
|
||||
|
||||
Reference in New Issue
Block a user