k2safe/orbit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: serialize keyed tasks across queues with durable fencing tokens

Browse Source
This commit is contained in:
k2safe
2026-09-17 18:25:00 +08:00
parent 993786476e
commit 958b4a8cea
43 changed files with 1319 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
internal/storage/mysql/worker_execution.go
View File

@@ -148,7 +148,11 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if len(matches) == 0 {
return nil
}
rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
keyFilter := keyEligibility
if !state.concurrencyKeys {
keyFilter = "concurrency_key_digest IS NULL"
}
rows, err := tx.QueryContext(ctx, `SELECT id,gate_id FROM pulse_tasks WHERE namespace_id=? AND queue_id=? AND state='WAITING' AND available_at<=UTC_TIMESTAMP(6) AND deadline_at>UTC_TIMESTAMP(6) AND (start_deadline_at IS NULL OR start_deadline_at>UTC_TIMESTAMP(6)) AND attempt_count<max_attempts AND (gate_id IS NULL OR EXISTS (SELECT 1 FROM pulse_execution_gates g WHERE g.namespace_id=pulse_tasks.namespace_id AND g.id=pulse_tasks.gate_id AND g.paused=FALSE)) AND (`+strings.Join(matches, " OR ")+`) AND `+keyFilter+` ORDER BY available_at,priority DESC,id LIMIT 16`, args...)
if err != nil {
return err
}
@@ -203,6 +207,13 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if task.epoch == ^uint64(0) {
return fault.Conflict
}
keyLimitID, concurrencyToken, err := reserveConcurrencyKey(ctx, tx, token.NamespaceID, task.view.ConcurrencyKey)
if err != nil {
return err
}
if task.view.ConcurrencyKey != "" && concurrencyToken == 0 {
continue
}
attempt, err := core.NewID()
if err != nil {
return err
@@ -215,7 +226,7 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if task.view.StartDeadlineAt != nil {
until = minimumTime(until, *task.view.StartDeadlineAt)
}
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until) VALUES (?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until)
_, err = tx.ExecContext(ctx, `INSERT INTO pulse_attempts(namespace_id,id,task_id,epoch,session_id,token_digest,lease_until,concurrency_token) VALUES (?,?,?,?,?,?,?,?)`, token.NamespaceID, attempt, locked, task.epoch+1, b.SessionID, digest, until, concurrencyToken)
if err != nil {
return err
}
@@ -227,6 +238,15 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if err != nil {
return err
}
if keyLimitID != "" {
id, err := core.NewID()
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO pulse_permits(namespace_id,id,limit_id,attempt_id,lease_until) VALUES (?,?,?,?,?)`, token.NamespaceID, id, keyLimitID, attempt, until); err != nil {
return err
}
}
changed, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits+1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits<max_permits`, now, token.NamespaceID, limit)
if err := exactlyOne(changed, err); err != nil {
return err
@@ -245,7 +265,7 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if err := emitEvent(ctx, tx, token.NamespaceID, locked, view.Revision, "task", "task.leased", map[string]any{"attempt_id": attempt, "epoch": fmt.Sprint(task.epoch + 1), "session_id": b.SessionID, "lease_until": until}); err != nil {
return err
}
assignment = &worker.Assignment{Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
assignment = &worker.Assignment{ConcurrencyToken: concurrencyToken, Task: view, AttemptID: attempt, Epoch: task.epoch + 1, LeaseSecret: secret, LeaseUntil: until, ServerTime: now}
return nil
}
return nil
@@ -328,7 +348,7 @@ func (s *ExecutionStore) lockExecution(ctx context.Context, tx *sql.Tx, token au
func readAttempt(ctx context.Context, tx *sql.Tx, ns, id core.ID) (pulse.Attempt, error) {
var a pulse.Attempt
a.NamespaceID = ns
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,retryable,result,result_digest FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.Result, &a.ResultDigest)
err := tx.QueryRowContext(ctx, `SELECT id,revision,task_id,epoch,session_id,token_digest,state,lease_until,authorized_at,ended_at,error_code,retryable,result,result_digest,concurrency_token FROM pulse_attempts WHERE namespace_id=? AND id=? FOR UPDATE`, ns, id).Scan(&a.ID, &a.Revision, &a.TaskID, &a.Epoch, &a.SessionID, &a.TokenDigest, &a.State, &a.LeaseUntil, &a.AuthorizedAt, &a.EndedAt, &a.ErrorCode, &a.Retryable, &a.Result, &a.ResultDigest, &a.ConcurrencyToken)
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound
}