feat: support durable task cancellation without foreign keys

This commit is contained in:
2026-09-17 17:03:23 +08:00
parent 6805506450
commit 3ce7d2c104
48 changed files with 1642 additions and 164 deletions

View File

@@ -84,8 +84,11 @@ func TestIntegrationSchemaAndConstraints(t *testing.T) {
taskA, taskB := id(), id()
exec(taskSQL, nsA, taskA, queueA)
exec(taskSQL, nsA, taskB, queueA)
t.Run("namespace_isolation_and_safe_task_states", func(t *testing.T) {
reject(1452, taskSQL, nsB, id(), queueA)
t.Run("no_foreign_keys_and_safe_task_states", func(t *testing.T) {
var count int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil || count != 0 {
t.Fatalf("foreign keys must be absent: %d %v", count, err)
}
reject(3819, "UPDATE pulse_tasks SET state='RUNNING' WHERE namespace_id=? AND id=?", nsA, taskA)
reject(3819, "UPDATE pulse_tasks SET state='UNRECOGNIZED' WHERE namespace_id=? AND id=?", nsA, taskA)
reject(3819, "UPDATE pulse_tasks SET state='SUCCEEDED' WHERE namespace_id=? AND id=?", nsA, taskA)

View File

@@ -183,8 +183,7 @@ func Status(ctx context.Context, db *sql.DB) ([]MigrationStatus, error) {
return result, rows.Err()
}
// CheckSchema rejects missing, modified, dirty and unexpected versions. It does
// not execute DDL. A matched migration ledger is not proof against manual DDL drift.
// CheckSchema 校验迁移记录并禁止外键,不执行 DDL;不代表已核验其他手工结构漂移。
func CheckSchema(ctx context.Context, db *sql.DB) error {
wanted, err := Migrations()
if err != nil {
@@ -203,5 +202,12 @@ func CheckSchema(ctx context.Context, db *sql.DB) error {
return errors.New("schema is dirty or does not match this binary")
}
}
var foreignKeys int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND LEFT(TABLE_NAME,6) IN ('orbit_','pulse_','weave_')`).Scan(&foreignKeys); err != nil {
return err
}
if foreignKeys != 0 {
return errors.New("foreign keys are forbidden on Orbit tables; apply the supported schema before starting")
}
return nil
}

View File

@@ -0,0 +1,4 @@
ALTER TABLE pulse_attempt_evidence
DROP CHECK ck_attempt_evidence_outcome,
MODIFY COLUMN outcome VARCHAR(16) NOT NULL COMMENT 'Worker 声明的结果:SUCCEEDED、FAILED、UNKNOWN 或已确认停止且本次无遗留效果的 CANCELLED',
ADD CONSTRAINT ck_attempt_evidence_outcome CHECK(outcome IN ('SUCCEEDED','FAILED','UNKNOWN','CANCELLED'));

View File

@@ -0,0 +1,103 @@
-- 移除全部历史外键,保留现有索引、唯一约束、状态约束和数据。
-- 资源存在性及归属由服务端按命名空间在事务内校验,禁止级联删除。
ALTER TABLE orbit_namespaces
DROP FOREIGN KEY fk_orbit_namespaces_0;
ALTER TABLE orbit_credentials
DROP FOREIGN KEY fk_orbit_credentials_0;
ALTER TABLE orbit_idempotency_keys
DROP FOREIGN KEY fk_orbit_idempotency_keys_0;
ALTER TABLE orbit_events
DROP FOREIGN KEY fk_orbit_events_0;
ALTER TABLE orbit_subscriptions
DROP FOREIGN KEY fk_orbit_subscriptions_0,
DROP FOREIGN KEY fk_orbit_subscriptions_1;
ALTER TABLE orbit_deliveries
DROP FOREIGN KEY fk_orbit_deliveries_0,
DROP FOREIGN KEY fk_orbit_deliveries_1,
DROP FOREIGN KEY fk_orbit_deliveries_2;
ALTER TABLE orbit_audit_events
DROP FOREIGN KEY fk_orbit_audit_events_0;
ALTER TABLE pulse_queues
DROP FOREIGN KEY fk_pulse_queues_0;
ALTER TABLE pulse_execution_gates
DROP FOREIGN KEY fk_pulse_execution_gates_0;
ALTER TABLE pulse_worker_sessions
DROP FOREIGN KEY fk_pulse_worker_sessions_0,
DROP FOREIGN KEY fk_pulse_worker_sessions_1,
DROP FOREIGN KEY fk_worker_session_instance;
ALTER TABLE pulse_tasks
DROP FOREIGN KEY fk_pulse_tasks_0,
DROP FOREIGN KEY fk_pulse_tasks_1,
DROP FOREIGN KEY fk_pulse_tasks_2,
DROP FOREIGN KEY fk_pulse_tasks_3,
DROP FOREIGN KEY fk_pulse_tasks_4;
ALTER TABLE pulse_attempts
DROP FOREIGN KEY fk_pulse_attempts_0,
DROP FOREIGN KEY fk_pulse_attempts_1,
DROP FOREIGN KEY fk_pulse_attempts_2;
ALTER TABLE pulse_attempt_receipts
DROP FOREIGN KEY fk_pulse_attempt_receipts_0,
DROP FOREIGN KEY fk_pulse_attempt_receipts_1;
ALTER TABLE pulse_concurrency_limits
DROP FOREIGN KEY fk_pulse_concurrency_limits_0;
ALTER TABLE pulse_permits
DROP FOREIGN KEY fk_pulse_permits_0,
DROP FOREIGN KEY fk_pulse_permits_1,
DROP FOREIGN KEY fk_pulse_permits_2;
ALTER TABLE pulse_schedules
DROP FOREIGN KEY fk_pulse_schedules_0,
DROP FOREIGN KEY fk_schedule_queue;
ALTER TABLE pulse_schedule_versions
DROP FOREIGN KEY fk_pulse_schedule_versions_0,
DROP FOREIGN KEY fk_pulse_schedule_versions_1;
ALTER TABLE pulse_schedule_occurrences
DROP FOREIGN KEY fk_pulse_schedule_occurrences_0,
DROP FOREIGN KEY fk_pulse_schedule_occurrences_1,
DROP FOREIGN KEY fk_pulse_schedule_occurrences_2;
ALTER TABLE weave_definitions
DROP FOREIGN KEY fk_weave_definitions_0;
ALTER TABLE weave_definition_versions
DROP FOREIGN KEY fk_weave_definition_versions_0,
DROP FOREIGN KEY fk_weave_definition_versions_1;
ALTER TABLE weave_instances
DROP FOREIGN KEY fk_weave_instances_0,
DROP FOREIGN KEY fk_weave_instances_1,
DROP FOREIGN KEY fk_weave_instances_2,
DROP FOREIGN KEY fk_weave_instances_3;
ALTER TABLE weave_steps
DROP FOREIGN KEY fk_weave_steps_0,
DROP FOREIGN KEY fk_weave_steps_1,
DROP FOREIGN KEY fk_weave_steps_2;
ALTER TABLE weave_activations
DROP FOREIGN KEY fk_weave_activations_0,
DROP FOREIGN KEY fk_weave_activations_1,
DROP FOREIGN KEY fk_weave_activations_2;
ALTER TABLE weave_waits
DROP FOREIGN KEY fk_weave_waits_0,
DROP FOREIGN KEY fk_weave_waits_1,
DROP FOREIGN KEY fk_weave_waits_2;
ALTER TABLE weave_signals
DROP FOREIGN KEY fk_weave_signals_0,
DROP FOREIGN KEY fk_weave_signals_1,
DROP FOREIGN KEY fk_weave_signals_2;
ALTER TABLE weave_poll_states
DROP FOREIGN KEY fk_weave_poll_states_0,
DROP FOREIGN KEY fk_weave_poll_states_1,
DROP FOREIGN KEY fk_weave_poll_states_2;
ALTER TABLE weave_poll_rounds
DROP FOREIGN KEY fk_weave_poll_rounds_0,
DROP FOREIGN KEY fk_weave_poll_rounds_1,
DROP FOREIGN KEY fk_weave_poll_rounds_2;
ALTER TABLE weave_inbox
DROP FOREIGN KEY fk_weave_inbox_0,
DROP FOREIGN KEY fk_weave_inbox_1,
DROP FOREIGN KEY fk_weave_inbox_2;
ALTER TABLE weave_pending_advances
DROP FOREIGN KEY fk_weave_pending_advances_0,
DROP FOREIGN KEY fk_weave_pending_advances_1;
ALTER TABLE pulse_worker_instances
DROP FOREIGN KEY fk_worker_instance_credential,
DROP FOREIGN KEY fk_worker_instance_queue,
DROP FOREIGN KEY fk_worker_instance_session;
ALTER TABLE pulse_attempt_evidence
DROP FOREIGN KEY fk_attempt_evidence_attempt;

View File

@@ -0,0 +1,224 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"os"
"reflect"
"strings"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
mysqldriver "github.com/go-sql-driver/mysql"
)
// 重建历史 v7,验证正式升级删除所有外键且保留索引、唯一约束和已有任务。
func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 60*time.Second)
defer stop()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
name := "orbit_no_fk_upgrade_" + string(executionID(t))
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
t.Fatal(err)
}
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
t.Error(err)
}
}()
cfg, err := mysqldriver.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid test DSN")
}
cfg.DBName = name
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
exec := func(q string, args ...any) {
t.Helper()
if _, err := db.ExecContext(ctx, q, args...); err != nil {
t.Fatal(err)
}
}
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
for _, m := range migrations {
if m.Version > 7 {
break
}
statements, err := splitSQL(m.SQL)
if err != nil {
t.Fatal(err)
}
for _, statement := range statements {
exec(statement)
}
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
}
foreignKeys := func() int {
t.Helper()
var count int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil {
t.Fatal(err)
}
return count
}
if got := foreignKeys(); got != 70 {
t.Fatalf("v7 foreign keys=%d, want 70", got)
}
indexes := func() []string {
t.Helper()
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var result []string
for rows.Next() {
var value string
if err := rows.Scan(&value); err != nil {
t.Fatal(err)
}
result = append(result, value)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return result
}
beforeIndexes := indexes()
app, ns, queue, task := executionID(t), executionID(t), executionID(t), executionID(t)
exec(`INSERT INTO orbit_applications(id,name) VALUES (?,?)`, app, string(app))
exec(`INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,'test')`, ns, app, string(ns))
exec(`INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,?,4,10,'{}')`, ns, queue, string(queue))
exec(`INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,available_at,deadline_at,attempt_timeout_ms,max_attempts) VALUES (?,?,?,'object.prepare','v1','v1','{"n":9007199254740993}','{}',UTC_TIMESTAMP(6),DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR),5000,3)`, ns, task, queue)
taskSnapshot := func() []string {
t.Helper()
rows, err := db.QueryContext(ctx, `SELECT * FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, task)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
t.Fatal(err)
}
if !rows.Next() {
t.Fatal("seeded task missing")
}
values, scan := make([]sql.NullString, len(cols)), make([]any, len(cols))
for i := range values {
scan[i] = &values[i]
}
if err := rows.Scan(scan...); err != nil {
t.Fatal(err)
}
result := make([]string, len(cols))
for i, value := range values {
if value.Valid {
result[i] = cols[i] + "=" + value.String
} else {
result[i] = cols[i] + "=<NULL>"
}
}
if rows.Next() || rows.Err() != nil {
t.Fatal("unexpected task rows")
}
return result
}
beforeTask := taskSnapshot()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if got := foreignKeys(); got != 0 {
t.Fatalf("remaining foreign keys=%d", got)
}
if !reflect.DeepEqual(beforeIndexes, indexes()) {
t.Fatal("upgrade changed index definitions")
}
if !reflect.DeepEqual(beforeTask, taskSnapshot()) {
t.Fatal("upgrade changed existing task")
}
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
// 仅在一次性测试库注入违规 DDL,验证启动检查能发现迁移账本以外的外键。
exec(`ALTER TABLE pulse_queues ADD CONSTRAINT test_forbidden_fk FOREIGN KEY (namespace_id) REFERENCES orbit_namespaces(id)`)
if err := CheckSchema(ctx, db); err == nil || !strings.Contains(err.Error(), "foreign keys are forbidden") {
t.Fatalf("foreign key drift not rejected: %v", err)
}
exec(`ALTER TABLE pulse_queues DROP FOREIGN KEY test_forbidden_fk`)
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
t.Logf("v7 foreign keys=70 -> current=0; %d index components unchanged; task preserved; drift rejected", len(beforeIndexes))
}
func TestIntegrationRelationsWithoutForeignKeys(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
defer stop()
db, err := Open(ctx, dsn, 8, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
a := newScheduleFixture(t, newExecutionFixture(t, ctx, db))
b := newExecutionFixture(t, ctx, db)
for name, queue := range map[string]core.ID{"missing_queue": executionID(t), "other_namespace_queue": b.resources.QueueID} {
t.Run(name, func(t *testing.T) {
_, err := a.tasks.Submit(ctx, a.token, string(executionID(t)), service.SubmitInput{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
expectWorkerCode(t, err, "NOT_FOUND")
hello := a.hello
hello.InstanceID, hello.QueueID = executionID(t), queue
_, _, err = a.a.Connect(ctx, a.token, hello)
expectWorkerCode(t, err, "NOT_FOUND")
definition := a.definition()
definition.Task.QueueID = queue
_, err = a.api.Create(ctx, a.token, string(executionID(t)), schedule.CreateInput{Name: name, Definition: definition})
expectWorkerCode(t, err, "NOT_FOUND")
for _, table := range []string{"pulse_tasks", "pulse_schedules", "orbit_idempotency_keys"} {
if a.count(table, "1=1") != 0 {
t.Fatalf("rejected relation left %s records", table)
}
}
if a.count("pulse_worker_instances", "id=?", hello.InstanceID) != 0 || a.count("pulse_worker_sessions", "worker_instance_id=?", hello.InstanceID) != 0 {
t.Fatal("rejected relation left worker records")
}
})
}
t.Run("session_relation_must_match_process", func(t *testing.T) {
a.exec(`UPDATE pulse_worker_sessions SET worker_instance_id=? WHERE namespace_id=? AND id=?`, executionID(t), a.resources.NamespaceID, a.binding.SessionID)
_, err := a.a.Capacity(ctx, a.token, a.binding, worker.CapacityInput{Sequence: 1, Capacity: 4})
expectWorkerCode(t, err, "STALE_SESSION")
})
}

View File

@@ -0,0 +1,121 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
)
// CancelTask 将取消意图与开始授权串行化;已开始的尝试仍须确认结果或到期核查。
func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID, key string, in service.CancelInput) (service.CancelResult, error) {
var out service.CancelResult
err := s.transaction(ctx, token, "tasks:cancel", func(tx *sql.Tx, p auth.Permissions) error {
view, err := readTask(ctx, tx, token.NamespaceID, id, false)
if err != nil {
return err
}
if !p.AllowsQueue(view.QueueID) {
return fault.NotFound
}
identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "task.cancel", string(id), key, "task", id, in)
if err != nil {
return err
}
if cached != nil {
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
// 队列 → 任务 → 尝试 → 许可;取消不获取 Worker 会话锁。
if _, _, err := lockQueue(ctx, tx, token.NamespaceID, view.QueueID); err != nil {
return err
}
task, err := readExecutionTask(ctx, tx, token.NamespaceID, id)
if err != nil {
return err
}
view = task.view
if view.Revision != in.ExpectedRevision || view.State.Terminal() || view.CancelRequestedAt != nil {
return fault.Conflict
}
var owner sql.NullString
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
return err
}
if owner.Valid && owner.String != "schedule" {
return fault.Conflict
}
var attempt *pulse.Attempt
if view.State == pulse.TaskLeased || view.State == pulse.TaskRunning {
if task.current == nil {
return errors.New("active task lacks attempt")
}
a, err := readAttempt(ctx, tx, token.NamespaceID, *task.current)
if err != nil {
return err
}
if a.TaskID != id || a.Epoch != task.epoch || string(a.State) != string(view.State) {
return errors.New("task attempt identity or state mismatch")
}
attempt = &a
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
next := view.State
lease := task.lease
var completed *time.Time
if view.State == pulse.TaskWaiting || view.State == pulse.TaskLeased {
next = pulse.TaskCancelled
completed = &now
lease = nil
if view.EffectUncertain {
next, completed = pulse.TaskNeedsReview, nil
}
if attempt != nil {
if attempt.AuthorizedAt != nil {
return errors.New("leased attempt already authorized")
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='CANCELLED',ended_at=?,error_code='CANCELLED_BEFORE_START',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, token.NamespaceID, attempt.ID, attempt.Revision)
if err := exactlyOne(result, err); err != nil {
return err
}
if err := releasePermits(ctx, tx, token.NamespaceID, attempt.ID, now); err != nil {
return err
}
}
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,lease_until=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=?`, next, now, lease, completed, now, token.NamespaceID, id, in.ExpectedRevision)
if err := exactlyOne(result, err); err != nil {
return err
}
out.Task, err = readTask(ctx, tx, token.NamespaceID, id, false)
if err != nil {
return err
}
event := resultEvent(next)
if next == view.State {
event = "task.cancel_requested"
}
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", event, map[string]any{"task_id": id, "state": next, "reason": "CANCEL_REQUESTED", "effect_uncertain": view.EffectUncertain}); err != nil {
return err
}
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": view.State, "revision": fmt.Sprint(view.Revision)}, map[string]any{"state": next, "revision": fmt.Sprint(out.Task.Revision), "cancel_requested_at": now}); err != nil {
return err
}
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
})
return out, err
}

View File

@@ -8,7 +8,6 @@ import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
@@ -318,80 +317,6 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
})
return out, err
}
func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID, key string, in service.CancelInput) (service.CancelResult, error) {
var out service.CancelResult
err := s.transaction(ctx, token, "tasks:cancel", func(tx *sql.Tx, p auth.Permissions) error {
// 先以查询权限范围核对队列,再锁幂等记录,统一所有写操作的锁定顺序。
task, err := readTask(ctx, tx, token.NamespaceID, id, false)
if err != nil {
return err
}
if !p.AllowsQueue(task.QueueID) {
return fault.NotFound
}
identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "task.cancel", string(id), key, "task", id, in)
if err != nil {
return err
}
if cached != nil {
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
task, err = readTask(ctx, tx, token.NamespaceID, id, true)
if err != nil {
return err
}
if task.Revision != in.ExpectedRevision {
return fault.Conflict
}
if task.State != pulse.TaskWaiting {
return fault.Conflict
}
var owner sql.NullString
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
return err
}
if owner.Valid && owner.String != "schedule" {
return fault.Conflict
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
}
next := pulse.TaskCancelled
var completed *time.Time = &now
if task.EffectUncertain {
next = pulse.TaskNeedsReview
completed = nil
}
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='WAITING'`, next, now, completed, now, token.NamespaceID, id, in.ExpectedRevision)
if err != nil {
return err
}
n, err := result.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fault.Conflict
}
out.Task, err = readTask(ctx, tx, token.NamespaceID, id, false)
if err != nil {
return err
}
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": id, "state": out.Task.State, "reason": "CANCEL_REQUESTED", "effect_uncertain": out.Task.EffectUncertain}); err != nil {
return err
}
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": task.State, "revision": fmt.Sprint(task.Revision)}, map[string]any{"state": out.Task.State, "revision": fmt.Sprint(out.Task.Revision)}); err != nil {
return err
}
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
})
return out, err
}
func (s *TaskStore) CreateQueue(ctx context.Context, token auth.Token, key string, in service.QueueInput) (service.QueueResult, error) {
var out service.QueueResult
err := s.transaction(ctx, token, "queues:manage", func(tx *sql.Tx, p auth.Permissions) error {

View File

@@ -0,0 +1,207 @@
package mysql
import (
"context"
"encoding/json"
"os"
"sync"
"testing"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
func TestIntegrationWorkerCancellation(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
defer stop()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
fixture := func(t *testing.T) *executionFixture { return newExecutionFixture(t, ctx, db) }
cancel := func(f *executionFixture, task service.TaskView) service.CancelResult {
t.Helper()
out, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "stop controlled test"})
if err != nil {
t.Fatal(err)
}
return out
}
t.Run("leased_cancel_prevents_start_and_releases_once", func(t *testing.T) {
f := fixture(t)
f.submit(3)
a, ref := f.claim()
out := cancel(f, a.Task)
if out.Task.State != pulse.TaskCancelled || out.Task.CompletedAt == nil || f.used() != 0 {
t.Fatal("leased cancel did not settle")
}
_, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
expectWorkerCode(t, err, "LEASE_LOST")
late := retryFinish(t, f, ref, "SUCCEEDED", false)
if late.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(a.Task.ID).State != pulse.TaskCancelled || f.used() != 0 {
t.Fatal("late result changed cancelled task")
}
})
for _, outcome := range []string{"SUCCEEDED", "FAILED", "UNKNOWN", "CANCELLED"} {
t.Run("running_"+outcome, func(t *testing.T) {
f := fixture(t)
f.submit(3)
_, ref := f.claim()
startID := executionID(t)
started, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
if err != nil {
t.Fatal(err)
}
before := f.get(ref.TaskID)
key := string(executionID(t))
input := service.CancelInput{ExpectedRevision: before.Revision, Reason: "cancel running test"}
out, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, input)
if err != nil {
t.Fatal(err)
}
if out.Task.State != pulse.TaskRunning || out.Task.CancelRequestedAt == nil || out.Task.CompletedAt != nil || f.used() != 1 {
t.Fatal("intent prematurely settled execution")
}
replayed, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
if err != nil || !replayed.Deduplicated || replayed.Lease.CancelRequestedAt == nil || !replayed.Lease.Active {
t.Fatal("old start ACK lost current cancellation")
}
renew, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
if err != nil || !renew.Lease.LeaseUntil.Equal(started.Lease.LeaseUntil) || renew.Lease.CancelRequestedAt == nil {
t.Fatal("cancelled lease extended")
}
notices, err := f.a.Cancellations(ctx, f.token, f.binding)
if err != nil || len(notices) != 1 || notices[0].AttemptID != ref.AttemptID {
t.Fatal("missing persisted notice")
}
// 新会话隔离旧连接,Resume 后仍能取得取消意图并确认原尝试。
binding, _, err := f.b.Connect(ctx, f.token, f.hello)
if err != nil {
t.Fatal(err)
}
_, err = f.a.Cancellations(ctx, f.token, f.binding)
expectWorkerCode(t, err, "STALE_SESSION")
lease, err := f.b.Resume(ctx, f.token, binding, ref)
if err != nil || lease.CancelRequestedAt == nil || !lease.Active {
t.Fatal("reconnect lost intent")
}
f.a, f.binding = f.b, binding
finished := retryFinish(t, f, ref, outcome, outcome == "FAILED")
want := pulse.TaskState(outcome)
if outcome == "UNKNOWN" {
want = pulse.TaskNeedsReview
}
if finished.Receipt.TaskState != want || f.used() != 0 || f.get(ref.TaskID).AttemptCount != 1 {
t.Fatal("wrong cancellation resolution or retry")
}
again, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, input)
if err != nil || !again.Deduplicated || again.Task.State != pulse.TaskRunning || again.Task.Revision != out.Task.Revision {
t.Fatal("cancel receipt changed after completion")
}
if lenMust := f.count("orbit_events", "aggregate_id=? AND type='task.cancel_requested'", ref.TaskID); lenMust != 1 {
t.Fatal("duplicate intent event")
}
})
}
t.Run("prior_unknown_not_erased_and_late_cancel_is_evidence", func(t *testing.T) {
f := fixture(t)
task := retryTask(t, f, 3, pulse.RecoveryIdempotent)
first := retryStart(t, f)
retryFinish(t, f, first, "UNKNOWN", false)
pending := f.get(task.ID)
timer := time.NewTimer(time.Until(pending.AvailableAt.Add(10 * time.Millisecond)))
defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
second := retryStart(t, f)
cancel(f, f.get(task.ID))
retryFinish(t, f, second, "CANCELLED", false)
got := f.get(task.ID)
if got.State != pulse.TaskNeedsReview || !got.EffectUncertain || got.CompletedAt != nil || got.AttemptCount != 2 {
t.Fatal("current cancellation erased earlier unknown effect")
}
})
t.Run("unrequested_cancel_is_rejected", func(t *testing.T) {
f := fixture(t)
f.submit(1)
ref := retryStart(t, f)
_, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "CANCELLED", ErrorCode: "CANCEL_CONFIRMED", Result: json.RawMessage(`{}`)})
expectWorkerCode(t, err, "STATE_CONFLICT")
if f.get(ref.TaskID).State != pulse.TaskRunning || f.used() != 1 {
t.Fatal("unrequested cancellation changed execution")
}
})
t.Run("intent_rollback_and_expiry_no_retry", func(t *testing.T) {
f := fixture(t)
f.submit(3)
ref := retryStart(t, f)
before := f.get(ref.TaskID)
key := string(executionID(t))
in := service.CancelInput{ExpectedRevision: before.Revision, Reason: "rollback test"}
f.exec(`CREATE TRIGGER orbit_test_cancel BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled cancel failure'`)
_, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, in)
f.exec(`DROP TRIGGER orbit_test_cancel`)
if err == nil || f.get(ref.TaskID).CancelRequestedAt != nil || f.get(ref.TaskID).Revision != before.Revision {
t.Fatal("partial cancel transaction")
}
if _, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, in); err != nil {
t.Fatal(err)
}
f.expire(ref)
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
if f.get(ref.TaskID).State != pulse.TaskNeedsReview || f.used() != 0 {
t.Fatal("expired running cancel became false cancellation")
}
late := retryFinish(t, f, ref, "CANCELLED", false)
if late.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(ref.TaskID).State != pulse.TaskNeedsReview {
t.Fatal("late cancellation overrode unknown")
}
})
t.Run("start_and_cancel_race", func(t *testing.T) {
for i := 0; i < 12; i++ {
f := fixture(t)
f.submit(1)
a, ref := f.claim()
var startErr, cancelErr error
var out service.CancelResult
gate := make(chan struct{})
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); <-gate; _, startErr = f.a.Start(ctx, f.token, f.binding, executionID(t), ref) }()
go func() {
defer wg.Done()
<-gate
out, cancelErr = f.tasks.CancelTask(ctx, f.token, ref.TaskID, string(executionID(t)), service.CancelInput{ExpectedRevision: a.Task.Revision, Reason: "race"})
}()
close(gate)
wg.Wait()
if cancelErr == nil {
if fault.Code(startErr) != "LEASE_LOST" || out.Task.State != pulse.TaskCancelled || f.used() != 0 {
t.Fatalf("cancel winner: start=%v task=%s used=%d", startErr, out.Task.State, f.used())
}
} else {
if fault.Code(cancelErr) != "STATE_CONFLICT" || startErr != nil || f.get(ref.TaskID).State != pulse.TaskRunning {
t.Fatal("invalid race resolution")
}
cancel(f, f.get(ref.TaskID))
retryFinish(t, f, ref, "CANCELLED", false)
}
}
})
}

View File

@@ -0,0 +1,35 @@
package mysql
import (
"context"
"database/sql"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
// Cancellations 读取当前有效会话持有的取消意图;断线后由新会话 Resume 和扫描重新交付。
// 通知允许重复或迟到,不能代替任务结果;256 对应单 Worker 最大持久化容量。
func (s *ExecutionStore) Cancellations(ctx context.Context, token auth.Token, b worker.Binding) ([]worker.Cancellation, error) {
out := []worker.Cancellation{}
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
state, err := s.lockSession(ctx, tx, token, b, p)
if err != nil {
return err
}
rows, err := tx.QueryContext(ctx, `SELECT t.id,a.id,a.epoch,t.cancel_requested_at FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id AND t.current_attempt_id=a.id WHERE a.namespace_id=? AND a.session_id=? AND a.state='RUNNING' AND t.state='RUNNING' AND t.queue_id=? AND t.cancel_requested_at IS NOT NULL ORDER BY a.id LIMIT 256`, token.NamespaceID, b.SessionID, state.queue)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var c worker.Cancellation
if err := rows.Scan(&c.TaskID, &c.AttemptID, &c.Epoch, &c.RequestedAt); err != nil {
return err
}
out = append(out, c)
}
return rows.Err()
})
return out, err
}

View File

@@ -181,7 +181,7 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if !state.expires.After(now) {
return worker.StaleSession
}
if task.view.State != pulse.TaskWaiting || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
if task.view.State != pulse.TaskWaiting || task.view.CancelRequestedAt != nil || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
continue
}
if task.epoch == ^uint64(0) {
@@ -327,7 +327,7 @@ func leaseStatus(rows executionRows, b worker.Binding, now time.Time) worker.Lea
active = active && deadline.After(now)
until = minimumTime(until, deadline)
}
return worker.LeaseStatus{Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
return worker.LeaseStatus{CancelRequestedAt: t.view.CancelRequestedAt, Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
}
func attemptIdentity(ref worker.AttemptRef) any {
return struct {
@@ -453,6 +453,9 @@ func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token,
task, a := rows.task, rows.attempt
authorized := a.AuthorizedAt
if operation == "start" {
if task.view.CancelRequestedAt != nil {
return worker.LeaseLost
}
if a.State != pulse.AttemptLeased || task.view.State != pulse.TaskLeased {
return fault.Conflict
}
@@ -466,6 +469,15 @@ func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token,
} else if a.State != pulse.AttemptRunning || authorized == nil {
return fault.Conflict
}
if task.view.CancelRequestedAt != nil {
// 取消后不延长执行窗口;当前权威结果仍可在原租约内提交。
receipt := worker.Receipt{Operation: operation, TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: status.LeaseUntil, Disposition: "APPLIED"}
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, operation, digest, receipt); err != nil {
return err
}
out = worker.CommandResult{Receipt: receipt, Lease: status}
return nil
}
until := minimumTime(now.Add(s.options.LeaseDuration), authorized.Add(time.Duration(task.attemptTimeout)*time.Millisecond), task.view.DeadlineAt, rows.session.expires)
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='RUNNING',authorized_at=?,lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, authorized, until, now, token.NamespaceID, a.ID, a.Revision)
if err := exactlyOne(result, err); err != nil {

View File

@@ -59,7 +59,7 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
out.Examined++
var changed pulse.TaskState
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
// 与 Worker 命令一致先锁队列;有其他节点正在处理时跳过,本批不占用等待线程。
// 队列 → 任务 → 尝试 → 许可;忙碌资源留给下一批。
var queue core.ID
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
if errors.Is(err, sql.ErrNoRows) {

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
@@ -62,6 +63,9 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true}
return nil
}
if in.Outcome == "CANCELLED" && task.view.CancelRequestedAt == nil {
return fault.Conflict
}
reason := ""
if a.ResultDigest != nil {
reason = "RESULT_CONFLICT"

View File

@@ -98,6 +98,15 @@ func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worke
if registration != storedRegistration || epoch == ^uint64(0) {
return fault.Conflict
}
// 不依赖数据库外键:创建会话前验证队列存在且属于当前命名空间。
// 进程头在队列之前加锁,与执行命令保持一致;失败时回滚新建的进程头。
var queue core.ID
if err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, token.NamespaceID, in.QueueID).Scan(&queue); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return fault.NotFound
}
return err
}
now, err := dbNow(ctx, tx)
if err != nil {
return err
@@ -160,9 +169,13 @@ func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth
if !p.AllowsQueue(state.queue) {
return state, fault.Forbidden
}
var node core.ID
var node, instance, sessionCredential core.ID
var sessionEpoch uint64
var raw []byte
err = tx.QueryRowContext(ctx, `SELECT node_id,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &state.capacity, &state.sequence, &raw, &state.expires)
err = tx.QueryRowContext(ctx, `SELECT node_id,worker_instance_id,credential_id,session_epoch,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &instance, &sessionCredential, &sessionEpoch, &state.capacity, &state.sequence, &raw, &state.expires)
if errors.Is(err, sql.ErrNoRows) {
return state, worker.StaleSession
}
if err != nil {
return state, err
}
@@ -170,7 +183,7 @@ func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth
if err != nil {
return state, err
}
if node != s.node || !state.expires.After(now) {
if node != s.node || instance != b.InstanceID || sessionCredential != token.CredentialID || sessionEpoch != b.Epoch || !state.expires.After(now) {
return state, worker.StaleSession
}
if err := json.Unmarshal(raw, &state.capabilities); err != nil {