feat: support durable task cancellation without foreign keys
This commit is contained in:
@@ -84,8 +84,11 @@ func TestIntegrationSchemaAndConstraints(t *testing.T) {
|
||||
taskA, taskB := id(), id()
|
||||
exec(taskSQL, nsA, taskA, queueA)
|
||||
exec(taskSQL, nsA, taskB, queueA)
|
||||
t.Run("namespace_isolation_and_safe_task_states", func(t *testing.T) {
|
||||
reject(1452, taskSQL, nsB, id(), queueA)
|
||||
t.Run("no_foreign_keys_and_safe_task_states", func(t *testing.T) {
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("foreign keys must be absent: %d %v", count, err)
|
||||
}
|
||||
reject(3819, "UPDATE pulse_tasks SET state='RUNNING' WHERE namespace_id=? AND id=?", nsA, taskA)
|
||||
reject(3819, "UPDATE pulse_tasks SET state='UNRECOGNIZED' WHERE namespace_id=? AND id=?", nsA, taskA)
|
||||
reject(3819, "UPDATE pulse_tasks SET state='SUCCEEDED' WHERE namespace_id=? AND id=?", nsA, taskA)
|
||||
|
||||
@@ -183,8 +183,7 @@ func Status(ctx context.Context, db *sql.DB) ([]MigrationStatus, error) {
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// CheckSchema rejects missing, modified, dirty and unexpected versions. It does
|
||||
// not execute DDL. A matched migration ledger is not proof against manual DDL drift.
|
||||
// CheckSchema 校验迁移记录并禁止外键,不执行 DDL;不代表已核验其他手工结构漂移。
|
||||
func CheckSchema(ctx context.Context, db *sql.DB) error {
|
||||
wanted, err := Migrations()
|
||||
if err != nil {
|
||||
@@ -203,5 +202,12 @@ func CheckSchema(ctx context.Context, db *sql.DB) error {
|
||||
return errors.New("schema is dirty or does not match this binary")
|
||||
}
|
||||
}
|
||||
var foreignKeys int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND LEFT(TABLE_NAME,6) IN ('orbit_','pulse_','weave_')`).Scan(&foreignKeys); err != nil {
|
||||
return err
|
||||
}
|
||||
if foreignKeys != 0 {
|
||||
return errors.New("foreign keys are forbidden on Orbit tables; apply the supported schema before starting")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE pulse_attempt_evidence
|
||||
DROP CHECK ck_attempt_evidence_outcome,
|
||||
MODIFY COLUMN outcome VARCHAR(16) NOT NULL COMMENT 'Worker 声明的结果:SUCCEEDED、FAILED、UNKNOWN 或已确认停止且本次无遗留效果的 CANCELLED',
|
||||
ADD CONSTRAINT ck_attempt_evidence_outcome CHECK(outcome IN ('SUCCEEDED','FAILED','UNKNOWN','CANCELLED'));
|
||||
103
internal/storage/mysql/migrations/000008_no_foreign_keys.up.sql
Normal file
103
internal/storage/mysql/migrations/000008_no_foreign_keys.up.sql
Normal file
@@ -0,0 +1,103 @@
|
||||
-- 移除全部历史外键,保留现有索引、唯一约束、状态约束和数据。
|
||||
-- 资源存在性及归属由服务端按命名空间在事务内校验,禁止级联删除。
|
||||
ALTER TABLE orbit_namespaces
|
||||
DROP FOREIGN KEY fk_orbit_namespaces_0;
|
||||
ALTER TABLE orbit_credentials
|
||||
DROP FOREIGN KEY fk_orbit_credentials_0;
|
||||
ALTER TABLE orbit_idempotency_keys
|
||||
DROP FOREIGN KEY fk_orbit_idempotency_keys_0;
|
||||
ALTER TABLE orbit_events
|
||||
DROP FOREIGN KEY fk_orbit_events_0;
|
||||
ALTER TABLE orbit_subscriptions
|
||||
DROP FOREIGN KEY fk_orbit_subscriptions_0,
|
||||
DROP FOREIGN KEY fk_orbit_subscriptions_1;
|
||||
ALTER TABLE orbit_deliveries
|
||||
DROP FOREIGN KEY fk_orbit_deliveries_0,
|
||||
DROP FOREIGN KEY fk_orbit_deliveries_1,
|
||||
DROP FOREIGN KEY fk_orbit_deliveries_2;
|
||||
ALTER TABLE orbit_audit_events
|
||||
DROP FOREIGN KEY fk_orbit_audit_events_0;
|
||||
ALTER TABLE pulse_queues
|
||||
DROP FOREIGN KEY fk_pulse_queues_0;
|
||||
ALTER TABLE pulse_execution_gates
|
||||
DROP FOREIGN KEY fk_pulse_execution_gates_0;
|
||||
ALTER TABLE pulse_worker_sessions
|
||||
DROP FOREIGN KEY fk_pulse_worker_sessions_0,
|
||||
DROP FOREIGN KEY fk_pulse_worker_sessions_1,
|
||||
DROP FOREIGN KEY fk_worker_session_instance;
|
||||
ALTER TABLE pulse_tasks
|
||||
DROP FOREIGN KEY fk_pulse_tasks_0,
|
||||
DROP FOREIGN KEY fk_pulse_tasks_1,
|
||||
DROP FOREIGN KEY fk_pulse_tasks_2,
|
||||
DROP FOREIGN KEY fk_pulse_tasks_3,
|
||||
DROP FOREIGN KEY fk_pulse_tasks_4;
|
||||
ALTER TABLE pulse_attempts
|
||||
DROP FOREIGN KEY fk_pulse_attempts_0,
|
||||
DROP FOREIGN KEY fk_pulse_attempts_1,
|
||||
DROP FOREIGN KEY fk_pulse_attempts_2;
|
||||
ALTER TABLE pulse_attempt_receipts
|
||||
DROP FOREIGN KEY fk_pulse_attempt_receipts_0,
|
||||
DROP FOREIGN KEY fk_pulse_attempt_receipts_1;
|
||||
ALTER TABLE pulse_concurrency_limits
|
||||
DROP FOREIGN KEY fk_pulse_concurrency_limits_0;
|
||||
ALTER TABLE pulse_permits
|
||||
DROP FOREIGN KEY fk_pulse_permits_0,
|
||||
DROP FOREIGN KEY fk_pulse_permits_1,
|
||||
DROP FOREIGN KEY fk_pulse_permits_2;
|
||||
ALTER TABLE pulse_schedules
|
||||
DROP FOREIGN KEY fk_pulse_schedules_0,
|
||||
DROP FOREIGN KEY fk_schedule_queue;
|
||||
ALTER TABLE pulse_schedule_versions
|
||||
DROP FOREIGN KEY fk_pulse_schedule_versions_0,
|
||||
DROP FOREIGN KEY fk_pulse_schedule_versions_1;
|
||||
ALTER TABLE pulse_schedule_occurrences
|
||||
DROP FOREIGN KEY fk_pulse_schedule_occurrences_0,
|
||||
DROP FOREIGN KEY fk_pulse_schedule_occurrences_1,
|
||||
DROP FOREIGN KEY fk_pulse_schedule_occurrences_2;
|
||||
ALTER TABLE weave_definitions
|
||||
DROP FOREIGN KEY fk_weave_definitions_0;
|
||||
ALTER TABLE weave_definition_versions
|
||||
DROP FOREIGN KEY fk_weave_definition_versions_0,
|
||||
DROP FOREIGN KEY fk_weave_definition_versions_1;
|
||||
ALTER TABLE weave_instances
|
||||
DROP FOREIGN KEY fk_weave_instances_0,
|
||||
DROP FOREIGN KEY fk_weave_instances_1,
|
||||
DROP FOREIGN KEY fk_weave_instances_2,
|
||||
DROP FOREIGN KEY fk_weave_instances_3;
|
||||
ALTER TABLE weave_steps
|
||||
DROP FOREIGN KEY fk_weave_steps_0,
|
||||
DROP FOREIGN KEY fk_weave_steps_1,
|
||||
DROP FOREIGN KEY fk_weave_steps_2;
|
||||
ALTER TABLE weave_activations
|
||||
DROP FOREIGN KEY fk_weave_activations_0,
|
||||
DROP FOREIGN KEY fk_weave_activations_1,
|
||||
DROP FOREIGN KEY fk_weave_activations_2;
|
||||
ALTER TABLE weave_waits
|
||||
DROP FOREIGN KEY fk_weave_waits_0,
|
||||
DROP FOREIGN KEY fk_weave_waits_1,
|
||||
DROP FOREIGN KEY fk_weave_waits_2;
|
||||
ALTER TABLE weave_signals
|
||||
DROP FOREIGN KEY fk_weave_signals_0,
|
||||
DROP FOREIGN KEY fk_weave_signals_1,
|
||||
DROP FOREIGN KEY fk_weave_signals_2;
|
||||
ALTER TABLE weave_poll_states
|
||||
DROP FOREIGN KEY fk_weave_poll_states_0,
|
||||
DROP FOREIGN KEY fk_weave_poll_states_1,
|
||||
DROP FOREIGN KEY fk_weave_poll_states_2;
|
||||
ALTER TABLE weave_poll_rounds
|
||||
DROP FOREIGN KEY fk_weave_poll_rounds_0,
|
||||
DROP FOREIGN KEY fk_weave_poll_rounds_1,
|
||||
DROP FOREIGN KEY fk_weave_poll_rounds_2;
|
||||
ALTER TABLE weave_inbox
|
||||
DROP FOREIGN KEY fk_weave_inbox_0,
|
||||
DROP FOREIGN KEY fk_weave_inbox_1,
|
||||
DROP FOREIGN KEY fk_weave_inbox_2;
|
||||
ALTER TABLE weave_pending_advances
|
||||
DROP FOREIGN KEY fk_weave_pending_advances_0,
|
||||
DROP FOREIGN KEY fk_weave_pending_advances_1;
|
||||
ALTER TABLE pulse_worker_instances
|
||||
DROP FOREIGN KEY fk_worker_instance_credential,
|
||||
DROP FOREIGN KEY fk_worker_instance_queue,
|
||||
DROP FOREIGN KEY fk_worker_instance_session;
|
||||
ALTER TABLE pulse_attempt_evidence
|
||||
DROP FOREIGN KEY fk_attempt_evidence_attempt;
|
||||
224
internal/storage/mysql/no_foreign_keys_integration_test.go
Normal file
224
internal/storage/mysql/no_foreign_keys_integration_test.go
Normal file
@@ -0,0 +1,224 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/schedule"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// 重建历史 v7,验证正式升级删除所有外键且保留索引、唯一约束和已有任务。
|
||||
func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer stop()
|
||||
admin, err := Open(ctx, dsn, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
name := "orbit_no_fk_upgrade_" + string(executionID(t))
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
cfg, err := mysqldriver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test DSN")
|
||||
}
|
||||
cfg.DBName = name
|
||||
db, err := Open(ctx, cfg.FormatDSN(), 4, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE orbit_schema_migrations (version BIGINT UNSIGNED PRIMARY KEY,name VARCHAR(191) NOT NULL,checksum CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,dirty BOOLEAN NOT NULL,applied_at DATETIME(6) NULL) ENGINE=InnoDB`)
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if m.Version > 7 {
|
||||
break
|
||||
}
|
||||
statements, err := splitSQL(m.SQL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, statement := range statements {
|
||||
exec(statement)
|
||||
}
|
||||
exec(`INSERT INTO orbit_schema_migrations(version,name,checksum,dirty,applied_at) VALUES (?,?,?,FALSE,UTC_TIMESTAMP(6))`, m.Version, m.Name, m.Checksum)
|
||||
}
|
||||
foreignKeys := func() int {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return count
|
||||
}
|
||||
if got := foreignKeys(); got != 70 {
|
||||
t.Fatalf("v7 foreign keys=%d, want 70", got)
|
||||
}
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []string
|
||||
for rows.Next() {
|
||||
var value string
|
||||
if err := rows.Scan(&value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result = append(result, value)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
beforeIndexes := indexes()
|
||||
app, ns, queue, task := executionID(t), executionID(t), executionID(t), executionID(t)
|
||||
exec(`INSERT INTO orbit_applications(id,name) VALUES (?,?)`, app, string(app))
|
||||
exec(`INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,'test')`, ns, app, string(ns))
|
||||
exec(`INSERT INTO pulse_queues(namespace_id,id,name,max_concurrency,rate_per_second,policy) VALUES (?,?,?,4,10,'{}')`, ns, queue, string(queue))
|
||||
exec(`INSERT INTO pulse_tasks(namespace_id,id,queue_id,handler,handler_version,payload_version,payload,policy_snapshot,available_at,deadline_at,attempt_timeout_ms,max_attempts) VALUES (?,?,?,'object.prepare','v1','v1','{"n":9007199254740993}','{}',UTC_TIMESTAMP(6),DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR),5000,3)`, ns, task, queue)
|
||||
taskSnapshot := func() []string {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, `SELECT * FROM pulse_tasks WHERE namespace_id=? AND id=?`, ns, task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !rows.Next() {
|
||||
t.Fatal("seeded task missing")
|
||||
}
|
||||
values, scan := make([]sql.NullString, len(cols)), make([]any, len(cols))
|
||||
for i := range values {
|
||||
scan[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(scan...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := make([]string, len(cols))
|
||||
for i, value := range values {
|
||||
if value.Valid {
|
||||
result[i] = cols[i] + "=" + value.String
|
||||
} else {
|
||||
result[i] = cols[i] + "=<NULL>"
|
||||
}
|
||||
}
|
||||
if rows.Next() || rows.Err() != nil {
|
||||
t.Fatal("unexpected task rows")
|
||||
}
|
||||
return result
|
||||
}
|
||||
beforeTask := taskSnapshot()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := foreignKeys(); got != 0 {
|
||||
t.Fatalf("remaining foreign keys=%d", got)
|
||||
}
|
||||
if !reflect.DeepEqual(beforeIndexes, indexes()) {
|
||||
t.Fatal("upgrade changed index definitions")
|
||||
}
|
||||
if !reflect.DeepEqual(beforeTask, taskSnapshot()) {
|
||||
t.Fatal("upgrade changed existing task")
|
||||
}
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 仅在一次性测试库注入违规 DDL,验证启动检查能发现迁移账本以外的外键。
|
||||
exec(`ALTER TABLE pulse_queues ADD CONSTRAINT test_forbidden_fk FOREIGN KEY (namespace_id) REFERENCES orbit_namespaces(id)`)
|
||||
if err := CheckSchema(ctx, db); err == nil || !strings.Contains(err.Error(), "foreign keys are forbidden") {
|
||||
t.Fatalf("foreign key drift not rejected: %v", err)
|
||||
}
|
||||
exec(`ALTER TABLE pulse_queues DROP FOREIGN KEY test_forbidden_fk`)
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("v7 foreign keys=70 -> current=0; %d index components unchanged; task preserved; drift rejected", len(beforeIndexes))
|
||||
}
|
||||
|
||||
func TestIntegrationRelationsWithoutForeignKeys(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 8, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := newScheduleFixture(t, newExecutionFixture(t, ctx, db))
|
||||
b := newExecutionFixture(t, ctx, db)
|
||||
for name, queue := range map[string]core.ID{"missing_queue": executionID(t), "other_namespace_queue": b.resources.QueueID} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := a.tasks.Submit(ctx, a.token, string(executionID(t)), service.SubmitInput{QueueID: queue, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)})
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
hello := a.hello
|
||||
hello.InstanceID, hello.QueueID = executionID(t), queue
|
||||
_, _, err = a.a.Connect(ctx, a.token, hello)
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
definition := a.definition()
|
||||
definition.Task.QueueID = queue
|
||||
_, err = a.api.Create(ctx, a.token, string(executionID(t)), schedule.CreateInput{Name: name, Definition: definition})
|
||||
expectWorkerCode(t, err, "NOT_FOUND")
|
||||
for _, table := range []string{"pulse_tasks", "pulse_schedules", "orbit_idempotency_keys"} {
|
||||
if a.count(table, "1=1") != 0 {
|
||||
t.Fatalf("rejected relation left %s records", table)
|
||||
}
|
||||
}
|
||||
if a.count("pulse_worker_instances", "id=?", hello.InstanceID) != 0 || a.count("pulse_worker_sessions", "worker_instance_id=?", hello.InstanceID) != 0 {
|
||||
t.Fatal("rejected relation left worker records")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("session_relation_must_match_process", func(t *testing.T) {
|
||||
a.exec(`UPDATE pulse_worker_sessions SET worker_instance_id=? WHERE namespace_id=? AND id=?`, executionID(t), a.resources.NamespaceID, a.binding.SessionID)
|
||||
_, err := a.a.Capacity(ctx, a.token, a.binding, worker.CapacityInput{Sequence: 1, Capacity: 4})
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
})
|
||||
}
|
||||
121
internal/storage/mysql/task_cancel.go
Normal file
121
internal/storage/mysql/task_cancel.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
)
|
||||
|
||||
// CancelTask 将取消意图与开始授权串行化;已开始的尝试仍须确认结果或到期核查。
|
||||
func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID, key string, in service.CancelInput) (service.CancelResult, error) {
|
||||
var out service.CancelResult
|
||||
err := s.transaction(ctx, token, "tasks:cancel", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
view, err := readTask(ctx, tx, token.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.AllowsQueue(view.QueueID) {
|
||||
return fault.NotFound
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "task.cancel", string(id), key, "task", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
// 队列 → 任务 → 尝试 → 许可;取消不获取 Worker 会话锁。
|
||||
if _, _, err := lockQueue(ctx, tx, token.NamespaceID, view.QueueID); err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := readExecutionTask(ctx, tx, token.NamespaceID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
view = task.view
|
||||
if view.Revision != in.ExpectedRevision || view.State.Terminal() || view.CancelRequestedAt != nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
var owner sql.NullString
|
||||
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if owner.Valid && owner.String != "schedule" {
|
||||
return fault.Conflict
|
||||
}
|
||||
var attempt *pulse.Attempt
|
||||
if view.State == pulse.TaskLeased || view.State == pulse.TaskRunning {
|
||||
if task.current == nil {
|
||||
return errors.New("active task lacks attempt")
|
||||
}
|
||||
a, err := readAttempt(ctx, tx, token.NamespaceID, *task.current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.TaskID != id || a.Epoch != task.epoch || string(a.State) != string(view.State) {
|
||||
return errors.New("task attempt identity or state mismatch")
|
||||
}
|
||||
attempt = &a
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := view.State
|
||||
lease := task.lease
|
||||
var completed *time.Time
|
||||
if view.State == pulse.TaskWaiting || view.State == pulse.TaskLeased {
|
||||
next = pulse.TaskCancelled
|
||||
completed = &now
|
||||
lease = nil
|
||||
if view.EffectUncertain {
|
||||
next, completed = pulse.TaskNeedsReview, nil
|
||||
}
|
||||
if attempt != nil {
|
||||
if attempt.AuthorizedAt != nil {
|
||||
return errors.New("leased attempt already authorized")
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='CANCELLED',ended_at=?,error_code='CANCELLED_BEFORE_START',revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, now, now, token.NamespaceID, attempt.ID, attempt.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := releasePermits(ctx, tx, token.NamespaceID, attempt.ID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,lease_until=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=?`, next, now, lease, completed, now, token.NamespaceID, id, in.ExpectedRevision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Task, err = readTask(ctx, tx, token.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
event := resultEvent(next)
|
||||
if next == view.State {
|
||||
event = "task.cancel_requested"
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", event, map[string]any{"task_id": id, "state": next, "reason": "CANCEL_REQUESTED", "effect_uncertain": view.EffectUncertain}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": view.State, "revision": fmt.Sprint(view.Revision)}, map[string]any{"state": next, "revision": fmt.Sprint(out.Task.Revision), "cancel_requested_at": now}); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
@@ -318,80 +317,6 @@ func (s *TaskStore) ListTasks(ctx context.Context, token auth.Token, in service.
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID, key string, in service.CancelInput) (service.CancelResult, error) {
|
||||
var out service.CancelResult
|
||||
err := s.transaction(ctx, token, "tasks:cancel", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
// 先以查询权限范围核对队列,再锁幂等记录,统一所有写操作的锁定顺序。
|
||||
task, err := readTask(ctx, tx, token.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.AllowsQueue(task.QueueID) {
|
||||
return fault.NotFound
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "task.cancel", string(id), key, "task", id, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
task, err = readTask(ctx, tx, token.NamespaceID, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task.Revision != in.ExpectedRevision {
|
||||
return fault.Conflict
|
||||
}
|
||||
if task.State != pulse.TaskWaiting {
|
||||
return fault.Conflict
|
||||
}
|
||||
var owner sql.NullString
|
||||
if err := tx.QueryRowContext(ctx, "SELECT owner_kind FROM pulse_tasks WHERE namespace_id=? AND id=?", token.NamespaceID, id).Scan(&owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if owner.Valid && owner.String != "schedule" {
|
||||
return fault.Conflict
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := pulse.TaskCancelled
|
||||
var completed *time.Time = &now
|
||||
if task.EffectUncertain {
|
||||
next = pulse.TaskNeedsReview
|
||||
completed = nil
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_tasks SET state=?,cancel_requested_at=?,completed_at=?,updated_at=?,revision=revision+1 WHERE namespace_id=? AND id=? AND revision=? AND state='WAITING'`, next, now, completed, now, token.NamespaceID, id, in.ExpectedRevision)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return fault.Conflict
|
||||
}
|
||||
out.Task, err = readTask(ctx, tx, token.NamespaceID, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, id, out.Task.Revision, "task", resultEvent(out.Task.State), map[string]any{"task_id": id, "state": out.Task.State, "reason": "CANCEL_REQUESTED", "effect_uncertain": out.Task.EffectUncertain}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, identity, id, "task", "task.cancel", in.Reason, map[string]any{"state": task.State, "revision": fmt.Sprint(task.Revision)}, map[string]any{"state": out.Task.State, "revision": fmt.Sprint(out.Task.Revision)}); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *TaskStore) CreateQueue(ctx context.Context, token auth.Token, key string, in service.QueueInput) (service.QueueResult, error) {
|
||||
var out service.QueueResult
|
||||
err := s.transaction(ctx, token, "queues:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
|
||||
207
internal/storage/mysql/worker_cancel_integration_test.go
Normal file
207
internal/storage/mysql/worker_cancel_integration_test.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func TestIntegrationWorkerCancellation(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer stop()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := func(t *testing.T) *executionFixture { return newExecutionFixture(t, ctx, db) }
|
||||
cancel := func(f *executionFixture, task service.TaskView) service.CancelResult {
|
||||
t.Helper()
|
||||
out, err := f.tasks.CancelTask(ctx, f.token, task.ID, string(executionID(t)), service.CancelInput{ExpectedRevision: task.Revision, Reason: "stop controlled test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
t.Run("leased_cancel_prevents_start_and_releases_once", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
f.submit(3)
|
||||
a, ref := f.claim()
|
||||
out := cancel(f, a.Task)
|
||||
if out.Task.State != pulse.TaskCancelled || out.Task.CompletedAt == nil || f.used() != 0 {
|
||||
t.Fatal("leased cancel did not settle")
|
||||
}
|
||||
_, err := f.a.Start(ctx, f.token, f.binding, executionID(t), ref)
|
||||
expectWorkerCode(t, err, "LEASE_LOST")
|
||||
late := retryFinish(t, f, ref, "SUCCEEDED", false)
|
||||
if late.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(a.Task.ID).State != pulse.TaskCancelled || f.used() != 0 {
|
||||
t.Fatal("late result changed cancelled task")
|
||||
}
|
||||
})
|
||||
for _, outcome := range []string{"SUCCEEDED", "FAILED", "UNKNOWN", "CANCELLED"} {
|
||||
t.Run("running_"+outcome, func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
f.submit(3)
|
||||
_, ref := f.claim()
|
||||
startID := executionID(t)
|
||||
started, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := f.get(ref.TaskID)
|
||||
key := string(executionID(t))
|
||||
input := service.CancelInput{ExpectedRevision: before.Revision, Reason: "cancel running test"}
|
||||
out, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Task.State != pulse.TaskRunning || out.Task.CancelRequestedAt == nil || out.Task.CompletedAt != nil || f.used() != 1 {
|
||||
t.Fatal("intent prematurely settled execution")
|
||||
}
|
||||
replayed, err := f.a.Start(ctx, f.token, f.binding, startID, ref)
|
||||
if err != nil || !replayed.Deduplicated || replayed.Lease.CancelRequestedAt == nil || !replayed.Lease.Active {
|
||||
t.Fatal("old start ACK lost current cancellation")
|
||||
}
|
||||
renew, err := f.a.Renew(ctx, f.token, f.binding, executionID(t), ref)
|
||||
if err != nil || !renew.Lease.LeaseUntil.Equal(started.Lease.LeaseUntil) || renew.Lease.CancelRequestedAt == nil {
|
||||
t.Fatal("cancelled lease extended")
|
||||
}
|
||||
notices, err := f.a.Cancellations(ctx, f.token, f.binding)
|
||||
if err != nil || len(notices) != 1 || notices[0].AttemptID != ref.AttemptID {
|
||||
t.Fatal("missing persisted notice")
|
||||
}
|
||||
// 新会话隔离旧连接,Resume 后仍能取得取消意图并确认原尝试。
|
||||
binding, _, err := f.b.Connect(ctx, f.token, f.hello)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = f.a.Cancellations(ctx, f.token, f.binding)
|
||||
expectWorkerCode(t, err, "STALE_SESSION")
|
||||
lease, err := f.b.Resume(ctx, f.token, binding, ref)
|
||||
if err != nil || lease.CancelRequestedAt == nil || !lease.Active {
|
||||
t.Fatal("reconnect lost intent")
|
||||
}
|
||||
f.a, f.binding = f.b, binding
|
||||
finished := retryFinish(t, f, ref, outcome, outcome == "FAILED")
|
||||
want := pulse.TaskState(outcome)
|
||||
if outcome == "UNKNOWN" {
|
||||
want = pulse.TaskNeedsReview
|
||||
}
|
||||
if finished.Receipt.TaskState != want || f.used() != 0 || f.get(ref.TaskID).AttemptCount != 1 {
|
||||
t.Fatal("wrong cancellation resolution or retry")
|
||||
}
|
||||
again, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, input)
|
||||
if err != nil || !again.Deduplicated || again.Task.State != pulse.TaskRunning || again.Task.Revision != out.Task.Revision {
|
||||
t.Fatal("cancel receipt changed after completion")
|
||||
}
|
||||
if lenMust := f.count("orbit_events", "aggregate_id=? AND type='task.cancel_requested'", ref.TaskID); lenMust != 1 {
|
||||
t.Fatal("duplicate intent event")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("prior_unknown_not_erased_and_late_cancel_is_evidence", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
task := retryTask(t, f, 3, pulse.RecoveryIdempotent)
|
||||
first := retryStart(t, f)
|
||||
retryFinish(t, f, first, "UNKNOWN", false)
|
||||
pending := f.get(task.ID)
|
||||
timer := time.NewTimer(time.Until(pending.AvailableAt.Add(10 * time.Millisecond)))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
second := retryStart(t, f)
|
||||
cancel(f, f.get(task.ID))
|
||||
retryFinish(t, f, second, "CANCELLED", false)
|
||||
got := f.get(task.ID)
|
||||
if got.State != pulse.TaskNeedsReview || !got.EffectUncertain || got.CompletedAt != nil || got.AttemptCount != 2 {
|
||||
t.Fatal("current cancellation erased earlier unknown effect")
|
||||
}
|
||||
})
|
||||
t.Run("unrequested_cancel_is_rejected", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
f.submit(1)
|
||||
ref := retryStart(t, f)
|
||||
_, err := f.a.Finish(ctx, f.token, f.binding, executionID(t), worker.FinishInput{Ref: ref, Outcome: "CANCELLED", ErrorCode: "CANCEL_CONFIRMED", Result: json.RawMessage(`{}`)})
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
if f.get(ref.TaskID).State != pulse.TaskRunning || f.used() != 1 {
|
||||
t.Fatal("unrequested cancellation changed execution")
|
||||
}
|
||||
})
|
||||
t.Run("intent_rollback_and_expiry_no_retry", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
f.submit(3)
|
||||
ref := retryStart(t, f)
|
||||
before := f.get(ref.TaskID)
|
||||
key := string(executionID(t))
|
||||
in := service.CancelInput{ExpectedRevision: before.Revision, Reason: "rollback test"}
|
||||
f.exec(`CREATE TRIGGER orbit_test_cancel BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='controlled cancel failure'`)
|
||||
_, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, in)
|
||||
f.exec(`DROP TRIGGER orbit_test_cancel`)
|
||||
if err == nil || f.get(ref.TaskID).CancelRequestedAt != nil || f.get(ref.TaskID).Revision != before.Revision {
|
||||
t.Fatal("partial cancel transaction")
|
||||
}
|
||||
if _, err := f.tasks.CancelTask(ctx, f.token, ref.TaskID, key, in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.expire(ref)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.get(ref.TaskID).State != pulse.TaskNeedsReview || f.used() != 0 {
|
||||
t.Fatal("expired running cancel became false cancellation")
|
||||
}
|
||||
late := retryFinish(t, f, ref, "CANCELLED", false)
|
||||
if late.Receipt.Disposition != "EVIDENCE_ONLY" || f.get(ref.TaskID).State != pulse.TaskNeedsReview {
|
||||
t.Fatal("late cancellation overrode unknown")
|
||||
}
|
||||
})
|
||||
t.Run("start_and_cancel_race", func(t *testing.T) {
|
||||
for i := 0; i < 12; i++ {
|
||||
f := fixture(t)
|
||||
f.submit(1)
|
||||
a, ref := f.claim()
|
||||
var startErr, cancelErr error
|
||||
var out service.CancelResult
|
||||
gate := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); <-gate; _, startErr = f.a.Start(ctx, f.token, f.binding, executionID(t), ref) }()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-gate
|
||||
out, cancelErr = f.tasks.CancelTask(ctx, f.token, ref.TaskID, string(executionID(t)), service.CancelInput{ExpectedRevision: a.Task.Revision, Reason: "race"})
|
||||
}()
|
||||
close(gate)
|
||||
wg.Wait()
|
||||
if cancelErr == nil {
|
||||
if fault.Code(startErr) != "LEASE_LOST" || out.Task.State != pulse.TaskCancelled || f.used() != 0 {
|
||||
t.Fatalf("cancel winner: start=%v task=%s used=%d", startErr, out.Task.State, f.used())
|
||||
}
|
||||
} else {
|
||||
if fault.Code(cancelErr) != "STATE_CONFLICT" || startErr != nil || f.get(ref.TaskID).State != pulse.TaskRunning {
|
||||
t.Fatal("invalid race resolution")
|
||||
}
|
||||
cancel(f, f.get(ref.TaskID))
|
||||
retryFinish(t, f, ref, "CANCELLED", false)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
35
internal/storage/mysql/worker_cancellation.go
Normal file
35
internal/storage/mysql/worker_cancellation.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// Cancellations 读取当前有效会话持有的取消意图;断线后由新会话 Resume 和扫描重新交付。
|
||||
// 通知允许重复或迟到,不能代替任务结果;256 对应单 Worker 最大持久化容量。
|
||||
func (s *ExecutionStore) Cancellations(ctx context.Context, token auth.Token, b worker.Binding) ([]worker.Cancellation, error) {
|
||||
out := []worker.Cancellation{}
|
||||
err := s.transaction(ctx, token, "workers:consume", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
state, err := s.lockSession(ctx, tx, token, b, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT t.id,a.id,a.epoch,t.cancel_requested_at FROM pulse_attempts a JOIN pulse_tasks t ON t.namespace_id=a.namespace_id AND t.id=a.task_id AND t.current_attempt_id=a.id WHERE a.namespace_id=? AND a.session_id=? AND a.state='RUNNING' AND t.state='RUNNING' AND t.queue_id=? AND t.cancel_requested_at IS NOT NULL ORDER BY a.id LIMIT 256`, token.NamespaceID, b.SessionID, state.queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var c worker.Cancellation
|
||||
if err := rows.Scan(&c.TaskID, &c.AttemptID, &c.Epoch, &c.RequestedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return rows.Err()
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -181,7 +181,7 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
|
||||
if !state.expires.After(now) {
|
||||
return worker.StaleSession
|
||||
}
|
||||
if task.view.State != pulse.TaskWaiting || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
|
||||
if task.view.State != pulse.TaskWaiting || task.view.CancelRequestedAt != nil || task.view.AvailableAt.After(now) || !task.view.DeadlineAt.After(now) || task.view.AttemptCount >= task.maxAttempts || (task.view.StartDeadlineAt != nil && !task.view.StartDeadlineAt.After(now)) {
|
||||
continue
|
||||
}
|
||||
if task.epoch == ^uint64(0) {
|
||||
@@ -327,7 +327,7 @@ func leaseStatus(rows executionRows, b worker.Binding, now time.Time) worker.Lea
|
||||
active = active && deadline.After(now)
|
||||
until = minimumTime(until, deadline)
|
||||
}
|
||||
return worker.LeaseStatus{Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
|
||||
return worker.LeaseStatus{CancelRequestedAt: t.view.CancelRequestedAt, Active: active, State: a.State, LeaseUntil: until, ServerTime: now}
|
||||
}
|
||||
func attemptIdentity(ref worker.AttemptRef) any {
|
||||
return struct {
|
||||
@@ -453,6 +453,9 @@ func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token,
|
||||
task, a := rows.task, rows.attempt
|
||||
authorized := a.AuthorizedAt
|
||||
if operation == "start" {
|
||||
if task.view.CancelRequestedAt != nil {
|
||||
return worker.LeaseLost
|
||||
}
|
||||
if a.State != pulse.AttemptLeased || task.view.State != pulse.TaskLeased {
|
||||
return fault.Conflict
|
||||
}
|
||||
@@ -466,6 +469,15 @@ func (s *ExecutionStore) executionCommand(ctx context.Context, token auth.Token,
|
||||
} else if a.State != pulse.AttemptRunning || authorized == nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
if task.view.CancelRequestedAt != nil {
|
||||
// 取消后不延长执行窗口;当前权威结果仍可在原租约内提交。
|
||||
receipt := worker.Receipt{Operation: operation, TaskState: task.view.State, AttemptState: a.State, TaskRevision: task.view.Revision, LeaseUntil: status.LeaseUntil, Disposition: "APPLIED"}
|
||||
if err := saveReceipt(ctx, tx, token.NamespaceID, a.ID, message, operation, digest, receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status}
|
||||
return nil
|
||||
}
|
||||
until := minimumTime(now.Add(s.options.LeaseDuration), authorized.Add(time.Duration(task.attemptTimeout)*time.Millisecond), task.view.DeadlineAt, rows.session.expires)
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_attempts SET state='RUNNING',authorized_at=?,lease_until=?,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND revision=?`, authorized, until, now, token.NamespaceID, a.ID, a.Revision)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
|
||||
@@ -59,7 +59,7 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
|
||||
out.Examined++
|
||||
var changed pulse.TaskState
|
||||
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
// 与 Worker 命令一致先锁队列;有其他节点正在处理时跳过,本批不占用等待线程。
|
||||
// 队列 → 任务 → 尝试 → 许可;忙碌资源留给下一批。
|
||||
var queue core.ID
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
@@ -62,6 +63,9 @@ func (s *ExecutionStore) Finish(ctx context.Context, token auth.Token, b worker.
|
||||
out = worker.CommandResult{Receipt: receipt, Lease: status, Deduplicated: true}
|
||||
return nil
|
||||
}
|
||||
if in.Outcome == "CANCELLED" && task.view.CancelRequestedAt == nil {
|
||||
return fault.Conflict
|
||||
}
|
||||
reason := ""
|
||||
if a.ResultDigest != nil {
|
||||
reason = "RESULT_CONFLICT"
|
||||
|
||||
@@ -98,6 +98,15 @@ func (s *ExecutionStore) Connect(ctx context.Context, token auth.Token, in worke
|
||||
if registration != storedRegistration || epoch == ^uint64(0) {
|
||||
return fault.Conflict
|
||||
}
|
||||
// 不依赖数据库外键:创建会话前验证队列存在且属于当前命名空间。
|
||||
// 进程头在队列之前加锁,与执行命令保持一致;失败时回滚新建的进程头。
|
||||
var queue core.ID
|
||||
if err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR SHARE`, token.NamespaceID, in.QueueID).Scan(&queue); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -160,9 +169,13 @@ func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth
|
||||
if !p.AllowsQueue(state.queue) {
|
||||
return state, fault.Forbidden
|
||||
}
|
||||
var node core.ID
|
||||
var node, instance, sessionCredential core.ID
|
||||
var sessionEpoch uint64
|
||||
var raw []byte
|
||||
err = tx.QueryRowContext(ctx, `SELECT node_id,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &state.capacity, &state.sequence, &raw, &state.expires)
|
||||
err = tx.QueryRowContext(ctx, `SELECT node_id,worker_instance_id,credential_id,session_epoch,capacity,capacity_sequence,capabilities,expires_at FROM pulse_worker_sessions WHERE namespace_id=? AND id=? FOR UPDATE`, token.NamespaceID, b.SessionID).Scan(&node, &instance, &sessionCredential, &sessionEpoch, &state.capacity, &state.sequence, &raw, &state.expires)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
@@ -170,7 +183,7 @@ func (s *ExecutionStore) lockSession(ctx context.Context, tx *sql.Tx, token auth
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if node != s.node || !state.expires.After(now) {
|
||||
if node != s.node || instance != b.InstanceID || sessionCredential != token.CredentialID || sessionEpoch != b.Epoch || !state.expires.After(now) {
|
||||
return state, worker.StaleSession
|
||||
}
|
||||
if err := json.Unmarshal(raw, &state.capabilities); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user