feat: add bounded task priority and fair application dispatch

internal/storage/mysql/priority_integration_test.go

@@ -0,0 +1,126 @@
+package mysql
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
+	"git.yuntuo.hk/k2safe/orbit/pulse/service"
+)
+
+func TestIntegrationDispatchPriority(t *testing.T) {
+	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
+	if dsn == "" {
+		t.Skip("requires disposable MySQL")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	db, err := Open(ctx, dsn, 16, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if err = Migrate(ctx, db); err != nil {
+		t.Fatal(err)
+	}
+	submit := func(f *executionFixture, p int16, max uint32) service.TaskView {
+		t.Helper()
+		out, err := f.tasks.Submit(ctx, f.token, string(executionID(t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), Priority: p, Retry: pulse.RetryPolicy{MaxAttempts: max}})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return out.Task
+	}
+	t.Run("priority_overtakes_recent_work_but_not_future_release", func(t *testing.T) {
+		f := newExecutionFixture(t, ctx, db)
+		low := submit(f, 0, 1)
+		high := submit(f, 9, 1)
+		later := submit(f, 9, 1)
+		f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 5 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, low.ID)
+		f.exec(`UPDATE pulse_tasks SET available_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, later.ID)
+		a, _ := f.claim()
+		if a.Task.ID != high.ID || a.Task.Priority != 9 {
+			t.Fatal("priority not honored")
+		}
+		a, _ = f.claim()
+		if a.Task.ID != low.ID {
+			t.Fatal("priority bypassed actual availability")
+		}
+		noKeyClaim(t, f)
+		page, err := f.tasks.ListTasks(ctx, f.token, service.ListInput{QueueID: f.resources.QueueID, Limit: 20})
+		if err != nil {
+			t.Fatal(err)
+		}
+		found := false
+		for _, v := range page.Items {
+			if v.ID == high.ID {
+				found = v.Priority == 9
+			}
+		}
+		if !found {
+			t.Fatal("summary lost frozen priority")
+		}
+	})
+	t.Run("old_low_priority_cannot_be_overtaken_by_unbounded_new_high_work", func(t *testing.T) {
+		f := newExecutionFixture(t, ctx, db)
+		old := submit(f, 0, 1)
+		f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 271 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, old.ID)
+		for range 24 {
+			submit(f, 9, 1)
+		}
+		a, _ := f.claim()
+		if a.Task.ID != old.ID {
+			t.Fatal("aged task starved behind fresh high priority prefix")
+		}
+	})
+	t.Run("retry_keeps_priority_and_recomputes_sort_time", func(t *testing.T) {
+		f := newExecutionFixture(t, ctx, db)
+		task := submit(f, 9, 2)
+		first := retryStart(t, f)
+		retryFinish(t, f, first, "FAILED", true)
+		var gap int
+		var priority int16
+		if err := db.QueryRowContext(ctx, `SELECT TIMESTAMPDIFF(SECOND,dispatch_at,available_at),priority FROM pulse_tasks WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID).Scan(&gap, &priority); err != nil {
+			t.Fatal(err)
+		}
+		if gap != 270 || priority != 9 {
+			t.Fatal("retry lost priority or sort derivation")
+		}
+		noKeyClaim(t, f)
+		f.exec(`UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
+		second := retryStart(t, f)
+		retryFinish(t, f, second, "SUCCEEDED", false)
+		if f.get(task.ID).Priority != 9 {
+			t.Fatal("completed task lost priority")
+		}
+	})
+	t.Run("identity_validation_and_database_constraint", func(t *testing.T) {
+		f := newExecutionFixture(t, ctx, db)
+		in := service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), Priority: 0}
+		first, err := f.tasks.Submit(ctx, f.token, "same", in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dupe, err := f.tasks.Submit(ctx, f.token, "same", in)
+		if err != nil || !dupe.Deduplicated || dupe.Task.ID != first.Task.ID {
+			t.Fatal("default priority identity changed")
+		}
+		in.Priority = 9
+		_, err = f.tasks.Submit(ctx, f.token, "same", in)
+		expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
+		for _, p := range []int16{-1, 10} {
+			in.Priority = p
+			_, err = f.tasks.Submit(ctx, f.token, string(executionID(t)), in)
+			expectWorkerCode(t, err, "INVALID_ARGUMENT")
+		}
+		if _, err := db.ExecContext(ctx, `UPDATE pulse_tasks SET priority=10 WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, first.Task.ID); err == nil {
+			t.Fatal("database priority range unchecked")
+		}
+		if _, err := db.ExecContext(ctx, `UPDATE pulse_tasks SET dispatch_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, first.Task.ID); err == nil {
+			t.Fatal("generated dispatch order independently writable")
+		}
+	})
+}