orbit/internal/storage/mysql/priority_integration_test.go

package mysql

import (
	"context"
	"encoding/json"
	"os"
	"testing"
	"time"

	pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
	"git.yuntuo.hk/k2safe/orbit/pulse/service"
)

func TestIntegrationDispatchPriority(t *testing.T) {
	dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
	if dsn == "" {
		t.Skip("requires disposable MySQL")
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	db, err := Open(ctx, dsn, 16, 4)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if err = Migrate(ctx, db); err != nil {
		t.Fatal(err)
	}
	submit := func(f *executionFixture, p int16, max uint32) service.TaskView {
		t.Helper()
		out, err := f.tasks.Submit(ctx, f.token, string(executionID(t)), service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), Priority: p, Retry: pulse.RetryPolicy{MaxAttempts: max}})
		if err != nil {
			t.Fatal(err)
		}
		return out.Task
	}
	t.Run("priority_overtakes_recent_work_but_not_future_release", func(t *testing.T) {
		f := newExecutionFixture(t, ctx, db)
		low := submit(f, 0, 1)
		high := submit(f, 9, 1)
		later := submit(f, 9, 1)
		f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 5 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, low.ID)
		f.exec(`UPDATE pulse_tasks SET available_at=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, later.ID)
		a, _ := f.claim()
		if a.Task.ID != high.ID || a.Task.Priority != 9 {
			t.Fatal("priority not honored")
		}
		a, _ = f.claim()
		if a.Task.ID != low.ID {
			t.Fatal("priority bypassed actual availability")
		}
		noKeyClaim(t, f)
		page, err := f.tasks.ListTasks(ctx, f.token, service.ListInput{QueueID: f.resources.QueueID, Limit: 20})
		if err != nil {
			t.Fatal(err)
		}
		found := false
		for _, v := range page.Items {
			if v.ID == high.ID {
				found = v.Priority == 9
			}
		}
		if !found {
			t.Fatal("summary lost frozen priority")
		}
	})
	t.Run("old_low_priority_cannot_be_overtaken_by_unbounded_new_high_work", func(t *testing.T) {
		f := newExecutionFixture(t, ctx, db)
		old := submit(f, 0, 1)
		f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 271 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, old.ID)
		for range 24 {
			submit(f, 9, 1)
		}
		a, _ := f.claim()
		if a.Task.ID != old.ID {
			t.Fatal("aged task starved behind fresh high priority prefix")
		}
	})
	t.Run("retry_keeps_priority_and_recomputes_sort_time", func(t *testing.T) {
		f := newExecutionFixture(t, ctx, db)
		task := submit(f, 9, 2)
		first := retryStart(t, f)
		retryFinish(t, f, first, "FAILED", true)
		var gap int
		var priority int16
		if err := db.QueryRowContext(ctx, `SELECT TIMESTAMPDIFF(SECOND,dispatch_at,available_at),priority FROM pulse_tasks WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID).Scan(&gap, &priority); err != nil {
			t.Fatal(err)
		}
		if gap != 270 || priority != 9 {
			t.Fatal("retry lost priority or sort derivation")
		}
		noKeyClaim(t, f)
		f.exec(`UPDATE pulse_tasks SET available_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, task.ID)
		second := retryStart(t, f)
		retryFinish(t, f, second, "SUCCEEDED", false)
		if f.get(task.ID).Priority != 9 {
			t.Fatal("completed task lost priority")
		}
	})
	t.Run("identity_validation_and_database_constraint", func(t *testing.T) {
		f := newExecutionFixture(t, ctx, db)
		in := service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`), Priority: 0}
		first, err := f.tasks.Submit(ctx, f.token, "same", in)
		if err != nil {
			t.Fatal(err)
		}
		dupe, err := f.tasks.Submit(ctx, f.token, "same", in)
		if err != nil || !dupe.Deduplicated || dupe.Task.ID != first.Task.ID {
			t.Fatal("default priority identity changed")
		}
		in.Priority = 9
		_, err = f.tasks.Submit(ctx, f.token, "same", in)
		expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
		for _, p := range []int16{-1, 10} {
			in.Priority = p
			_, err = f.tasks.Submit(ctx, f.token, string(executionID(t)), in)
			expectWorkerCode(t, err, "INVALID_ARGUMENT")
		}
		if _, err := db.ExecContext(ctx, `UPDATE pulse_tasks SET priority=10 WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, first.Task.ID); err == nil {
			t.Fatal("database priority range unchecked")
		}
		if _, err := db.ExecContext(ctx, `UPDATE pulse_tasks SET dispatch_at=UTC_TIMESTAMP(6) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, first.Task.ID); err == nil {
			t.Fatal("generated dispatch order independently writable")
		}
	})
}