feat: add bounded task priority and fair application dispatch

internal/dispatch/pool_test.go

@@ -0,0 +1,127 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestRoundRobinApplicationsAndQueues(t *testing.T) {
+	var r roster
+	for i := 0; i < 40; i++ {
+		r.add(Job{Context: context.Background(), Application: "noisy", Queue: "many-workers"})
+	}
+	for i := 0; i < 4; i++ {
+		r.add(Job{Context: context.Background(), Application: "noisy", Queue: "other-queue"})
+		r.add(Job{Context: context.Background(), Application: "quiet", Queue: "only-worker"})
+	}
+	want := []string{"noisy/many-workers", "quiet/only-worker", "noisy/other-queue", "quiet/only-worker", "noisy/many-workers", "quiet/only-worker", "noisy/other-queue", "quiet/only-worker"}
+	for _, expected := range want {
+		j, ok := r.next()
+		if !ok || j.Application+"/"+j.Queue != expected {
+			t.Fatalf("dispatch=%s/%s want=%s", j.Application, j.Queue, expected)
+		}
+		r.finish(j.Application)
+	}
+}
+func TestPoolBoundsCancellationAndIsolation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	p := New(ctx, 2, 4)
+	started := make(chan string, 4)
+	var active, same atomic.Int32
+	block := func(app string) Job {
+		return Job{Context: ctx, Application: app, Queue: "queue", Run: func(ctx context.Context) {
+			if active.Add(1) > 2 {
+				t.Error("global dispatch concurrency exceeded")
+			}
+			defer active.Add(-1)
+			if app == "noisy" {
+				if same.Add(1) > 1 {
+					t.Error("application ran concurrent claims")
+				}
+				defer same.Add(-1)
+			}
+			started <- app
+			<-ctx.Done()
+		}}
+	}
+	for _, app := range []string{"noisy", "noisy", "noisy", "quiet"} {
+		if !p.Submit(block(app)) {
+			t.Fatal("unexpected rejection")
+		}
+	}
+	if p.Submit(block("overflow")) {
+		t.Fatal("unbounded pending jobs")
+	}
+	seen := map[string]bool{}
+	for range 2 {
+		select {
+		case app := <-started:
+			seen[app] = true
+		case <-time.After(time.Second):
+			t.Fatal("one application blocked another")
+		}
+	}
+	if !seen["quiet"] || !seen["noisy"] {
+		t.Fatal("missing isolated application")
+	}
+	cancel()
+	select {
+	case <-p.Done():
+	case <-time.After(time.Second):
+		t.Fatal("shutdown did not cancel claims")
+	}
+	if len(p.slots) != 0 || active.Load() != 0 || p.Submit(block("late")) {
+		t.Fatal("shutdown leaked slots or accepted more work")
+	}
+}
+func TestPoolConcurrentSubmitAndShutdown(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	p := New(ctx, 4, 32)
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 100; j++ {
+				p.Submit(Job{Context: ctx, Application: fmt.Sprint(i % 5), Queue: fmt.Sprint(i % 3), Run: func(ctx context.Context) {
+					select {
+					case <-ctx.Done():
+					case <-time.After(time.Millisecond):
+					}
+				}})
+			}
+		}()
+	}
+	cancel()
+	wg.Wait()
+	select {
+	case <-p.Done():
+	case <-time.After(time.Second):
+		t.Fatal("shutdown stuck")
+	}
+	if len(p.slots) != 0 {
+		t.Fatal("pending jobs leaked")
+	}
+}
+
+func TestCancelledJobsDoNotUseQueueTurnOrCapacity(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	var r roster
+	released := 0
+	r.add(Job{Context: ctx, Application: "stale", Queue: "gone", release: func() { released++ }})
+	r.add(Job{Context: context.Background(), Application: "ready", Queue: "ready"})
+	j, ok := r.next()
+	if !ok || j.Application != "ready" || released != 1 {
+		t.Fatal("cancelled job blocked live work")
+	}
+	r.finish("ready")
+	if len(r.apps) != 0 {
+		t.Fatal("empty applications retained")
+	}
+}