feat: enforce durable queue resource limits and backpressure

This commit is contained in:
2026-09-17 17:25:56 +08:00
parent 3ce7d2c104
commit 5c8825e4bc
38 changed files with 1231 additions and 30 deletions

View File

@@ -147,6 +147,49 @@ func awaitProcessDrop(t *testing.T, ctx context.Context, g *workerFaultGateway,
}
}
func TestIntegrationWorkerProcessQueueBackpressure(t *testing.T) {
a := startOrbitProcess(t)
_, _, token := integrationWorkerClient(t)
api := processClient(t, a, token)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
name := "quota-crash-" + workerTestID(t)
q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 1, RatePerSecond: 1, MaxUnfinishedTasks: 1})
if err != nil {
t.Fatal(err)
}
in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
first, err := api.SubmitTask(ctx, name+"-first", in)
if err != nil {
t.Fatal(err)
}
a.kill(t)
b := startOrbitProcess(t)
api = processClient(t, b, token)
if a.nodeID == b.nodeID {
t.Fatal("node identity reused")
}
_, err = api.SubmitTask(ctx, name+"-second", in)
var rejected *APIError
if !errors.As(err, &rejected) || rejected.Code != "QUEUE_FULL" {
t.Fatalf("restart lost quota: %v", err)
}
dupe, err := api.SubmitTask(ctx, name+"-first", in)
if err != nil || !dupe.Deduplicated || dupe.Task.ID != first.Task.ID {
t.Fatal("restart lost original receipt")
}
if _, err := api.CancelTask(ctx, first.Task.ID, name+"-release", CancelTaskRequest{ExpectedRevision: first.Task.Revision, Reason: "free recovered queue"}); err != nil {
t.Fatal(err)
}
if _, err := api.SubmitTask(ctx, name+"-second", in); err != nil {
t.Fatal(err)
}
current, err := api.GetQueue(ctx, q.Queue.ID)
if err != nil || current.UnfinishedTasks != 1 {
t.Fatal("recovered quota counter mismatch")
}
}
// 两次在真实数据库提交后丢 ACK 并强杀节点,由另外的真实进程核对原操作。
func TestIntegrationWorkerProcessACKRecovery(t *testing.T) {
a := startOrbitProcess(t)