feat: enforce durable queue resource limits and backpressure
This commit is contained in:
@@ -147,6 +147,49 @@ func awaitProcessDrop(t *testing.T, ctx context.Context, g *workerFaultGateway,
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerProcessQueueBackpressure(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
_, _, token := integrationWorkerClient(t)
|
||||
api := processClient(t, a, token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
name := "quota-crash-" + workerTestID(t)
|
||||
q, err := api.CreateQueue(ctx, name, CreateQueueRequest{Name: name, MaxConcurrency: 1, RatePerSecond: 1, MaxUnfinishedTasks: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := SubmitTaskRequest{QueueID: q.Queue.ID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
|
||||
first, err := api.SubmitTask(ctx, name+"-first", in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.kill(t)
|
||||
b := startOrbitProcess(t)
|
||||
api = processClient(t, b, token)
|
||||
if a.nodeID == b.nodeID {
|
||||
t.Fatal("node identity reused")
|
||||
}
|
||||
_, err = api.SubmitTask(ctx, name+"-second", in)
|
||||
var rejected *APIError
|
||||
if !errors.As(err, &rejected) || rejected.Code != "QUEUE_FULL" {
|
||||
t.Fatalf("restart lost quota: %v", err)
|
||||
}
|
||||
dupe, err := api.SubmitTask(ctx, name+"-first", in)
|
||||
if err != nil || !dupe.Deduplicated || dupe.Task.ID != first.Task.ID {
|
||||
t.Fatal("restart lost original receipt")
|
||||
}
|
||||
if _, err := api.CancelTask(ctx, first.Task.ID, name+"-release", CancelTaskRequest{ExpectedRevision: first.Task.Revision, Reason: "free recovered queue"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.SubmitTask(ctx, name+"-second", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := api.GetQueue(ctx, q.Queue.ID)
|
||||
if err != nil || current.UnfinishedTasks != 1 {
|
||||
t.Fatal("recovered quota counter mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// 两次在真实数据库提交后丢 ACK 并强杀节点,由另外的真实进程核对原操作。
|
||||
func TestIntegrationWorkerProcessACKRecovery(t *testing.T) {
|
||||
a := startOrbitProcess(t)
|
||||
|
||||
Reference in New Issue
Block a user