feat: enforce shared application resource quotas across namespaces
This commit is contained in:
188
sdk/go/application_resources_integration_test.go
Normal file
188
sdk/go/application_resources_integration_test.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func applicationIntegrationClient(t *testing.T, key string) (*Client, string) {
|
||||
t.Helper()
|
||||
base := os.Getenv("ORBIT_TEST_WORKER_URL")
|
||||
path := os.Getenv(key)
|
||||
if base == "" || path == "" {
|
||||
t.Skip("requires disposable application resource fixtures")
|
||||
}
|
||||
if !strings.HasPrefix(base, "http://127.0.0.1:") {
|
||||
t.Fatal("requires isolated loopback server")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &v) != nil || v.Token == "" || !validID(v.QueueID) {
|
||||
t.Fatal("invalid private credentials")
|
||||
}
|
||||
api, err := NewClient(base, Options{Token: v.Token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return api, v.QueueID
|
||||
}
|
||||
func TestIntegrationApplicationResourcesSDK(t *testing.T) {
|
||||
a, qa := applicationIntegrationClient(t, "ORBIT_TEST_APP_CREDENTIALS_FILE")
|
||||
b, qb := applicationIntegrationClient(t, "ORBIT_TEST_APP_SECOND_CREDENTIALS_FILE")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
before, err := a.GetApplicationResources(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peer, err := b.GetApplicationResources(ctx)
|
||||
if err != nil || peer.ApplicationID != before.ApplicationID || qa == qb {
|
||||
t.Fatal("invalid cross namespace fixture", err)
|
||||
}
|
||||
ordinary, _, _ := integrationWorkerClient(t)
|
||||
if _, err := ordinary.GetApplicationResources(ctx); err == nil {
|
||||
t.Fatal("queue admin inherited application permission")
|
||||
} else {
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.StatusCode != 403 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
key := workerTestID(t)
|
||||
in := ConfigureApplicationResourcesRequest{ExpectedRevision: before.Revision, Reason: "SDK application quota regression", MaxConcurrency: 2, RatePerSecond: 10, MaxUnfinishedTasks: 1}
|
||||
configured, err := a.ConfigureApplicationResources(ctx, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replay, err := a.ConfigureApplicationResources(ctx, key, in)
|
||||
if err != nil || !replay.Deduplicated || replay.Resources.Revision != configured.Resources.Revision {
|
||||
t.Fatal("configure replay failed", err)
|
||||
}
|
||||
if _, err := b.ConfigureApplicationResources(ctx, key, in); err == nil {
|
||||
t.Fatal("same key in different namespace replayed authority")
|
||||
} else {
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.Code != "STATE_CONFLICT" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var ignored ConfigureApplicationResourcesResult
|
||||
for _, body := range []any{
|
||||
map[string]any{"expected_revision": "2", "reason": "missing limits"},
|
||||
map[string]any{"expected_revision": 2, "reason": "bad number", "max_unfinished_tasks": 1, "max_concurrency": 1, "rate_per_second": 1},
|
||||
map[string]any{"expected_revision": "2", "reason": "override application", "max_unfinished_tasks": 1, "max_concurrency": 1, "rate_per_second": 1, "application_id": before.ApplicationID},
|
||||
} {
|
||||
if err := a.call(ctx, "POST", "/v1/application/resources:configure", workerTestID(t), body, &ignored); err == nil {
|
||||
t.Fatal("invalid configuration accepted")
|
||||
} else {
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.StatusCode != 400 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
input := SubmitTaskRequest{QueueID: qa, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
first, err := a.SubmitTask(ctx, key+"-one", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.QueueID = qb
|
||||
_, err = b.SubmitTask(ctx, key+"-two", input)
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.Code != "APPLICATION_FULL" || ae.StatusCode != 429 || ae.RetryAfter != time.Second {
|
||||
t.Fatal("missing application backpressure", err)
|
||||
}
|
||||
peer, err = b.GetApplicationResources(ctx)
|
||||
if err != nil || peer.UnfinishedTasks != 1 || peer.Revision != configured.Resources.Revision {
|
||||
t.Fatal(peer, err)
|
||||
}
|
||||
if _, err := a.CancelTask(ctx, first.Task.ID, key+"-cancel", CancelTaskRequest{ExpectedRevision: 1, Reason: "free shared slot"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := b.SubmitTask(ctx, key+"-two", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.CancelTask(ctx, second.Task.ID, key+"-cancel-two", CancelTaskRequest{ExpectedRevision: 1, Reason: "finish regression"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := a.GetApplicationResources(ctx)
|
||||
if err != nil || after.UnfinishedTasks != 0 || after.ActiveAttempts != 0 || after.PendingStarts != 0 {
|
||||
t.Fatal(after, err)
|
||||
}
|
||||
if _, err := a.ConfigureApplicationResources(ctx, key+"-restore", ConfigureApplicationResourcesRequest{ExpectedRevision: configured.Resources.Revision, Reason: "restore sample defaults", MaxUnfinishedTasks: before.MaxUnfinishedTasks, MaxConcurrency: before.MaxConcurrency, RatePerSecond: before.RatePerSecond}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerProcessApplicationBackpressure(t *testing.T) {
|
||||
p := startOrbitProcess(t)
|
||||
ca, qa := applicationIntegrationClient(t, "ORBIT_TEST_APP_CREDENTIALS_FILE")
|
||||
cb, qb := applicationIntegrationClient(t, "ORBIT_TEST_APP_SECOND_CREDENTIALS_FILE")
|
||||
a := processClient(t, p, ca.token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
before, err := a.GetApplicationResources(ctx)
|
||||
if err != nil || before.UnfinishedTasks != 0 {
|
||||
t.Fatal("dedicated app must be idle", err)
|
||||
}
|
||||
key := workerTestID(t)
|
||||
changed, err := a.ConfigureApplicationResources(ctx, key, ConfigureApplicationResourcesRequest{ExpectedRevision: before.Revision, Reason: "application crash regression", MaxUnfinishedTasks: 1, MaxConcurrency: 1, RatePerSecond: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := SubmitTaskRequest{QueueID: qa, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
|
||||
first, err := a.SubmitTask(ctx, key+"-first", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.kill(t)
|
||||
next := startOrbitProcess(t)
|
||||
if next.nodeID == p.nodeID {
|
||||
t.Fatal("node identity reused")
|
||||
}
|
||||
a = processClient(t, next, ca.token)
|
||||
b := processClient(t, next, cb.token)
|
||||
v, err := b.GetApplicationResources(ctx)
|
||||
if err != nil || v.Revision != changed.Resources.Revision || v.UnfinishedTasks != 1 || v.MaxUnfinishedTasks != 1 {
|
||||
t.Fatal("restart lost shared configuration or usage", v, err)
|
||||
}
|
||||
if out, err := a.SubmitTask(ctx, key+"-first", input); err != nil || !out.Deduplicated || out.Task.ID != first.Task.ID {
|
||||
t.Fatal("restart lost admission receipt", err)
|
||||
}
|
||||
input.QueueID = qb
|
||||
_, err = b.SubmitTask(ctx, key+"-second", input)
|
||||
var rejected *APIError
|
||||
if !errors.As(err, &rejected) || rejected.Code != "APPLICATION_FULL" {
|
||||
t.Fatal("restart allowed another namespace to exceed quota", err)
|
||||
}
|
||||
if _, err := a.CancelTask(ctx, first.Task.ID, key+"-cancel-first", CancelTaskRequest{ExpectedRevision: 1, Reason: "release after restart"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := b.SubmitTask(ctx, key+"-second", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.CancelTask(ctx, second.Task.ID, key+"-cancel-second", CancelTaskRequest{ExpectedRevision: 1, Reason: "finish process regression"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err = a.GetApplicationResources(ctx)
|
||||
if err != nil || v.UnfinishedTasks != 0 || v.ActiveAttempts != 0 || v.PendingStarts != 0 {
|
||||
t.Fatal("release after restart leaked usage", err)
|
||||
}
|
||||
if _, err := a.ConfigureApplicationResources(ctx, key+"-restore", ConfigureApplicationResourcesRequest{ExpectedRevision: changed.Resources.Revision, Reason: "restore defaults", MaxUnfinishedTasks: before.MaxUnfinishedTasks, MaxConcurrency: before.MaxConcurrency, RatePerSecond: before.RatePerSecond}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("server killed and exited; new node preserved shared quota and receipt; second namespace blocked until original task cancelled")
|
||||
}
|
||||
Reference in New Issue
Block a user