feat: enforce shared application resource quotas across namespaces
This commit is contained in:
55
sdk/go/application_resources.go
Normal file
55
sdk/go/application_resources.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package orbit
|
||||
|
||||
import "context"
|
||||
|
||||
// ApplicationResources 是当前应用下所有命名空间共享的配置与用量快照。
|
||||
// 配置变更需要应用级权限；修改计数不会增加 Revision。
|
||||
type ApplicationResources struct {
|
||||
// ApplicationID 所属应用标识，由凭据归属确定。
|
||||
ApplicationID string `json:"application_id"`
|
||||
// Revision 仅配置变更增加的版本；JSON 使用十进制字符串。
|
||||
Revision uint64 `json:"revision,string"`
|
||||
// MaxUnfinishedTasks 跨命名空间的未终结任务上限，包含待核查任务。
|
||||
MaxUnfinishedTasks uint32 `json:"max_unfinished_tasks"`
|
||||
// UnfinishedTasks 当前未终结任务总数；自动重试不重复占用。
|
||||
UnfinishedTasks uint64 `json:"unfinished_tasks,string"`
|
||||
// MaxConcurrency 跨队列的预占及运行尝试总上限。
|
||||
MaxConcurrency uint32 `json:"max_concurrency"`
|
||||
// ActiveAttempts 当前预占与运行的尝试数。
|
||||
ActiveAttempts uint64 `json:"active_attempts,string"`
|
||||
// PendingStarts 已预占但尚未开始的尝试数，属于 ActiveAttempts。
|
||||
PendingStarts uint64 `json:"pending_starts,string"`
|
||||
// RatePerSecond 每秒固定窗口允许的开始授权次数。
|
||||
RatePerSecond uint32 `json:"rate_per_second"`
|
||||
}
|
||||
|
||||
// ConfigureApplicationResourcesRequest 完整替换上限，重复请求必须保留原版本与幂等键。
|
||||
type ConfigureApplicationResourcesRequest struct {
|
||||
// ExpectedRevision 管理命令预期配置版本，须保留原值重报。
|
||||
ExpectedRevision uint64 `json:"expected_revision,string"`
|
||||
// Reason 非空审计原因，最多 1024 个 Unicode 字符。
|
||||
Reason string `json:"reason"`
|
||||
// MaxUnfinishedTasks 跨命名空间的未终结任务上限，包含待核查任务。
|
||||
MaxUnfinishedTasks uint32 `json:"max_unfinished_tasks"`
|
||||
// MaxConcurrency 跨队列的预占及运行尝试总上限。
|
||||
MaxConcurrency uint32 `json:"max_concurrency"`
|
||||
// RatePerSecond 每秒固定窗口允许的开始授权次数。
|
||||
RatePerSecond uint32 `json:"rate_per_second"`
|
||||
}
|
||||
type ConfigureApplicationResourcesResult struct {
|
||||
// Resources 配置成功时的原用量快照，重报不刷新。
|
||||
Resources ApplicationResources `json:"resources"`
|
||||
// Deduplicated 是否返回先前已提交的同身份回执。
|
||||
Deduplicated bool `json:"deduplicated"`
|
||||
}
|
||||
|
||||
func (c *Client) GetApplicationResources(ctx context.Context) (ApplicationResources, error) {
|
||||
var out ApplicationResources
|
||||
err := c.call(ctx, "GET", "/v1/application/resources", "", nil, &out)
|
||||
return out, err
|
||||
}
|
||||
func (c *Client) ConfigureApplicationResources(ctx context.Context, key string, in ConfigureApplicationResourcesRequest) (ConfigureApplicationResourcesResult, error) {
|
||||
var out ConfigureApplicationResourcesResult
|
||||
err := c.call(ctx, "POST", "/v1/application/resources:configure", key, in, &out)
|
||||
return out, err
|
||||
}
|
||||
188
sdk/go/application_resources_integration_test.go
Normal file
188
sdk/go/application_resources_integration_test.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func applicationIntegrationClient(t *testing.T, key string) (*Client, string) {
|
||||
t.Helper()
|
||||
base := os.Getenv("ORBIT_TEST_WORKER_URL")
|
||||
path := os.Getenv(key)
|
||||
if base == "" || path == "" {
|
||||
t.Skip("requires disposable application resource fixtures")
|
||||
}
|
||||
if !strings.HasPrefix(base, "http://127.0.0.1:") {
|
||||
t.Fatal("requires isolated loopback server")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &v) != nil || v.Token == "" || !validID(v.QueueID) {
|
||||
t.Fatal("invalid private credentials")
|
||||
}
|
||||
api, err := NewClient(base, Options{Token: v.Token})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return api, v.QueueID
|
||||
}
|
||||
func TestIntegrationApplicationResourcesSDK(t *testing.T) {
|
||||
a, qa := applicationIntegrationClient(t, "ORBIT_TEST_APP_CREDENTIALS_FILE")
|
||||
b, qb := applicationIntegrationClient(t, "ORBIT_TEST_APP_SECOND_CREDENTIALS_FILE")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
before, err := a.GetApplicationResources(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peer, err := b.GetApplicationResources(ctx)
|
||||
if err != nil || peer.ApplicationID != before.ApplicationID || qa == qb {
|
||||
t.Fatal("invalid cross namespace fixture", err)
|
||||
}
|
||||
ordinary, _, _ := integrationWorkerClient(t)
|
||||
if _, err := ordinary.GetApplicationResources(ctx); err == nil {
|
||||
t.Fatal("queue admin inherited application permission")
|
||||
} else {
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.StatusCode != 403 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
key := workerTestID(t)
|
||||
in := ConfigureApplicationResourcesRequest{ExpectedRevision: before.Revision, Reason: "SDK application quota regression", MaxConcurrency: 2, RatePerSecond: 10, MaxUnfinishedTasks: 1}
|
||||
configured, err := a.ConfigureApplicationResources(ctx, key, in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replay, err := a.ConfigureApplicationResources(ctx, key, in)
|
||||
if err != nil || !replay.Deduplicated || replay.Resources.Revision != configured.Resources.Revision {
|
||||
t.Fatal("configure replay failed", err)
|
||||
}
|
||||
if _, err := b.ConfigureApplicationResources(ctx, key, in); err == nil {
|
||||
t.Fatal("same key in different namespace replayed authority")
|
||||
} else {
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.Code != "STATE_CONFLICT" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var ignored ConfigureApplicationResourcesResult
|
||||
for _, body := range []any{
|
||||
map[string]any{"expected_revision": "2", "reason": "missing limits"},
|
||||
map[string]any{"expected_revision": 2, "reason": "bad number", "max_unfinished_tasks": 1, "max_concurrency": 1, "rate_per_second": 1},
|
||||
map[string]any{"expected_revision": "2", "reason": "override application", "max_unfinished_tasks": 1, "max_concurrency": 1, "rate_per_second": 1, "application_id": before.ApplicationID},
|
||||
} {
|
||||
if err := a.call(ctx, "POST", "/v1/application/resources:configure", workerTestID(t), body, &ignored); err == nil {
|
||||
t.Fatal("invalid configuration accepted")
|
||||
} else {
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.StatusCode != 400 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
input := SubmitTaskRequest{QueueID: qa, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"n":9007199254740993}`)}
|
||||
first, err := a.SubmitTask(ctx, key+"-one", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.QueueID = qb
|
||||
_, err = b.SubmitTask(ctx, key+"-two", input)
|
||||
var ae *APIError
|
||||
if !errors.As(err, &ae) || ae.Code != "APPLICATION_FULL" || ae.StatusCode != 429 || ae.RetryAfter != time.Second {
|
||||
t.Fatal("missing application backpressure", err)
|
||||
}
|
||||
peer, err = b.GetApplicationResources(ctx)
|
||||
if err != nil || peer.UnfinishedTasks != 1 || peer.Revision != configured.Resources.Revision {
|
||||
t.Fatal(peer, err)
|
||||
}
|
||||
if _, err := a.CancelTask(ctx, first.Task.ID, key+"-cancel", CancelTaskRequest{ExpectedRevision: 1, Reason: "free shared slot"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := b.SubmitTask(ctx, key+"-two", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.CancelTask(ctx, second.Task.ID, key+"-cancel-two", CancelTaskRequest{ExpectedRevision: 1, Reason: "finish regression"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := a.GetApplicationResources(ctx)
|
||||
if err != nil || after.UnfinishedTasks != 0 || after.ActiveAttempts != 0 || after.PendingStarts != 0 {
|
||||
t.Fatal(after, err)
|
||||
}
|
||||
if _, err := a.ConfigureApplicationResources(ctx, key+"-restore", ConfigureApplicationResourcesRequest{ExpectedRevision: configured.Resources.Revision, Reason: "restore sample defaults", MaxUnfinishedTasks: before.MaxUnfinishedTasks, MaxConcurrency: before.MaxConcurrency, RatePerSecond: before.RatePerSecond}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerProcessApplicationBackpressure(t *testing.T) {
|
||||
p := startOrbitProcess(t)
|
||||
ca, qa := applicationIntegrationClient(t, "ORBIT_TEST_APP_CREDENTIALS_FILE")
|
||||
cb, qb := applicationIntegrationClient(t, "ORBIT_TEST_APP_SECOND_CREDENTIALS_FILE")
|
||||
a := processClient(t, p, ca.token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
before, err := a.GetApplicationResources(ctx)
|
||||
if err != nil || before.UnfinishedTasks != 0 {
|
||||
t.Fatal("dedicated app must be idle", err)
|
||||
}
|
||||
key := workerTestID(t)
|
||||
changed, err := a.ConfigureApplicationResources(ctx, key, ConfigureApplicationResourcesRequest{ExpectedRevision: before.Revision, Reason: "application crash regression", MaxUnfinishedTasks: 1, MaxConcurrency: 1, RatePerSecond: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := SubmitTaskRequest{QueueID: qa, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
|
||||
first, err := a.SubmitTask(ctx, key+"-first", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.kill(t)
|
||||
next := startOrbitProcess(t)
|
||||
if next.nodeID == p.nodeID {
|
||||
t.Fatal("node identity reused")
|
||||
}
|
||||
a = processClient(t, next, ca.token)
|
||||
b := processClient(t, next, cb.token)
|
||||
v, err := b.GetApplicationResources(ctx)
|
||||
if err != nil || v.Revision != changed.Resources.Revision || v.UnfinishedTasks != 1 || v.MaxUnfinishedTasks != 1 {
|
||||
t.Fatal("restart lost shared configuration or usage", v, err)
|
||||
}
|
||||
if out, err := a.SubmitTask(ctx, key+"-first", input); err != nil || !out.Deduplicated || out.Task.ID != first.Task.ID {
|
||||
t.Fatal("restart lost admission receipt", err)
|
||||
}
|
||||
input.QueueID = qb
|
||||
_, err = b.SubmitTask(ctx, key+"-second", input)
|
||||
var rejected *APIError
|
||||
if !errors.As(err, &rejected) || rejected.Code != "APPLICATION_FULL" {
|
||||
t.Fatal("restart allowed another namespace to exceed quota", err)
|
||||
}
|
||||
if _, err := a.CancelTask(ctx, first.Task.ID, key+"-cancel-first", CancelTaskRequest{ExpectedRevision: 1, Reason: "release after restart"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := b.SubmitTask(ctx, key+"-second", input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.CancelTask(ctx, second.Task.ID, key+"-cancel-second", CancelTaskRequest{ExpectedRevision: 1, Reason: "finish process regression"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err = a.GetApplicationResources(ctx)
|
||||
if err != nil || v.UnfinishedTasks != 0 || v.ActiveAttempts != 0 || v.PendingStarts != 0 {
|
||||
t.Fatal("release after restart leaked usage", err)
|
||||
}
|
||||
if _, err := a.ConfigureApplicationResources(ctx, key+"-restore", ConfigureApplicationResourcesRequest{ExpectedRevision: changed.Resources.Revision, Reason: "restore defaults", MaxUnfinishedTasks: before.MaxUnfinishedTasks, MaxConcurrency: before.MaxConcurrency, RatePerSecond: before.RatePerSecond}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("server killed and exited; new node preserved shared quota and receipt; second namespace blocked until original task cancelled")
|
||||
}
|
||||
101
sdk/go/examples/application-resources/main.go
Normal file
101
sdk/go/examples/application-resources/main.go
Normal file
@@ -0,0 +1,101 @@
|
||||
// application-resources 演示同一应用的两个命名空间共享积压额度。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func client(path string) (*orbit.Client, string, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var v struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
c, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: v.Token})
|
||||
return c, v.QueueID, err
|
||||
}
|
||||
func run() (err error) {
|
||||
a, qa, err := client(os.Getenv("ORBIT_CREDENTIALS_FILE"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, qb, err := client(os.Getenv("ORBIT_SECOND_CREDENTIALS_FILE"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
before, err := a.GetApplicationResources(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
peer, err := b.GetApplicationResources(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if before.ApplicationID != peer.ApplicationID || qa == qb || before.UnfinishedTasks != 0 {
|
||||
return errors.New("use two namespaces in the same idle, dedicated example application")
|
||||
}
|
||||
op, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
config, err := a.ConfigureApplicationResources(ctx, "config-"+op, orbit.ConfigureApplicationResourcesRequest{ExpectedRevision: before.Revision, Reason: "cross namespace resource example", MaxUnfinishedTasks: 1, MaxConcurrency: 2, RatePerSecond: 10})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), 8*time.Second)
|
||||
defer stop()
|
||||
_, restoreErr := a.ConfigureApplicationResources(cleanup, "restore-"+op, orbit.ConfigureApplicationResourcesRequest{ExpectedRevision: config.Resources.Revision, Reason: "restore example configuration", MaxUnfinishedTasks: before.MaxUnfinishedTasks, MaxConcurrency: before.MaxConcurrency, RatePerSecond: before.RatePerSecond})
|
||||
err = errors.Join(err, restoreErr)
|
||||
}()
|
||||
input := orbit.SubmitTaskRequest{QueueID: qa, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"object_ref":"objects/example"}`), TaskTimeoutMS: 20000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}}
|
||||
first, err := a.SubmitTask(ctx, "first-"+op, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
input.QueueID = qb
|
||||
_, err = b.SubmitTask(ctx, "second-"+op, input)
|
||||
var full *orbit.APIError
|
||||
if !errors.As(err, &full) || full.Code != "APPLICATION_FULL" {
|
||||
return fmt.Errorf("expected APPLICATION_FULL, got %v", err)
|
||||
}
|
||||
fmt.Printf("cross_namespace_backpressure=%s http=%d\n", full.Code, full.StatusCode)
|
||||
if _, err := a.CancelTask(ctx, first.Task.ID, "cancel-first-"+op, orbit.CancelTaskRequest{ExpectedRevision: 1, Reason: "free shared slot"}); err != nil {
|
||||
return err
|
||||
}
|
||||
second, err := b.SubmitTask(ctx, "second-"+op, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("same_identity_retry_accepted=%t\n", second.Task.State == orbit.TaskWaiting)
|
||||
if _, err := b.CancelTask(ctx, second.Task.ID, "cancel-second-"+op, orbit.CancelTaskRequest{ExpectedRevision: 1, Reason: "complete example"}); err != nil {
|
||||
return err
|
||||
}
|
||||
after, err := a.GetApplicationResources(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("unfinished=%d active=%d pending_starts=%d\n", after.UnfinishedTasks, after.ActiveAttempts, after.PendingStarts)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user