feat: enforce shared application resource quotas across namespaces

This commit is contained in:
2026-09-17 17:57:54 +08:00
parent 5c8825e4bc
commit 993786476e
43 changed files with 1606 additions and 49 deletions

View File

@@ -26,6 +26,7 @@ var (
IdempotencyConflict = New("IDEMPOTENCY_CONFLICT", "idempotency key was used with different content")
Unavailable = New("UNAVAILABLE", "service temporarily unavailable; retry using the original operation identity")
QueueFull = New("QUEUE_FULL", "queue unfinished task limit reached; retry later using the original identity")
ApplicationFull = New("APPLICATION_FULL", "application unfinished task limit reached; retry later using the original identity")
)
func Invalid(message string) error { return New("INVALID_ARGUMENT", message) }

View File

@@ -0,0 +1,142 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"time"
"git.yuntuo.hk/k2safe/orbit/internal/fault"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/platform/resources"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
// 归属不允许在线修改;认证已经锁定身份。恢复也需释放被停用应用的额度。
const applicationForNamespace = `(SELECT application_id FROM orbit_namespaces WHERE id=?)`
// 统一顺序:认证 → Worker 会话(如有)→ 应用额度 → 队列 → 任务/尝试 → 许可。
// 与身份行分离,避免认证共享锁升级造成的互相等待。
func lockApplicationResources(ctx context.Context, tx *sql.Tx, ns core.ID, skip bool) (bool, error) {
q := `SELECT application_id FROM orbit_application_resources WHERE application_id=` + applicationForNamespace + ` FOR UPDATE`
if skip {
q += ` SKIP LOCKED`
}
var id core.ID
err := tx.QueryRowContext(ctx, q, ns).Scan(&id)
if errors.Is(err, sql.ErrNoRows) && skip {
return false, nil
}
if errors.Is(err, sql.ErrNoRows) {
return false, errors.New("application resources missing")
}
return err == nil, err
}
func readApplicationResources(ctx context.Context, tx *sql.Tx, ns core.ID) (resources.View, error) {
var v resources.View
err := tx.QueryRowContext(ctx, `SELECT application_id,revision,max_unfinished_tasks,unfinished_tasks,max_concurrency,active_attempts,pending_starts,rate_per_second FROM orbit_application_resources WHERE application_id=`+applicationForNamespace, ns).Scan(&v.ApplicationID, &v.Revision, &v.MaxUnfinishedTasks, &v.UnfinishedTasks, &v.MaxConcurrency, &v.ActiveAttempts, &v.PendingStarts, &v.RatePerSecond)
return v, err
}
func applicationStartBudget(ctx context.Context, tx *sql.Tx, ns core.ID, now time.Time) (rate, used uint32, pending, active uint64, max uint32, until *time.Time, err error) {
err = tx.QueryRowContext(ctx, `SELECT rate_per_second,starts_in_window,pending_starts,active_attempts,max_concurrency,start_window_until FROM orbit_application_resources WHERE application_id=`+applicationForNamespace, ns).Scan(&rate, &used, &pending, &active, &max, &until)
if until == nil || !until.After(now) {
used, until = 0, nil
}
return
}
func applicationCanClaim(ctx context.Context, tx *sql.Tx, ns core.ID, now time.Time) (bool, error) {
rate, used, pending, active, max, _, err := applicationStartBudget(ctx, tx, ns, now)
return err == nil && active < uint64(max) && used < rate && pending < uint64(rate-used), err
}
func reserveApplicationAttempt(ctx context.Context, tx *sql.Tx, ns core.ID) error {
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET active_attempts=active_attempts+1,pending_starts=pending_starts+1 WHERE application_id=`+applicationForNamespace+` AND active_attempts<max_concurrency`, ns)
return exactlyOne(r, err)
}
func consumeApplicationStart(ctx context.Context, tx *sql.Tx, ns core.ID, now time.Time) error {
rate, used, _, _, _, until, err := applicationStartBudget(ctx, tx, ns, now)
if err != nil {
return err
}
if used >= rate {
return worker.RateLimited
}
if until == nil {
t := now.Add(time.Second)
until = &t
}
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET starts_in_window=?,start_window_until=?,pending_starts=pending_starts-1 WHERE application_id=`+applicationForNamespace+` AND pending_starts>0`, used+1, until, ns)
return exactlyOne(r, err)
}
func releaseApplicationAttempt(ctx context.Context, tx *sql.Tx, ns, attempt core.ID) error {
var unstarted bool
if err := tx.QueryRowContext(ctx, `SELECT authorized_at IS NULL FROM pulse_attempts WHERE namespace_id=? AND id=?`, ns, attempt).Scan(&unstarted); err != nil {
return err
}
var pending uint64
if unstarted {
pending = 1
}
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET active_attempts=active_attempts-1,pending_starts=pending_starts-? WHERE application_id=`+applicationForNamespace+` AND active_attempts>0 AND pending_starts>=?`, pending, ns, pending)
return exactlyOne(r, err)
}
var _ resources.Store = (*TaskStore)(nil)
func (s *TaskStore) GetApplicationResources(ctx context.Context, token auth.Token) (resources.View, error) {
var out resources.View
err := s.transaction(ctx, token, "application:resources:read", func(tx *sql.Tx, p auth.Permissions) error {
var err error
out, err = readApplicationResources(ctx, tx, token.NamespaceID)
return err
})
return out, err
}
func (s *TaskStore) ConfigureApplicationResources(ctx context.Context, token auth.Token, key string, in resources.ConfigureInput) (resources.Result, error) {
var out resources.Result
err := s.transaction(ctx, token, "application:resources:manage", func(tx *sql.Tx, p auth.Permissions) error {
// 应用级动作显式授权;不从 queues:* 推导跨命名空间权限。
v, err := readApplicationResources(ctx, tx, token.NamespaceID)
if err != nil {
return err
}
identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "application.resources.configure", string(v.ApplicationID), key, "application", v.ApplicationID, in)
if err != nil {
return err
}
if cached != nil {
if err := json.Unmarshal(cached, &out); err != nil {
return err
}
out.Deduplicated = true
return nil
}
if _, err := lockApplicationResources(ctx, tx, token.NamespaceID, false); err != nil {
return err
}
before, err := readApplicationResources(ctx, tx, token.NamespaceID)
if err != nil {
return err
}
if before.Revision != in.ExpectedRevision {
return fault.Conflict
}
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET max_unfinished_tasks=?,max_concurrency=?,rate_per_second=?,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE application_id=? AND revision=?`, in.MaxUnfinishedTasks, in.MaxConcurrency, in.RatePerSecond, before.ApplicationID, in.ExpectedRevision)
if err := exactlyOne(r, err); err != nil {
return err
}
out.Resources, err = readApplicationResources(ctx, tx, token.NamespaceID)
if err != nil {
return err
}
if err := appendAudit(ctx, tx, token, identity, before.ApplicationID, "application", "application.resources.configure", in.Reason, before, out.Resources); err != nil {
return err
}
if err := emitEvent(ctx, tx, token.NamespaceID, before.ApplicationID, out.Resources.Revision, "application", "application.resources.configured", out.Resources); err != nil {
return err
}
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
})
return out, err
}

View File

@@ -0,0 +1,425 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"os"
"sync"
"testing"
"time"
core "git.yuntuo.hk/k2safe/orbit/model"
"git.yuntuo.hk/k2safe/orbit/platform/resources"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
func grantApplicationResources(f *executionFixture) {
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.actions',JSON_ARRAY_APPEND(JSON_EXTRACT(permissions,'$.actions'),'$','application:resources:read','$','application:resources:manage')) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
}
func applicationAPI(f *executionFixture) *resources.Service { return resources.New(NewTaskStore(f.db)) }
func configureApplication(t *testing.T, f *executionFixture, rate, concurrency, unfinished uint32) resources.View {
t.Helper()
api := applicationAPI(f)
v, err := api.Get(f.ctx, f.token)
if err != nil {
t.Fatal(err)
}
out, err := api.Configure(f.ctx, f.token, string(executionID(t)), resources.ConfigureInput{ExpectedRevision: v.Revision, Reason: "application quota regression", RatePerSecond: rate, MaxConcurrency: concurrency, MaxUnfinishedTasks: unfinished})
if err != nil {
t.Fatal(err)
}
return out.Resources
}
func assertApplicationCounts(t *testing.T, f *executionFixture, unfinished, active, pending uint64) {
t.Helper()
// 数据库核对不依赖当前应用是否停用,恢复仍应释放额度。
var got [3]uint64
if err := f.db.QueryRowContext(f.ctx, `SELECT unfinished_tasks,active_attempts,pending_starts FROM orbit_application_resources WHERE application_id=?`, f.resources.ApplicationID).Scan(&got[0], &got[1], &got[2]); err != nil {
t.Fatal(err)
}
if got != [3]uint64{unfinished, active, pending} {
t.Fatalf("application counters=%v want=%v", got, [3]uint64{unfinished, active, pending})
}
var actual [3]uint64
if err := f.db.QueryRowContext(f.ctx, `SELECT COUNT(*),COALESCE(SUM(t.state IN ('LEASED','RUNNING')),0),COALESCE(SUM(t.state='LEASED'),0) FROM pulse_tasks t JOIN orbit_namespaces n ON n.id=t.namespace_id WHERE n.application_id=? AND t.state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW')`, f.resources.ApplicationID).Scan(&actual[0], &actual[1], &actual[2]); err != nil {
t.Fatal(err)
}
if actual != got {
t.Fatalf("application counters=%v actual states=%v", got, actual)
}
}
func TestIntegrationApplicationResources(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
db, err := Open(ctx, dsn, 32, 8)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
fixture := func(t *testing.T, app ...core.ID) *executionFixture {
f := newExecutionFixture(t, ctx, db, app...)
grantApplicationResources(f)
return f
}
t.Run("explicit_authority_CAS_idempotency_and_atomic_audit", func(t *testing.T) {
f := newExecutionFixture(t, ctx, db)
api := applicationAPI(f)
_, err := api.Get(ctx, f.token)
expectWorkerCode(t, err, "PERMISSION_DENIED")
in := resources.ConfigureInput{ExpectedRevision: 1, Reason: "configure", MaxConcurrency: 2, RatePerSecond: 10, MaxUnfinishedTasks: 10}
_, err = api.Configure(ctx, f.token, "manage", in)
expectWorkerCode(t, err, "PERMISSION_DENIED")
grantApplicationResources(f)
v, err := api.Configure(ctx, f.token, "manage", in)
if err != nil || v.Resources.Revision != 2 {
t.Fatal(v, err)
}
f.submit(1)
replay, err := api.Configure(ctx, f.token, "manage", in)
if err != nil || !replay.Deduplicated || replay.Resources.UnfinishedTasks != 0 {
t.Fatal("receipt changed with usage", err)
}
_, err = api.Configure(ctx, f.token, "stale", in)
expectWorkerCode(t, err, "STATE_CONFLICT")
in.RatePerSecond = 20
_, err = api.Configure(ctx, f.token, "manage", in)
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
in.ExpectedRevision = 2
trigger := "test_app_audit_" + string(executionID(t))
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected audit failure'")
_, err = api.Configure(ctx, f.token, "rollback", in)
f.exec("DROP TRIGGER " + trigger)
if err == nil {
t.Fatal("audit fault ignored")
}
v2, err := api.Get(ctx, f.token)
if err != nil || v2.Revision != 2 || v2.RatePerSecond != 10 {
t.Fatal("configuration did not roll back", v2, err)
}
if _, err := api.Configure(ctx, f.token, "rollback", in); err != nil {
t.Fatal(err)
}
if f.count("orbit_audit_events", "action='application.resources.configure'") != 2 || f.count("orbit_events", "type='application.resources.configured'") != 2 {
t.Fatal("duplicate or missing audit/event")
}
g := fixture(t, f.resources.ApplicationID)
shared, err := applicationAPI(g).Get(ctx, g.token)
if err != nil || shared.Revision != 3 || shared.UnfinishedTasks != 1 {
t.Fatal(shared, err)
}
other := fixture(t)
if v, err := applicationAPI(other).Get(ctx, other.token); err != nil || v.Revision != 1 || v.UnfinishedTasks != 0 {
t.Fatal("other application leaked", v, err)
}
})
t.Run("concurrent_admission_across_namespaces_and_full_replay", func(t *testing.T) {
f := fixture(t)
g := fixture(t, f.resources.ApplicationID)
configureApplication(t, f, 100, 100, 3)
type answer struct {
f *executionFixture
key string
in service.SubmitInput
out service.SubmitResult
err error
}
ch := make(chan answer, 16)
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
target := f
if i%2 == 1 {
target = g
}
key := string(executionID(t))
wg.Add(1)
go func() {
defer wg.Done()
in := service.SubmitInput{QueueID: target.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
out, err := target.tasks.Submit(ctx, target.token, key, in)
ch <- answer{target, key, in, out, err}
}()
}
wg.Wait()
close(ch)
var accepted []answer
for a := range ch {
if a.err == nil {
accepted = append(accepted, a)
} else {
expectWorkerCode(t, a.err, "APPLICATION_FULL")
}
}
if len(accepted) != 3 {
t.Fatalf("accepted=%d", len(accepted))
}
assertApplicationCounts(t, f, 3, 0, 0)
a := accepted[0]
replay, err := a.f.tasks.Submit(ctx, a.f.token, a.key, a.in)
if err != nil || !replay.Deduplicated || replay.Task.ID != a.out.Task.ID {
t.Fatal("full app lost receipt", err)
}
configureApplication(t, g, 100, 100, 1)
assertApplicationCounts(t, f, 3, 0, 0)
for _, a := range accepted {
_, err := a.f.tasks.CancelTask(ctx, a.f.token, a.out.Task.ID, "cancel-"+a.key, service.CancelInput{ExpectedRevision: 1, Reason: "release"})
if err != nil {
t.Fatal(err)
}
}
assertApplicationCounts(t, g, 0, 0, 0)
g.submit(1)
assertApplicationCounts(t, f, 1, 0, 0)
})
t.Run("queue_rejection_and_failed_admission_do_not_leak_application_budget", func(t *testing.T) {
f := fixture(t)
configureApplication(t, f, 100, 10, 10)
configureResources(t, f, 100, 10, 1, false)
first := f.submit(1)
in := service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
_, err := f.tasks.Submit(ctx, f.token, "second", in)
expectWorkerCode(t, err, "QUEUE_FULL")
assertApplicationCounts(t, f, 1, 0, 0)
if _, err := f.tasks.CancelTask(ctx, f.token, first.ID, "cancel-first", service.CancelInput{ExpectedRevision: first.Revision, Reason: "release"}); err != nil {
t.Fatal(err)
}
trigger := "test_app_admission_" + string(executionID(t))
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected admission failure'")
_, err = f.tasks.Submit(ctx, f.token, "second", in)
f.exec("DROP TRIGGER " + trigger)
if err == nil {
t.Fatal("admission failure ignored")
}
assertApplicationCounts(t, f, 0, 0, 0)
assertQueueCount(t, f, 0)
if _, err := f.tasks.Submit(ctx, f.token, "second", in); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
assertQueueCount(t, f, 1)
})
t.Run("shared_concurrency_and_exact_release", func(t *testing.T) {
f := fixture(t)
g := fixture(t, f.resources.ApplicationID)
configureApplication(t, f, 100, 1, 10)
f.submit(1)
g.submit(1)
type claim struct {
f *executionFixture
a *worker.Assignment
err error
}
ch := make(chan claim, 2)
for _, target := range []*executionFixture{f, g} {
go func() { a, err := target.a.Claim(ctx, target.token, target.binding); ch <- claim{target, a, err} }()
}
var winner claim
for i := 0; i < 2; i++ {
a := <-ch
if a.err != nil {
t.Fatal(a.err)
}
if a.a != nil {
if winner.a != nil {
t.Fatal("cross namespace concurrency exceeded")
}
winner = a
}
}
if winner.a == nil {
t.Fatal("no claim")
}
assertApplicationCounts(t, f, 2, 1, 1)
task := winner.a.Task
_, err := winner.f.tasks.CancelTask(ctx, winner.f.token, task.ID, "cancel-leased", service.CancelInput{ExpectedRevision: task.Revision, Reason: "release before start"})
if err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
other := f
if winner.f == f {
other = g
}
ref := retryStart(t, other)
assertApplicationCounts(t, f, 1, 1, 0)
message := executionID(t)
input := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}
if _, err := other.a.Finish(ctx, other.token, other.binding, message, input); err != nil {
t.Fatal(err)
}
if out, err := other.a.Finish(ctx, other.token, other.binding, message, input); err != nil || !out.Deduplicated {
t.Fatal("finish replay", err)
}
assertApplicationCounts(t, f, 0, 0, 0)
})
t.Run("shared_start_window_downgrade_and_rollback", func(t *testing.T) {
f := fixture(t)
g := fixture(t, f.resources.ApplicationID)
configureApplication(t, f, 2, 10, 10)
f.submit(1)
g.submit(1)
_, a := f.claim()
_, b := g.claim()
assertApplicationCounts(t, f, 2, 2, 2)
configureApplication(t, g, 1, 1, 10)
trigger := "test_app_start_" + string(executionID(t))
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected start failure'")
message := executionID(t)
_, err := f.a.Start(ctx, f.token, f.binding, message, a)
f.exec("DROP TRIGGER " + trigger)
if err == nil {
t.Fatal("start failure ignored")
}
assertApplicationCounts(t, f, 2, 2, 2)
if _, err := f.a.Start(ctx, f.token, f.binding, message, a); err != nil {
t.Fatal(err)
}
// 固定尚未过期窗口,避免慢测试主机把竞争断言跨到下一秒。
f.exec(`UPDATE orbit_application_resources SET start_window_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE application_id=?`, f.resources.ApplicationID)
if out, err := f.a.Start(ctx, f.token, f.binding, message, a); err != nil || !out.Deduplicated {
t.Fatal("start replay charged again", err)
}
_, err = g.a.Start(ctx, g.token, g.binding, executionID(t), b)
expectWorkerCode(t, err, "RATE_LIMITED")
assertApplicationCounts(t, f, 2, 2, 1)
retryFinish(t, f, a, "SUCCEEDED", false)
configureApplication(t, f, 1, 10, 10)
g.submit(1)
if a, err := g.a.Claim(ctx, g.token, g.binding); err != nil || a != nil {
t.Fatal("pending start reservation bypassed", err)
}
g.exec(`UPDATE orbit_application_resources SET start_window_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE application_id=?`, g.resources.ApplicationID)
if a, err := g.a.Claim(ctx, g.token, g.binding); err != nil || a != nil {
t.Fatal("unstarted reservation lost across window", err)
}
if _, err := g.a.Start(ctx, g.token, g.binding, executionID(t), b); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 2, 1, 0)
})
t.Run("retry_recovery_unknown_and_disabled_application", func(t *testing.T) {
f := fixture(t)
configureApplication(t, f, 100, 10, 10)
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
ref := retryStart(t, f)
retryFinish(t, f, ref, "FAILED", true)
assertApplicationCounts(t, f, 1, 0, 0)
makeRetryDue(f, task.ID)
ref = retryStart(t, f)
f.expire(ref)
f.exec(`UPDATE orbit_applications SET enabled=FALSE WHERE id=?`, f.resources.ApplicationID)
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
if _, err := f.storeA.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
f.exec(`UPDATE orbit_applications SET enabled=TRUE WHERE id=?`, f.resources.ApplicationID)
if f.get(task.ID).State != pulse.TaskNeedsReview {
t.Fatal("unknown effect lost")
}
f.submit(1)
_, leased := f.claim()
f.expire(leased)
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
waiting := f.submit(1)
f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 SECOND),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, waiting.ID)
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 1, 0, 0)
})
t.Run("schedule_partial_batch_and_global_backpressure", func(t *testing.T) {
f := fixture(t)
g := fixture(t, f.resources.ApplicationID)
configureApplication(t, f, 100, 10, 2)
g.submit(1)
sf := newScheduleFixture(t, f)
v := sf.create(sf.definition())
sf.due(v, 2*time.Second)
sf.tick()
items := sf.occurrences(v.ID)
if len(items) != 1 || items[0].TaskID == nil {
t.Fatal("application schedule limit exceeded", len(items))
}
next := sf.current(v.ID).NextFireAt
if next == nil || !next.Equal(items[0].ScheduledAt.Add(time.Second)) {
t.Fatal("backpressure advanced unaccepted cursor")
}
assertApplicationCounts(t, f, 2, 0, 0)
assertQueueCount(t, f, 1)
assertQueueCount(t, g, 1)
_, err := f.tasks.CancelTask(ctx, f.token, *items[0].TaskID, "free-schedule", service.CancelInput{ExpectedRevision: 1, Reason: "free one"})
if err != nil {
t.Fatal(err)
}
sf.due(v, 0)
sf.tick()
items = sf.occurrences(v.ID)
if len(items) != 2 {
t.Fatal("original point not resumed")
}
assertApplicationCounts(t, f, 2, 0, 0)
assertQueueCount(t, f, 1)
})
t.Run("busy_application_is_skipped_during_recovery", func(t *testing.T) {
f := fixture(t)
g := fixture(t)
for _, v := range []*executionFixture{f, g} {
v.submit(1)
_, ref := v.claim()
v.expire(ref)
}
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
if _, err := lockApplicationResources(ctx, tx, f.resources.NamespaceID, false); err != nil {
t.Fatal(err)
}
short, stop := context.WithTimeout(ctx, time.Second)
defer stop()
if _, err := g.storeB.Recover(short, 100); err != nil {
t.Fatal("hot application blocked unrelated recovery", err)
}
assertApplicationCounts(t, f, 1, 1, 1)
assertApplicationCounts(t, g, 0, 0, 0)
if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
if _, err := f.storeB.Recover(ctx, 100); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 0, 0, 0)
})
t.Run("bootstrap_existing_application_rejects_invalid_parent", func(t *testing.T) {
f := fixture(t)
f.exec(`UPDATE orbit_applications SET enabled=FALSE WHERE id=?`, f.resources.ApplicationID)
for _, in := range []BootstrapInput{
{ExistingApplicationID: executionID(t), Namespace: "missing", Environment: "test", Queue: "q"},
{ExistingApplicationID: f.resources.ApplicationID, Namespace: "disabled", Environment: "test", Queue: "q"},
{Application: "new", ExistingApplicationID: f.resources.ApplicationID, Namespace: "ambiguous", Environment: "test", Queue: "q"},
} {
delivered := false
err := Bootstrap(ctx, db, in, func(BootstrapResult) error { delivered = true; return nil })
if err == nil || delivered {
t.Fatal("invalid application accepted")
}
}
})
}

View File

@@ -0,0 +1,96 @@
package mysql
import (
"context"
"os"
"testing"
"time"
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
"git.yuntuo.hk/k2safe/orbit/pulse/service"
mysqldriver "github.com/go-sql-driver/mysql"
)
// 建立有效在途任务,再将一次性库还原为结构完全相同的 v9,验证新增额度的真实回填和继续释放。
func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("requires disposable MySQL")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
admin, err := Open(ctx, dsn, 2, 1)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
name := "orbit_app_upgrade_" + string(executionID(t))
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
t.Fatal(err)
}
defer func() {
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
defer stop()
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
t.Error(err)
}
}()
config, err := mysqldriver.ParseDSN(dsn)
if err != nil {
t.Fatal("invalid test DSN")
}
config.DBName = name
db, err := Open(ctx, config.FormatDSN(), 8, 2)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
f := newExecutionFixture(t, ctx, db)
g := newExecutionFixture(t, ctx, db, f.resources.ApplicationID)
empty := newExecutionFixture(t, ctx, db)
f.submit(1)
ref := retryStart(t, f)
retryFinish(t, f, ref, "UNKNOWN", false)
f.submit(1)
_, leased := f.claim()
g.submit(1)
running := retryStart(t, g)
waiting := f.submit(1)
g.submit(1)
finished := retryStart(t, g)
retryFinish(t, g, finished, "SUCCEEDED", false)
assertApplicationCounts(t, f, 4, 2, 1)
// v10 只新增此表;其余列、索引与数据均是 v9 结构。
f.exec(`DROP TABLE orbit_application_resources`)
f.exec(`DELETE FROM orbit_schema_migrations WHERE version=10`)
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
if err := CheckSchema(ctx, db); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 4, 2, 1)
assertApplicationCounts(t, empty, 0, 0, 0)
var revision, rate, maxConcurrency, maxUnfinished uint64
var until *time.Time
if err := db.QueryRowContext(ctx, `SELECT revision,rate_per_second,max_concurrency,max_unfinished_tasks,start_window_until FROM orbit_application_resources WHERE application_id=?`, f.resources.ApplicationID).Scan(&revision, &rate, &maxConcurrency, &maxUnfinished, &until); err != nil || revision != 1 || rate != 100000 || maxConcurrency != 10000 || maxUnfinished != 1000000 || until != nil {
t.Fatal("invalid backfill defaults", err)
}
if v := f.get(waiting.ID); v.State != pulse.TaskWaiting || v.Revision != waiting.Revision {
t.Fatal("migration modified task")
}
if err := Migrate(ctx, db); err != nil {
t.Fatal(err)
}
assertApplicationCounts(t, f, 4, 2, 1)
current := f.get(leased.TaskID)
if _, err := f.tasks.CancelTask(ctx, f.token, current.ID, "cancel-upgraded", service.CancelInput{ExpectedRevision: current.Revision, Reason: "verify migrated lease release"}); err != nil {
t.Fatal(err)
}
retryFinish(t, g, running, "SUCCEEDED", false)
assertApplicationCounts(t, f, 2, 0, 0)
t.Log("v9 backfill: shared unfinished=4 active=2 pending=1; empty app=0; repeated migration preserved counts; old attempts released correctly")
}

View File

@@ -14,17 +14,19 @@ import (
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用,不暴露为公共 HTTP。
type BootstrapInput struct {
Application string // Application 应用名称,每次初始化创建独立应用。
Namespace string // Namespace 命名空间名称。
Environment string // Environment 环境名称,必须显式指定。
Queue string // Queue 初始队列名称。
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
Application string // Application 新应用名称,与 ExistingApplicationID 二选一。
Namespace string // Namespace 命名空间名称。
Environment string // Environment 环境名称,必须显式指定。
Queue string // Queue 初始队列名称。
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
ResourceAccess bool // ResourceAccess 显式授予整个应用的额度查询及管理权限。
ExistingApplicationID core.ID // ExistingApplicationID 在指定现有应用下创建命名空间,与 Application 二选一。
}
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
type BootstrapResult struct {
ApplicationID core.ID `json:"application_id"` // ApplicationID 新建应用标识。
ApplicationID core.ID `json:"application_id"` // ApplicationID 创建或关联的应用标识。
NamespaceID core.ID `json:"namespace_id"` // NamespaceID 新建隔离范围。
QueueID core.ID `json:"queue_id"` // QueueID 初始队列标识。
CredentialID core.ID `json:"credential_id"` // CredentialID 凭据记录标识。
@@ -33,11 +35,14 @@ type BootstrapResult struct {
}
func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(BootstrapResult) error) error {
for _, v := range []string{in.Application, in.Namespace, in.Queue} {
for _, v := range []string{in.Namespace, in.Queue} {
if !service.ValidName(v, 128) {
return fmt.Errorf("invalid bootstrap resource name")
}
}
if (in.ExistingApplicationID == "" && !service.ValidName(in.Application, 128)) || (in.ExistingApplicationID != "" && (in.ExistingApplicationID.Validate() != nil || in.Application != "")) {
return fmt.Errorf("exactly one new application name or existing application id is required")
}
if !service.ValidName(in.Environment, 32) || deliver == nil {
return fmt.Errorf("environment and secure delivery are required")
}
@@ -45,7 +50,15 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
return err
}
var out BootstrapResult
for _, dest := range []*core.ID{&out.ApplicationID, &out.NamespaceID, &out.QueueID, &out.CredentialID} {
var err error
out.ApplicationID = in.ExistingApplicationID
if out.ApplicationID == "" {
out.ApplicationID, err = core.NewID()
if err != nil {
return err
}
}
for _, dest := range []*core.ID{&out.NamespaceID, &out.QueueID, &out.CredentialID} {
v, err := core.NewID()
if err != nil {
return err
@@ -64,6 +77,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
if in.ScheduleAccess {
permissions.Actions = append(permissions.Actions, "schedules:read", "schedules:manage")
}
if in.ResourceAccess {
permissions.Actions = append(permissions.Actions, "application:resources:read", "application:resources:manage")
}
raw, err := json.Marshal(permissions)
if err != nil {
return err
@@ -74,8 +90,18 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
return err
}
out.ExpiresAt = now.Add(30 * 24 * time.Hour)
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_applications(id,name) VALUES (?,?)", out.ApplicationID, in.Application); err != nil {
return err
if in.ExistingApplicationID == "" {
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_applications(id,name) VALUES (?,?)", out.ApplicationID, in.Application); err != nil {
return err
}
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_application_resources(application_id) VALUES (?)", out.ApplicationID); err != nil {
return err
}
} else {
var id core.ID
if err := tx.QueryRowContext(ctx, `SELECT a.id FROM orbit_applications a JOIN orbit_application_resources r ON r.application_id=a.id WHERE a.id=? AND a.enabled=TRUE FOR SHARE`, out.ApplicationID).Scan(&id); err != nil {
return err
}
}
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,?)", out.NamespaceID, out.ApplicationID, in.Namespace, in.Environment); err != nil {
return err

View File

@@ -184,6 +184,7 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) {
t.Helper()
models := []tableModel{
platform.Application{},
platform.ApplicationResources{},
platform.Namespace{},
platform.Credential{},
platform.IdempotencyKey{},

View File

@@ -0,0 +1,29 @@
-- 停止旧服务后执行。身份与额度分表,避免认证共享锁升级;禁止数据库外键。
CREATE TABLE orbit_application_resources (
application_id BINARY(16) NOT NULL COMMENT '所属应用标识及主键,跨命名空间共享额度',
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '配置乐观版本,用量变化不增加版本',
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '创建时间,UTC 微秒',
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最后配置更新时间,UTC 微秒',
max_unfinished_tasks INT UNSIGNED NOT NULL DEFAULT 1000000 COMMENT '应用未终结任务上限,降额保留已有任务',
unfinished_tasks BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '等待、预占、运行及待核查任务总数,重试不重复计数',
max_concurrency INT UNSIGNED NOT NULL DEFAULT 10000 COMMENT '预占与运行尝试总上限,降额不撤销已有尝试',
active_attempts BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '当前预占与运行尝试总数,结果确认或到期恢复释放',
pending_starts BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已预占但未开始尝试数,包含在活跃尝试中',
rate_per_second INT UNSIGNED NOT NULL DEFAULT 100000 COMMENT '一秒固定窗口内开始授权上限,不是业务请求速率',
start_window_until DATETIME(6) NULL COMMENT '数据库时间的一秒窗口结束时刻,空表示尚未建立窗口',
starts_in_window INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '窗口中已确认的开始次数,重复回执不计数',
PRIMARY KEY (application_id),
CONSTRAINT ck_app_resources_revision CHECK (revision > 0),
CONSTRAINT ck_app_resources_limits CHECK (max_unfinished_tasks > 0 AND max_concurrency > 0 AND rate_per_second > 0),
CONSTRAINT ck_app_resources_pending CHECK (pending_starts <= active_attempts),
CONSTRAINT ck_app_resources_window CHECK (start_window_until IS NOT NULL OR starts_in_window=0)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='应用跨命名空间共享额度与持久化用量,无数据库外键';
INSERT INTO orbit_application_resources(application_id,unfinished_tasks,active_attempts,pending_starts)
SELECT a.id,COALESCE(t.unfinished,0),COALESCE(t.active,0),COALESCE(t.pending,0)
FROM orbit_applications a LEFT JOIN (
SELECT n.application_id,COUNT(*) AS unfinished,
SUM(t.state IN ('LEASED','RUNNING')) AS active,SUM(t.state='LEASED') AS pending
FROM pulse_tasks t JOIN orbit_namespaces n ON n.id=t.namespace_id
WHERE t.state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW') GROUP BY n.application_id
) t ON t.application_id=a.id;

View File

@@ -88,7 +88,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
}
indexes := func() []string {
t.Helper()
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME<>'orbit_application_resources' ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
if err != nil {
t.Fatal(err)
}

View File

@@ -13,8 +13,15 @@ import (
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
)
// 以下额度操作要求调用者已取得队列排他锁,且在同一事务中写入任务/尝试。
// 以下额度操作要求调用者先取得应用额度锁,再取得队列排他锁,且在同一事务中写入任务/尝试。
func reserveQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error {
v, err := readApplicationResources(ctx, tx, ns)
if err != nil {
return err
}
if v.UnfinishedTasks >= uint64(v.MaxUnfinishedTasks) {
return fault.ApplicationFull
}
r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks+1 WHERE namespace_id=? AND id=? AND unfinished_tasks<max_unfinished_tasks`, ns, queue)
if err != nil {
return err
@@ -26,11 +33,16 @@ func reserveQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error
if n != 1 {
return fault.QueueFull
}
return nil
r, err = tx.ExecContext(ctx, `UPDATE orbit_application_resources SET unfinished_tasks=unfinished_tasks+1 WHERE application_id=? AND unfinished_tasks<max_unfinished_tasks`, v.ApplicationID)
return exactlyOne(r, err)
}
func releaseQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error {
r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks-1 WHERE namespace_id=? AND id=? AND unfinished_tasks>0`, ns, queue)
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET unfinished_tasks=unfinished_tasks-1 WHERE application_id=`+applicationForNamespace+` AND unfinished_tasks>0`, ns)
if err := exactlyOne(r, err); err != nil {
return err
}
r, err = tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks-1 WHERE namespace_id=? AND id=? AND unfinished_tasks>0`, ns, queue)
return exactlyOne(r, err)
}
@@ -51,6 +63,9 @@ func queueStartBudget(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now ti
// queueCanClaim 把尚未 Start 的预占也算入当前窗口可用空间,避免限流时空耗尝试预算。
func queueCanClaim(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now time.Time) (bool, error) {
if ok, err := applicationCanClaim(ctx, tx, ns, now); err != nil || !ok {
return false, err
}
rate, used, _, err := queueStartBudget(ctx, tx, ns, queue, now)
if err != nil || used >= rate {
return false, err
@@ -72,6 +87,9 @@ func consumeQueueStart(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now t
end := now.Add(time.Second)
until = &end
}
if err := consumeApplicationStart(ctx, tx, ns, now); err != nil {
return err
}
r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET start_window_until=?,starts_in_window=? WHERE namespace_id=? AND id=?`, until, used+1, ns, queue)
return exactlyOne(r, err)
}

View File

@@ -100,6 +100,10 @@ func TestIntegrationSafeRetryUpgrade(t *testing.T) {
if err := db.QueryRowContext(ctx, `SELECT unfinished_tasks,max_unfinished_tasks,revision FROM pulse_queues WHERE namespace_id=? AND id=?`, ns, queue).Scan(&unfinished, &maxUnfinished, &queueRevision); err != nil || unfinished != 2 || maxUnfinished != 100000 || queueRevision != 1 {
t.Fatalf("resource upgrade changed queue or missed states: count=%d max=%d revision=%d err=%v", unfinished, maxUnfinished, queueRevision, err)
}
var appCount, appActive, appPending uint64
if err := db.QueryRowContext(ctx, `SELECT unfinished_tasks,active_attempts,pending_starts FROM orbit_application_resources WHERE application_id=?`, app).Scan(&appCount, &appActive, &appPending); err != nil || appCount != 2 || appActive != 0 || appPending != 0 {
t.Fatalf("application backfill incorrect: %d %d %d %v", appCount, appActive, appPending, err)
}
for state, id := range ids {
var gotState, gotPolicy string
var uncertain bool

View File

@@ -106,7 +106,11 @@ func TestIntegrationSchedules(t *testing.T) {
f := fixture(t)
finite := func(delay time.Duration) schedule.Definition {
in := f.definition()
in.Rule.StartAt = time.Now().UTC().Truncate(time.Microsecond).Add(delay)
var now time.Time
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
t.Fatal(err)
}
in.Rule.StartAt = now.Add(delay)
end := in.Rule.StartAt.Add(time.Millisecond)
in.Rule.EndAt = &end
return in
@@ -122,12 +126,20 @@ func TestIntegrationSchedules(t *testing.T) {
}
waitUntil := func(at time.Time) {
t.Helper()
timer := time.NewTimer(time.Until(at.Add(10 * time.Millisecond)))
defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
t.Fatal(ctx.Err())
// 调度使用数据库时间,不能依赖容器与宿主机时钟完全同步。
for {
var now time.Time
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
t.Fatal(err)
}
if now.After(at) {
return
}
select {
case <-time.After(10 * time.Millisecond):
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
}
v := f.create(finite(300 * time.Millisecond))
@@ -146,7 +158,7 @@ func TestIntegrationSchedules(t *testing.T) {
f.tick() // 第一版本结束,第二版本尚未计算第一个触发点。
pending := f.current(v.ID)
if pending.ProcessingVersion != 2 || pending.NextFireAt != nil || pending.CheckAt == nil {
t.Fatal("expected pending initialization of version 2")
t.Fatalf("expected pending initialization of version 2: processing=%d next=%v check=%v current=%d occurrences=%d", pending.ProcessingVersion, pending.NextFireAt, pending.CheckAt, pending.CurrentVersion, len(f.occurrences(v.ID)))
}
if !emptyMiddle {
waitUntil(middle.Rule.StartAt)

View File

@@ -72,7 +72,10 @@ func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int
if err != nil {
return err
}
// 和任务执行一致先锁队列,再锁计划;其他扫描节点不等待同一计划所在队列。
// 应用额度 → 队列 → 计划;忙碌资源留给下一批。
if ok, err := lockApplicationResources(ctx, tx, ns, true); err != nil || !ok {
return err
}
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed)
if errors.Is(err, sql.ErrNoRows) {
return nil
@@ -156,7 +159,7 @@ func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int
backpressured := false
for _, decision := range batch.Decisions {
if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
if errors.Is(err, fault.QueueFull) {
if errors.Is(err, fault.QueueFull) || errors.Is(err, fault.ApplicationFull) {
// 已提交本批较早的决定仍保留;满额的原触发点留待下次按冻结策略处理。
t := decision.ScheduledAt
next, backpressured = &t, true

View File

@@ -37,7 +37,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
out.Deduplicated = true
return nil
}
// 队列 → 任务 → 尝试 → 许可;取消不获取 Worker 会话锁。
// 应用额度 → 队列 → 任务 → 尝试 → 许可;取消不获取 Worker 会话锁。
if _, _, err := lockQueue(ctx, tx, token.NamespaceID, view.QueueID); err != nil {
return err
}

View File

@@ -170,10 +170,7 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
out.Deduplicated = true
return nil
}
var queue core.ID
if err := tx.QueryRowContext(ctx, "SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", token.NamespaceID, in.QueueID).Scan(&queue); errors.Is(err, sql.ErrNoRows) {
return fault.NotFound
} else if err != nil {
if _, _, err := lockQueue(ctx, tx, token.NamespaceID, in.QueueID); err != nil {
return err
}
now, err := dbNow(ctx, tx)

View File

@@ -46,6 +46,9 @@ func readExecutionTask(ctx context.Context, tx *sql.Tx, ns, id core.ID) (executi
return row, err
}
func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) {
if _, err = lockApplicationResources(ctx, tx, ns, false); err != nil {
return
}
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max)
if errors.Is(err, sql.ErrNoRows) {
err = fault.NotFound
@@ -232,6 +235,9 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
if err := exactlyOne(changed, err); err != nil {
return err
}
if err := reserveApplicationAttempt(ctx, tx, token.NamespaceID); err != nil {
return err
}
view, err := readTask(ctx, tx, token.NamespaceID, locked, false)
if err != nil {
return err
@@ -417,6 +423,9 @@ func releasePermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, now ti
if len(permits) == 0 {
return errors.New("active attempt has no permit to release")
}
if err := releaseApplicationAttempt(ctx, tx, ns, attempt); err != nil {
return err
}
for _, p := range permits {
result, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits-1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits>0`, now, ns, p.limit)
if err := exactlyOne(result, err); err != nil {

View File

@@ -41,10 +41,16 @@ func executionID(t *testing.T) core.ID {
}
return id
}
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB, application ...core.ID) *executionFixture {
t.Helper()
f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))}
err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil })
in := BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}
if len(application) > 0 {
in.Application = ""
in.ExistingApplicationID = application[0]
in.Namespace = "ns-" + string(executionID(t))
}
err := Bootstrap(ctx, db, in, func(v BootstrapResult) error { f.resources = v; return nil })
if err != nil {
t.Fatal(err)
}

View File

@@ -59,7 +59,10 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
out.Examined++
var changed pulse.TaskState
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
// 队列 → 任务 → 尝试 → 许可;忙碌资源留给下一批。
// 应用额度 → 队列 → 任务 → 尝试 → 许可;忙碌资源留给下一批。
if ok, err := lockApplicationResources(ctx, tx, c.ns, true); err != nil || !ok {
return err
}
var queue core.ID
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
if errors.Is(err, sql.ErrNoRows) {

View File

@@ -0,0 +1,32 @@
package httpserver
import (
"net/http"
"git.yuntuo.hk/k2safe/orbit/platform/auth"
"git.yuntuo.hk/k2safe/orbit/platform/resources"
)
// RegisterApplicationResources 只操作已认证凭据所属应用,不接受请求体覆盖归属。
func RegisterApplicationResources(server *http.Server, s *resources.Service) {
mux := server.Handler.(*http.ServeMux)
mux.HandleFunc("GET /v1/application/resources", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
if _, err := historyQuery(r, false); err != nil {
return nil, 0, err
}
out, err := s.Get(r.Context(), t)
return out, 200, err
}))
mux.HandleFunc("POST /v1/application/resources:configure", wrap(func(r *http.Request, t auth.Token) (any, int, error) {
var in resources.ConfigureInput
if err := decodeBody(r, &in); err != nil {
return nil, 0, err
}
key, err := idempotencyKey(r)
if err != nil {
return nil, 0, err
}
out, err := s.Configure(r.Context(), t, key, in)
return out, 200, err
}))
}

View File

@@ -189,7 +189,7 @@ func writeError(w http.ResponseWriter, id string, err error) {
if status == 0 {
status = 503
}
if known.Code == "QUEUE_FULL" {
if known.Code == "QUEUE_FULL" || known.Code == "APPLICATION_FULL" {
status = http.StatusTooManyRequests
w.Header().Set("Retry-After", "1")
}