feat: enforce shared application resource quotas across namespaces
This commit is contained in:
142
internal/storage/mysql/application_resources.go
Normal file
142
internal/storage/mysql/application_resources.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.yuntuo.hk/k2safe/orbit/internal/fault"
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/auth"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/resources"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// 归属不允许在线修改;认证已经锁定身份。恢复也需释放被停用应用的额度。
|
||||
const applicationForNamespace = `(SELECT application_id FROM orbit_namespaces WHERE id=?)`
|
||||
|
||||
// 统一顺序:认证 → Worker 会话(如有)→ 应用额度 → 队列 → 任务/尝试 → 许可。
|
||||
// 与身份行分离,避免认证共享锁升级造成的互相等待。
|
||||
func lockApplicationResources(ctx context.Context, tx *sql.Tx, ns core.ID, skip bool) (bool, error) {
|
||||
q := `SELECT application_id FROM orbit_application_resources WHERE application_id=` + applicationForNamespace + ` FOR UPDATE`
|
||||
if skip {
|
||||
q += ` SKIP LOCKED`
|
||||
}
|
||||
var id core.ID
|
||||
err := tx.QueryRowContext(ctx, q, ns).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) && skip {
|
||||
return false, nil
|
||||
}
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, errors.New("application resources missing")
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
func readApplicationResources(ctx context.Context, tx *sql.Tx, ns core.ID) (resources.View, error) {
|
||||
var v resources.View
|
||||
err := tx.QueryRowContext(ctx, `SELECT application_id,revision,max_unfinished_tasks,unfinished_tasks,max_concurrency,active_attempts,pending_starts,rate_per_second FROM orbit_application_resources WHERE application_id=`+applicationForNamespace, ns).Scan(&v.ApplicationID, &v.Revision, &v.MaxUnfinishedTasks, &v.UnfinishedTasks, &v.MaxConcurrency, &v.ActiveAttempts, &v.PendingStarts, &v.RatePerSecond)
|
||||
return v, err
|
||||
}
|
||||
func applicationStartBudget(ctx context.Context, tx *sql.Tx, ns core.ID, now time.Time) (rate, used uint32, pending, active uint64, max uint32, until *time.Time, err error) {
|
||||
err = tx.QueryRowContext(ctx, `SELECT rate_per_second,starts_in_window,pending_starts,active_attempts,max_concurrency,start_window_until FROM orbit_application_resources WHERE application_id=`+applicationForNamespace, ns).Scan(&rate, &used, &pending, &active, &max, &until)
|
||||
if until == nil || !until.After(now) {
|
||||
used, until = 0, nil
|
||||
}
|
||||
return
|
||||
}
|
||||
func applicationCanClaim(ctx context.Context, tx *sql.Tx, ns core.ID, now time.Time) (bool, error) {
|
||||
rate, used, pending, active, max, _, err := applicationStartBudget(ctx, tx, ns, now)
|
||||
return err == nil && active < uint64(max) && used < rate && pending < uint64(rate-used), err
|
||||
}
|
||||
func reserveApplicationAttempt(ctx context.Context, tx *sql.Tx, ns core.ID) error {
|
||||
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET active_attempts=active_attempts+1,pending_starts=pending_starts+1 WHERE application_id=`+applicationForNamespace+` AND active_attempts<max_concurrency`, ns)
|
||||
return exactlyOne(r, err)
|
||||
}
|
||||
func consumeApplicationStart(ctx context.Context, tx *sql.Tx, ns core.ID, now time.Time) error {
|
||||
rate, used, _, _, _, until, err := applicationStartBudget(ctx, tx, ns, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if used >= rate {
|
||||
return worker.RateLimited
|
||||
}
|
||||
if until == nil {
|
||||
t := now.Add(time.Second)
|
||||
until = &t
|
||||
}
|
||||
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET starts_in_window=?,start_window_until=?,pending_starts=pending_starts-1 WHERE application_id=`+applicationForNamespace+` AND pending_starts>0`, used+1, until, ns)
|
||||
return exactlyOne(r, err)
|
||||
}
|
||||
func releaseApplicationAttempt(ctx context.Context, tx *sql.Tx, ns, attempt core.ID) error {
|
||||
var unstarted bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT authorized_at IS NULL FROM pulse_attempts WHERE namespace_id=? AND id=?`, ns, attempt).Scan(&unstarted); err != nil {
|
||||
return err
|
||||
}
|
||||
var pending uint64
|
||||
if unstarted {
|
||||
pending = 1
|
||||
}
|
||||
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET active_attempts=active_attempts-1,pending_starts=pending_starts-? WHERE application_id=`+applicationForNamespace+` AND active_attempts>0 AND pending_starts>=?`, pending, ns, pending)
|
||||
return exactlyOne(r, err)
|
||||
}
|
||||
|
||||
var _ resources.Store = (*TaskStore)(nil)
|
||||
|
||||
func (s *TaskStore) GetApplicationResources(ctx context.Context, token auth.Token) (resources.View, error) {
|
||||
var out resources.View
|
||||
err := s.transaction(ctx, token, "application:resources:read", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
var err error
|
||||
out, err = readApplicationResources(ctx, tx, token.NamespaceID)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
func (s *TaskStore) ConfigureApplicationResources(ctx context.Context, token auth.Token, key string, in resources.ConfigureInput) (resources.Result, error) {
|
||||
var out resources.Result
|
||||
err := s.transaction(ctx, token, "application:resources:manage", func(tx *sql.Tx, p auth.Permissions) error {
|
||||
// 应用级动作显式授权;不从 queues:* 推导跨命名空间权限。
|
||||
v, err := readApplicationResources(ctx, tx, token.NamespaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
identity, cached, err := reserveIdentity(ctx, tx, token.NamespaceID, "application.resources.configure", string(v.ApplicationID), key, "application", v.ApplicationID, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cached != nil {
|
||||
if err := json.Unmarshal(cached, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
if _, err := lockApplicationResources(ctx, tx, token.NamespaceID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
before, err := readApplicationResources(ctx, tx, token.NamespaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if before.Revision != in.ExpectedRevision {
|
||||
return fault.Conflict
|
||||
}
|
||||
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET max_unfinished_tasks=?,max_concurrency=?,rate_per_second=?,revision=revision+1,updated_at=UTC_TIMESTAMP(6) WHERE application_id=? AND revision=?`, in.MaxUnfinishedTasks, in.MaxConcurrency, in.RatePerSecond, before.ApplicationID, in.ExpectedRevision)
|
||||
if err := exactlyOne(r, err); err != nil {
|
||||
return err
|
||||
}
|
||||
out.Resources, err = readApplicationResources(ctx, tx, token.NamespaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := appendAudit(ctx, tx, token, identity, before.ApplicationID, "application", "application.resources.configure", in.Reason, before, out.Resources); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emitEvent(ctx, tx, token.NamespaceID, before.ApplicationID, out.Resources.Revision, "application", "application.resources.configured", out.Resources); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveIdentity(ctx, tx, token.NamespaceID, identity, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
425
internal/storage/mysql/application_resources_integration_test.go
Normal file
425
internal/storage/mysql/application_resources_integration_test.go
Normal file
@@ -0,0 +1,425 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
core "git.yuntuo.hk/k2safe/orbit/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/platform/resources"
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
func grantApplicationResources(f *executionFixture) {
|
||||
f.exec(`UPDATE orbit_credentials SET permissions=JSON_SET(permissions,'$.actions',JSON_ARRAY_APPEND(JSON_EXTRACT(permissions,'$.actions'),'$','application:resources:read','$','application:resources:manage')) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, f.resources.CredentialID)
|
||||
}
|
||||
func applicationAPI(f *executionFixture) *resources.Service { return resources.New(NewTaskStore(f.db)) }
|
||||
func configureApplication(t *testing.T, f *executionFixture, rate, concurrency, unfinished uint32) resources.View {
|
||||
t.Helper()
|
||||
api := applicationAPI(f)
|
||||
v, err := api.Get(f.ctx, f.token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := api.Configure(f.ctx, f.token, string(executionID(t)), resources.ConfigureInput{ExpectedRevision: v.Revision, Reason: "application quota regression", RatePerSecond: rate, MaxConcurrency: concurrency, MaxUnfinishedTasks: unfinished})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Resources
|
||||
}
|
||||
func assertApplicationCounts(t *testing.T, f *executionFixture, unfinished, active, pending uint64) {
|
||||
t.Helper()
|
||||
// 数据库核对不依赖当前应用是否停用,恢复仍应释放额度。
|
||||
var got [3]uint64
|
||||
if err := f.db.QueryRowContext(f.ctx, `SELECT unfinished_tasks,active_attempts,pending_starts FROM orbit_application_resources WHERE application_id=?`, f.resources.ApplicationID).Scan(&got[0], &got[1], &got[2]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != [3]uint64{unfinished, active, pending} {
|
||||
t.Fatalf("application counters=%v want=%v", got, [3]uint64{unfinished, active, pending})
|
||||
}
|
||||
var actual [3]uint64
|
||||
if err := f.db.QueryRowContext(f.ctx, `SELECT COUNT(*),COALESCE(SUM(t.state IN ('LEASED','RUNNING')),0),COALESCE(SUM(t.state='LEASED'),0) FROM pulse_tasks t JOIN orbit_namespaces n ON n.id=t.namespace_id WHERE n.application_id=? AND t.state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW')`, f.resources.ApplicationID).Scan(&actual[0], &actual[1], &actual[2]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if actual != got {
|
||||
t.Fatalf("application counters=%v actual states=%v", got, actual)
|
||||
}
|
||||
}
|
||||
func TestIntegrationApplicationResources(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 32, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := func(t *testing.T, app ...core.ID) *executionFixture {
|
||||
f := newExecutionFixture(t, ctx, db, app...)
|
||||
grantApplicationResources(f)
|
||||
return f
|
||||
}
|
||||
t.Run("explicit_authority_CAS_idempotency_and_atomic_audit", func(t *testing.T) {
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
api := applicationAPI(f)
|
||||
_, err := api.Get(ctx, f.token)
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
in := resources.ConfigureInput{ExpectedRevision: 1, Reason: "configure", MaxConcurrency: 2, RatePerSecond: 10, MaxUnfinishedTasks: 10}
|
||||
_, err = api.Configure(ctx, f.token, "manage", in)
|
||||
expectWorkerCode(t, err, "PERMISSION_DENIED")
|
||||
grantApplicationResources(f)
|
||||
v, err := api.Configure(ctx, f.token, "manage", in)
|
||||
if err != nil || v.Resources.Revision != 2 {
|
||||
t.Fatal(v, err)
|
||||
}
|
||||
f.submit(1)
|
||||
replay, err := api.Configure(ctx, f.token, "manage", in)
|
||||
if err != nil || !replay.Deduplicated || replay.Resources.UnfinishedTasks != 0 {
|
||||
t.Fatal("receipt changed with usage", err)
|
||||
}
|
||||
_, err = api.Configure(ctx, f.token, "stale", in)
|
||||
expectWorkerCode(t, err, "STATE_CONFLICT")
|
||||
in.RatePerSecond = 20
|
||||
_, err = api.Configure(ctx, f.token, "manage", in)
|
||||
expectWorkerCode(t, err, "IDEMPOTENCY_CONFLICT")
|
||||
in.ExpectedRevision = 2
|
||||
trigger := "test_app_audit_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_audit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected audit failure'")
|
||||
_, err = api.Configure(ctx, f.token, "rollback", in)
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil {
|
||||
t.Fatal("audit fault ignored")
|
||||
}
|
||||
v2, err := api.Get(ctx, f.token)
|
||||
if err != nil || v2.Revision != 2 || v2.RatePerSecond != 10 {
|
||||
t.Fatal("configuration did not roll back", v2, err)
|
||||
}
|
||||
if _, err := api.Configure(ctx, f.token, "rollback", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("orbit_audit_events", "action='application.resources.configure'") != 2 || f.count("orbit_events", "type='application.resources.configured'") != 2 {
|
||||
t.Fatal("duplicate or missing audit/event")
|
||||
}
|
||||
g := fixture(t, f.resources.ApplicationID)
|
||||
shared, err := applicationAPI(g).Get(ctx, g.token)
|
||||
if err != nil || shared.Revision != 3 || shared.UnfinishedTasks != 1 {
|
||||
t.Fatal(shared, err)
|
||||
}
|
||||
other := fixture(t)
|
||||
if v, err := applicationAPI(other).Get(ctx, other.token); err != nil || v.Revision != 1 || v.UnfinishedTasks != 0 {
|
||||
t.Fatal("other application leaked", v, err)
|
||||
}
|
||||
})
|
||||
t.Run("concurrent_admission_across_namespaces_and_full_replay", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
g := fixture(t, f.resources.ApplicationID)
|
||||
configureApplication(t, f, 100, 100, 3)
|
||||
type answer struct {
|
||||
f *executionFixture
|
||||
key string
|
||||
in service.SubmitInput
|
||||
out service.SubmitResult
|
||||
err error
|
||||
}
|
||||
ch := make(chan answer, 16)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
target := f
|
||||
if i%2 == 1 {
|
||||
target = g
|
||||
}
|
||||
key := string(executionID(t))
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
in := service.SubmitInput{QueueID: target.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
|
||||
out, err := target.tasks.Submit(ctx, target.token, key, in)
|
||||
ch <- answer{target, key, in, out, err}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
var accepted []answer
|
||||
for a := range ch {
|
||||
if a.err == nil {
|
||||
accepted = append(accepted, a)
|
||||
} else {
|
||||
expectWorkerCode(t, a.err, "APPLICATION_FULL")
|
||||
}
|
||||
}
|
||||
if len(accepted) != 3 {
|
||||
t.Fatalf("accepted=%d", len(accepted))
|
||||
}
|
||||
assertApplicationCounts(t, f, 3, 0, 0)
|
||||
a := accepted[0]
|
||||
replay, err := a.f.tasks.Submit(ctx, a.f.token, a.key, a.in)
|
||||
if err != nil || !replay.Deduplicated || replay.Task.ID != a.out.Task.ID {
|
||||
t.Fatal("full app lost receipt", err)
|
||||
}
|
||||
configureApplication(t, g, 100, 100, 1)
|
||||
assertApplicationCounts(t, f, 3, 0, 0)
|
||||
for _, a := range accepted {
|
||||
_, err := a.f.tasks.CancelTask(ctx, a.f.token, a.out.Task.ID, "cancel-"+a.key, service.CancelInput{ExpectedRevision: 1, Reason: "release"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assertApplicationCounts(t, g, 0, 0, 0)
|
||||
g.submit(1)
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
})
|
||||
t.Run("queue_rejection_and_failed_admission_do_not_leak_application_budget", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
configureApplication(t, f, 100, 10, 10)
|
||||
configureResources(t, f, 100, 10, 1, false)
|
||||
first := f.submit(1)
|
||||
in := service.SubmitInput{QueueID: f.resources.QueueID, Handler: "object.prepare", HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{}`)}
|
||||
_, err := f.tasks.Submit(ctx, f.token, "second", in)
|
||||
expectWorkerCode(t, err, "QUEUE_FULL")
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
if _, err := f.tasks.CancelTask(ctx, f.token, first.ID, "cancel-first", service.CancelInput{ExpectedRevision: first.Revision, Reason: "release"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trigger := "test_app_admission_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected admission failure'")
|
||||
_, err = f.tasks.Submit(ctx, f.token, "second", in)
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil {
|
||||
t.Fatal("admission failure ignored")
|
||||
}
|
||||
assertApplicationCounts(t, f, 0, 0, 0)
|
||||
assertQueueCount(t, f, 0)
|
||||
if _, err := f.tasks.Submit(ctx, f.token, "second", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
assertQueueCount(t, f, 1)
|
||||
})
|
||||
t.Run("shared_concurrency_and_exact_release", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
g := fixture(t, f.resources.ApplicationID)
|
||||
configureApplication(t, f, 100, 1, 10)
|
||||
f.submit(1)
|
||||
g.submit(1)
|
||||
type claim struct {
|
||||
f *executionFixture
|
||||
a *worker.Assignment
|
||||
err error
|
||||
}
|
||||
ch := make(chan claim, 2)
|
||||
for _, target := range []*executionFixture{f, g} {
|
||||
go func() { a, err := target.a.Claim(ctx, target.token, target.binding); ch <- claim{target, a, err} }()
|
||||
}
|
||||
var winner claim
|
||||
for i := 0; i < 2; i++ {
|
||||
a := <-ch
|
||||
if a.err != nil {
|
||||
t.Fatal(a.err)
|
||||
}
|
||||
if a.a != nil {
|
||||
if winner.a != nil {
|
||||
t.Fatal("cross namespace concurrency exceeded")
|
||||
}
|
||||
winner = a
|
||||
}
|
||||
}
|
||||
if winner.a == nil {
|
||||
t.Fatal("no claim")
|
||||
}
|
||||
assertApplicationCounts(t, f, 2, 1, 1)
|
||||
task := winner.a.Task
|
||||
_, err := winner.f.tasks.CancelTask(ctx, winner.f.token, task.ID, "cancel-leased", service.CancelInput{ExpectedRevision: task.Revision, Reason: "release before start"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
other := f
|
||||
if winner.f == f {
|
||||
other = g
|
||||
}
|
||||
ref := retryStart(t, other)
|
||||
assertApplicationCounts(t, f, 1, 1, 0)
|
||||
message := executionID(t)
|
||||
input := worker.FinishInput{Ref: ref, Outcome: "SUCCEEDED", Result: json.RawMessage(`{}`)}
|
||||
if _, err := other.a.Finish(ctx, other.token, other.binding, message, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := other.a.Finish(ctx, other.token, other.binding, message, input); err != nil || !out.Deduplicated {
|
||||
t.Fatal("finish replay", err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 0, 0, 0)
|
||||
})
|
||||
t.Run("shared_start_window_downgrade_and_rollback", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
g := fixture(t, f.resources.ApplicationID)
|
||||
configureApplication(t, f, 2, 10, 10)
|
||||
f.submit(1)
|
||||
g.submit(1)
|
||||
_, a := f.claim()
|
||||
_, b := g.claim()
|
||||
assertApplicationCounts(t, f, 2, 2, 2)
|
||||
configureApplication(t, g, 1, 1, 10)
|
||||
trigger := "test_app_start_" + string(executionID(t))
|
||||
f.exec("CREATE TRIGGER " + trigger + " BEFORE INSERT ON orbit_events FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='injected start failure'")
|
||||
message := executionID(t)
|
||||
_, err := f.a.Start(ctx, f.token, f.binding, message, a)
|
||||
f.exec("DROP TRIGGER " + trigger)
|
||||
if err == nil {
|
||||
t.Fatal("start failure ignored")
|
||||
}
|
||||
assertApplicationCounts(t, f, 2, 2, 2)
|
||||
if _, err := f.a.Start(ctx, f.token, f.binding, message, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 固定尚未过期窗口,避免慢测试主机把竞争断言跨到下一秒。
|
||||
f.exec(`UPDATE orbit_application_resources SET start_window_until=DATE_ADD(UTC_TIMESTAMP(6),INTERVAL 1 HOUR) WHERE application_id=?`, f.resources.ApplicationID)
|
||||
if out, err := f.a.Start(ctx, f.token, f.binding, message, a); err != nil || !out.Deduplicated {
|
||||
t.Fatal("start replay charged again", err)
|
||||
}
|
||||
_, err = g.a.Start(ctx, g.token, g.binding, executionID(t), b)
|
||||
expectWorkerCode(t, err, "RATE_LIMITED")
|
||||
assertApplicationCounts(t, f, 2, 2, 1)
|
||||
retryFinish(t, f, a, "SUCCEEDED", false)
|
||||
configureApplication(t, f, 1, 10, 10)
|
||||
g.submit(1)
|
||||
if a, err := g.a.Claim(ctx, g.token, g.binding); err != nil || a != nil {
|
||||
t.Fatal("pending start reservation bypassed", err)
|
||||
}
|
||||
g.exec(`UPDATE orbit_application_resources SET start_window_until=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE application_id=?`, g.resources.ApplicationID)
|
||||
if a, err := g.a.Claim(ctx, g.token, g.binding); err != nil || a != nil {
|
||||
t.Fatal("unstarted reservation lost across window", err)
|
||||
}
|
||||
if _, err := g.a.Start(ctx, g.token, g.binding, executionID(t), b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 2, 1, 0)
|
||||
})
|
||||
t.Run("retry_recovery_unknown_and_disabled_application", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
configureApplication(t, f, 100, 10, 10)
|
||||
task := retryTask(t, f, 2, pulse.RecoveryReconcileOnly)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "FAILED", true)
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
makeRetryDue(f, task.ID)
|
||||
ref = retryStart(t, f)
|
||||
f.expire(ref)
|
||||
f.exec(`UPDATE orbit_applications SET enabled=FALSE WHERE id=?`, f.resources.ApplicationID)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
if _, err := f.storeA.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
f.exec(`UPDATE orbit_applications SET enabled=TRUE WHERE id=?`, f.resources.ApplicationID)
|
||||
if f.get(task.ID).State != pulse.TaskNeedsReview {
|
||||
t.Fatal("unknown effect lost")
|
||||
}
|
||||
f.submit(1)
|
||||
_, leased := f.claim()
|
||||
f.expire(leased)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
waiting := f.submit(1)
|
||||
f.exec(`UPDATE pulse_tasks SET available_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 2 SECOND),deadline_at=DATE_SUB(UTC_TIMESTAMP(6),INTERVAL 1 SECOND) WHERE namespace_id=? AND id=?`, f.resources.NamespaceID, waiting.ID)
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 0, 0)
|
||||
})
|
||||
t.Run("schedule_partial_batch_and_global_backpressure", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
g := fixture(t, f.resources.ApplicationID)
|
||||
configureApplication(t, f, 100, 10, 2)
|
||||
g.submit(1)
|
||||
sf := newScheduleFixture(t, f)
|
||||
v := sf.create(sf.definition())
|
||||
sf.due(v, 2*time.Second)
|
||||
sf.tick()
|
||||
items := sf.occurrences(v.ID)
|
||||
if len(items) != 1 || items[0].TaskID == nil {
|
||||
t.Fatal("application schedule limit exceeded", len(items))
|
||||
}
|
||||
next := sf.current(v.ID).NextFireAt
|
||||
if next == nil || !next.Equal(items[0].ScheduledAt.Add(time.Second)) {
|
||||
t.Fatal("backpressure advanced unaccepted cursor")
|
||||
}
|
||||
assertApplicationCounts(t, f, 2, 0, 0)
|
||||
assertQueueCount(t, f, 1)
|
||||
assertQueueCount(t, g, 1)
|
||||
_, err := f.tasks.CancelTask(ctx, f.token, *items[0].TaskID, "free-schedule", service.CancelInput{ExpectedRevision: 1, Reason: "free one"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sf.due(v, 0)
|
||||
sf.tick()
|
||||
items = sf.occurrences(v.ID)
|
||||
if len(items) != 2 {
|
||||
t.Fatal("original point not resumed")
|
||||
}
|
||||
assertApplicationCounts(t, f, 2, 0, 0)
|
||||
assertQueueCount(t, f, 1)
|
||||
})
|
||||
t.Run("busy_application_is_skipped_during_recovery", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
g := fixture(t)
|
||||
for _, v := range []*executionFixture{f, g} {
|
||||
v.submit(1)
|
||||
_, ref := v.claim()
|
||||
v.expire(ref)
|
||||
}
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := lockApplicationResources(ctx, tx, f.resources.NamespaceID, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
short, stop := context.WithTimeout(ctx, time.Second)
|
||||
defer stop()
|
||||
if _, err := g.storeB.Recover(short, 100); err != nil {
|
||||
t.Fatal("hot application blocked unrelated recovery", err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 1, 1, 1)
|
||||
assertApplicationCounts(t, g, 0, 0, 0)
|
||||
if err := tx.Rollback(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.storeB.Recover(ctx, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 0, 0, 0)
|
||||
})
|
||||
t.Run("bootstrap_existing_application_rejects_invalid_parent", func(t *testing.T) {
|
||||
f := fixture(t)
|
||||
f.exec(`UPDATE orbit_applications SET enabled=FALSE WHERE id=?`, f.resources.ApplicationID)
|
||||
for _, in := range []BootstrapInput{
|
||||
{ExistingApplicationID: executionID(t), Namespace: "missing", Environment: "test", Queue: "q"},
|
||||
{ExistingApplicationID: f.resources.ApplicationID, Namespace: "disabled", Environment: "test", Queue: "q"},
|
||||
{Application: "new", ExistingApplicationID: f.resources.ApplicationID, Namespace: "ambiguous", Environment: "test", Queue: "q"},
|
||||
} {
|
||||
delivered := false
|
||||
err := Bootstrap(ctx, db, in, func(BootstrapResult) error { delivered = true; return nil })
|
||||
if err == nil || delivered {
|
||||
t.Fatal("invalid application accepted")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pulse "git.yuntuo.hk/k2safe/orbit/pulse/model"
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/service"
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// 建立有效在途任务,再将一次性库还原为结构完全相同的 v9,验证新增额度的真实回填和继续释放。
|
||||
func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires disposable MySQL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
admin, err := Open(ctx, dsn, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
name := "orbit_app_upgrade_" + string(executionID(t))
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"`"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stop()
|
||||
if _, err := admin.ExecContext(cleanup, "DROP DATABASE `"+name+"`"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
config, err := mysqldriver.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test DSN")
|
||||
}
|
||||
config.DBName = name
|
||||
db, err := Open(ctx, config.FormatDSN(), 8, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
g := newExecutionFixture(t, ctx, db, f.resources.ApplicationID)
|
||||
empty := newExecutionFixture(t, ctx, db)
|
||||
f.submit(1)
|
||||
ref := retryStart(t, f)
|
||||
retryFinish(t, f, ref, "UNKNOWN", false)
|
||||
f.submit(1)
|
||||
_, leased := f.claim()
|
||||
g.submit(1)
|
||||
running := retryStart(t, g)
|
||||
waiting := f.submit(1)
|
||||
g.submit(1)
|
||||
finished := retryStart(t, g)
|
||||
retryFinish(t, g, finished, "SUCCEEDED", false)
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
// v10 只新增此表;其余列、索引与数据均是 v9 结构。
|
||||
f.exec(`DROP TABLE orbit_application_resources`)
|
||||
f.exec(`DELETE FROM orbit_schema_migrations WHERE version=10`)
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
assertApplicationCounts(t, empty, 0, 0, 0)
|
||||
var revision, rate, maxConcurrency, maxUnfinished uint64
|
||||
var until *time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT revision,rate_per_second,max_concurrency,max_unfinished_tasks,start_window_until FROM orbit_application_resources WHERE application_id=?`, f.resources.ApplicationID).Scan(&revision, &rate, &maxConcurrency, &maxUnfinished, &until); err != nil || revision != 1 || rate != 100000 || maxConcurrency != 10000 || maxUnfinished != 1000000 || until != nil {
|
||||
t.Fatal("invalid backfill defaults", err)
|
||||
}
|
||||
if v := f.get(waiting.ID); v.State != pulse.TaskWaiting || v.Revision != waiting.Revision {
|
||||
t.Fatal("migration modified task")
|
||||
}
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
current := f.get(leased.TaskID)
|
||||
if _, err := f.tasks.CancelTask(ctx, f.token, current.ID, "cancel-upgraded", service.CancelInput{ExpectedRevision: current.Revision, Reason: "verify migrated lease release"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retryFinish(t, g, running, "SUCCEEDED", false)
|
||||
assertApplicationCounts(t, f, 2, 0, 0)
|
||||
t.Log("v9 backfill: shared unfinished=4 active=2 pending=1; empty app=0; repeated migration preserved counts; old attempts released correctly")
|
||||
}
|
||||
@@ -14,17 +14,19 @@ import (
|
||||
|
||||
// BootstrapInput 仅供持有数据库管理权限的初始化命令使用,不暴露为公共 HTTP。
|
||||
type BootstrapInput struct {
|
||||
Application string // Application 应用名称,每次初始化创建独立应用。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称,必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
|
||||
Application string // Application 新应用名称,与 ExistingApplicationID 二选一。
|
||||
Namespace string // Namespace 命名空间名称。
|
||||
Environment string // Environment 环境名称,必须显式指定。
|
||||
Queue string // Queue 初始队列名称。
|
||||
WorkerAccess bool // WorkerAccess 显式增加开发用 Worker 接入及消费权限。
|
||||
ScheduleAccess bool // ScheduleAccess 显式增加开发用计划查询及管理权限。
|
||||
ResourceAccess bool // ResourceAccess 显式授予整个应用的额度查询及管理权限。
|
||||
ExistingApplicationID core.ID // ExistingApplicationID 在指定现有应用下创建命名空间,与 Application 二选一。
|
||||
}
|
||||
|
||||
// BootstrapResult 包含一次性交付的原始凭据,禁止记录到日志或提交到版本库。
|
||||
type BootstrapResult struct {
|
||||
ApplicationID core.ID `json:"application_id"` // ApplicationID 新建应用标识。
|
||||
ApplicationID core.ID `json:"application_id"` // ApplicationID 创建或关联的应用标识。
|
||||
NamespaceID core.ID `json:"namespace_id"` // NamespaceID 新建隔离范围。
|
||||
QueueID core.ID `json:"queue_id"` // QueueID 初始队列标识。
|
||||
CredentialID core.ID `json:"credential_id"` // CredentialID 凭据记录标识。
|
||||
@@ -33,11 +35,14 @@ type BootstrapResult struct {
|
||||
}
|
||||
|
||||
func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(BootstrapResult) error) error {
|
||||
for _, v := range []string{in.Application, in.Namespace, in.Queue} {
|
||||
for _, v := range []string{in.Namespace, in.Queue} {
|
||||
if !service.ValidName(v, 128) {
|
||||
return fmt.Errorf("invalid bootstrap resource name")
|
||||
}
|
||||
}
|
||||
if (in.ExistingApplicationID == "" && !service.ValidName(in.Application, 128)) || (in.ExistingApplicationID != "" && (in.ExistingApplicationID.Validate() != nil || in.Application != "")) {
|
||||
return fmt.Errorf("exactly one new application name or existing application id is required")
|
||||
}
|
||||
if !service.ValidName(in.Environment, 32) || deliver == nil {
|
||||
return fmt.Errorf("environment and secure delivery are required")
|
||||
}
|
||||
@@ -45,7 +50,15 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
return err
|
||||
}
|
||||
var out BootstrapResult
|
||||
for _, dest := range []*core.ID{&out.ApplicationID, &out.NamespaceID, &out.QueueID, &out.CredentialID} {
|
||||
var err error
|
||||
out.ApplicationID = in.ExistingApplicationID
|
||||
if out.ApplicationID == "" {
|
||||
out.ApplicationID, err = core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, dest := range []*core.ID{&out.NamespaceID, &out.QueueID, &out.CredentialID} {
|
||||
v, err := core.NewID()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -64,6 +77,9 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
if in.ScheduleAccess {
|
||||
permissions.Actions = append(permissions.Actions, "schedules:read", "schedules:manage")
|
||||
}
|
||||
if in.ResourceAccess {
|
||||
permissions.Actions = append(permissions.Actions, "application:resources:read", "application:resources:manage")
|
||||
}
|
||||
raw, err := json.Marshal(permissions)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -74,8 +90,18 @@ func Bootstrap(ctx context.Context, db *sql.DB, in BootstrapInput, deliver func(
|
||||
return err
|
||||
}
|
||||
out.ExpiresAt = now.Add(30 * 24 * time.Hour)
|
||||
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_applications(id,name) VALUES (?,?)", out.ApplicationID, in.Application); err != nil {
|
||||
return err
|
||||
if in.ExistingApplicationID == "" {
|
||||
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_applications(id,name) VALUES (?,?)", out.ApplicationID, in.Application); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_application_resources(application_id) VALUES (?)", out.ApplicationID); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
var id core.ID
|
||||
if err := tx.QueryRowContext(ctx, `SELECT a.id FROM orbit_applications a JOIN orbit_application_resources r ON r.application_id=a.id WHERE a.id=? AND a.enabled=TRUE FOR SHARE`, out.ApplicationID).Scan(&id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "INSERT INTO orbit_namespaces(id,application_id,name,environment) VALUES (?,?,?,?)", out.NamespaceID, out.ApplicationID, in.Namespace, in.Environment); err != nil {
|
||||
return err
|
||||
|
||||
@@ -184,6 +184,7 @@ func checkModelColumns(t *testing.T, ctx context.Context, db *sql.DB) {
|
||||
t.Helper()
|
||||
models := []tableModel{
|
||||
platform.Application{},
|
||||
platform.ApplicationResources{},
|
||||
platform.Namespace{},
|
||||
platform.Credential{},
|
||||
platform.IdempotencyKey{},
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
-- 停止旧服务后执行。身份与额度分表,避免认证共享锁升级;禁止数据库外键。
|
||||
CREATE TABLE orbit_application_resources (
|
||||
application_id BINARY(16) NOT NULL COMMENT '所属应用标识及主键,跨命名空间共享额度',
|
||||
revision BIGINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '配置乐观版本,用量变化不增加版本',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '创建时间,UTC 微秒',
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT '最后配置更新时间,UTC 微秒',
|
||||
max_unfinished_tasks INT UNSIGNED NOT NULL DEFAULT 1000000 COMMENT '应用未终结任务上限,降额保留已有任务',
|
||||
unfinished_tasks BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '等待、预占、运行及待核查任务总数,重试不重复计数',
|
||||
max_concurrency INT UNSIGNED NOT NULL DEFAULT 10000 COMMENT '预占与运行尝试总上限,降额不撤销已有尝试',
|
||||
active_attempts BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '当前预占与运行尝试总数,结果确认或到期恢复释放',
|
||||
pending_starts BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '已预占但未开始尝试数,包含在活跃尝试中',
|
||||
rate_per_second INT UNSIGNED NOT NULL DEFAULT 100000 COMMENT '一秒固定窗口内开始授权上限,不是业务请求速率',
|
||||
start_window_until DATETIME(6) NULL COMMENT '数据库时间的一秒窗口结束时刻,空表示尚未建立窗口',
|
||||
starts_in_window INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '窗口中已确认的开始次数,重复回执不计数',
|
||||
PRIMARY KEY (application_id),
|
||||
CONSTRAINT ck_app_resources_revision CHECK (revision > 0),
|
||||
CONSTRAINT ck_app_resources_limits CHECK (max_unfinished_tasks > 0 AND max_concurrency > 0 AND rate_per_second > 0),
|
||||
CONSTRAINT ck_app_resources_pending CHECK (pending_starts <= active_attempts),
|
||||
CONSTRAINT ck_app_resources_window CHECK (start_window_until IS NOT NULL OR starts_in_window=0)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs COMMENT='应用跨命名空间共享额度与持久化用量,无数据库外键';
|
||||
|
||||
INSERT INTO orbit_application_resources(application_id,unfinished_tasks,active_attempts,pending_starts)
|
||||
SELECT a.id,COALESCE(t.unfinished,0),COALESCE(t.active,0),COALESCE(t.pending,0)
|
||||
FROM orbit_applications a LEFT JOIN (
|
||||
SELECT n.application_id,COUNT(*) AS unfinished,
|
||||
SUM(t.state IN ('LEASED','RUNNING')) AS active,SUM(t.state='LEASED') AS pending
|
||||
FROM pulse_tasks t JOIN orbit_namespaces n ON n.id=t.namespace_id
|
||||
WHERE t.state IN ('WAITING','LEASED','RUNNING','NEEDS_REVIEW') GROUP BY n.application_id
|
||||
) t ON t.application_id=a.id;
|
||||
@@ -88,7 +88,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
}
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME<>'orbit_application_resources' ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -13,8 +13,15 @@ import (
|
||||
"git.yuntuo.hk/k2safe/orbit/pulse/worker"
|
||||
)
|
||||
|
||||
// 以下额度操作要求调用者已取得队列排他锁,且在同一事务中写入任务/尝试。
|
||||
// 以下额度操作要求调用者先取得应用额度锁,再取得队列排他锁,且在同一事务中写入任务/尝试。
|
||||
func reserveQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error {
|
||||
v, err := readApplicationResources(ctx, tx, ns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v.UnfinishedTasks >= uint64(v.MaxUnfinishedTasks) {
|
||||
return fault.ApplicationFull
|
||||
}
|
||||
r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks+1 WHERE namespace_id=? AND id=? AND unfinished_tasks<max_unfinished_tasks`, ns, queue)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -26,11 +33,16 @@ func reserveQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error
|
||||
if n != 1 {
|
||||
return fault.QueueFull
|
||||
}
|
||||
return nil
|
||||
r, err = tx.ExecContext(ctx, `UPDATE orbit_application_resources SET unfinished_tasks=unfinished_tasks+1 WHERE application_id=? AND unfinished_tasks<max_unfinished_tasks`, v.ApplicationID)
|
||||
return exactlyOne(r, err)
|
||||
}
|
||||
|
||||
func releaseQueueTask(ctx context.Context, tx *sql.Tx, ns, queue core.ID) error {
|
||||
r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks-1 WHERE namespace_id=? AND id=? AND unfinished_tasks>0`, ns, queue)
|
||||
r, err := tx.ExecContext(ctx, `UPDATE orbit_application_resources SET unfinished_tasks=unfinished_tasks-1 WHERE application_id=`+applicationForNamespace+` AND unfinished_tasks>0`, ns)
|
||||
if err := exactlyOne(r, err); err != nil {
|
||||
return err
|
||||
}
|
||||
r, err = tx.ExecContext(ctx, `UPDATE pulse_queues SET unfinished_tasks=unfinished_tasks-1 WHERE namespace_id=? AND id=? AND unfinished_tasks>0`, ns, queue)
|
||||
return exactlyOne(r, err)
|
||||
}
|
||||
|
||||
@@ -51,6 +63,9 @@ func queueStartBudget(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now ti
|
||||
|
||||
// queueCanClaim 把尚未 Start 的预占也算入当前窗口可用空间,避免限流时空耗尝试预算。
|
||||
func queueCanClaim(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now time.Time) (bool, error) {
|
||||
if ok, err := applicationCanClaim(ctx, tx, ns, now); err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
rate, used, _, err := queueStartBudget(ctx, tx, ns, queue, now)
|
||||
if err != nil || used >= rate {
|
||||
return false, err
|
||||
@@ -72,6 +87,9 @@ func consumeQueueStart(ctx context.Context, tx *sql.Tx, ns, queue core.ID, now t
|
||||
end := now.Add(time.Second)
|
||||
until = &end
|
||||
}
|
||||
if err := consumeApplicationStart(ctx, tx, ns, now); err != nil {
|
||||
return err
|
||||
}
|
||||
r, err := tx.ExecContext(ctx, `UPDATE pulse_queues SET start_window_until=?,starts_in_window=? WHERE namespace_id=? AND id=?`, until, used+1, ns, queue)
|
||||
return exactlyOne(r, err)
|
||||
}
|
||||
|
||||
@@ -100,6 +100,10 @@ func TestIntegrationSafeRetryUpgrade(t *testing.T) {
|
||||
if err := db.QueryRowContext(ctx, `SELECT unfinished_tasks,max_unfinished_tasks,revision FROM pulse_queues WHERE namespace_id=? AND id=?`, ns, queue).Scan(&unfinished, &maxUnfinished, &queueRevision); err != nil || unfinished != 2 || maxUnfinished != 100000 || queueRevision != 1 {
|
||||
t.Fatalf("resource upgrade changed queue or missed states: count=%d max=%d revision=%d err=%v", unfinished, maxUnfinished, queueRevision, err)
|
||||
}
|
||||
var appCount, appActive, appPending uint64
|
||||
if err := db.QueryRowContext(ctx, `SELECT unfinished_tasks,active_attempts,pending_starts FROM orbit_application_resources WHERE application_id=?`, app).Scan(&appCount, &appActive, &appPending); err != nil || appCount != 2 || appActive != 0 || appPending != 0 {
|
||||
t.Fatalf("application backfill incorrect: %d %d %d %v", appCount, appActive, appPending, err)
|
||||
}
|
||||
for state, id := range ids {
|
||||
var gotState, gotPolicy string
|
||||
var uncertain bool
|
||||
|
||||
@@ -106,7 +106,11 @@ func TestIntegrationSchedules(t *testing.T) {
|
||||
f := fixture(t)
|
||||
finite := func(delay time.Duration) schedule.Definition {
|
||||
in := f.definition()
|
||||
in.Rule.StartAt = time.Now().UTC().Truncate(time.Microsecond).Add(delay)
|
||||
var now time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in.Rule.StartAt = now.Add(delay)
|
||||
end := in.Rule.StartAt.Add(time.Millisecond)
|
||||
in.Rule.EndAt = &end
|
||||
return in
|
||||
@@ -122,12 +126,20 @@ func TestIntegrationSchedules(t *testing.T) {
|
||||
}
|
||||
waitUntil := func(at time.Time) {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(time.Until(at.Add(10 * time.Millisecond)))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
// 调度使用数据库时间,不能依赖容器与宿主机时钟完全同步。
|
||||
for {
|
||||
var now time.Time
|
||||
if err := db.QueryRowContext(ctx, `SELECT UTC_TIMESTAMP(6)`).Scan(&now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now.After(at) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
v := f.create(finite(300 * time.Millisecond))
|
||||
@@ -146,7 +158,7 @@ func TestIntegrationSchedules(t *testing.T) {
|
||||
f.tick() // 第一版本结束,第二版本尚未计算第一个触发点。
|
||||
pending := f.current(v.ID)
|
||||
if pending.ProcessingVersion != 2 || pending.NextFireAt != nil || pending.CheckAt == nil {
|
||||
t.Fatal("expected pending initialization of version 2")
|
||||
t.Fatalf("expected pending initialization of version 2: processing=%d next=%v check=%v current=%d occurrences=%d", pending.ProcessingVersion, pending.NextFireAt, pending.CheckAt, pending.CurrentVersion, len(f.occurrences(v.ID)))
|
||||
}
|
||||
if !emptyMiddle {
|
||||
waitUntil(middle.Rule.StartAt)
|
||||
|
||||
@@ -72,7 +72,10 @@ func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 和任务执行一致先锁队列,再锁计划;其他扫描节点不等待同一计划所在队列。
|
||||
// 应用额度 → 队列 → 计划;忙碌资源留给下一批。
|
||||
if ok, err := lockApplicationResources(ctx, tx, ns, true); err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, ns, queue).Scan(&allowed)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
@@ -156,7 +159,7 @@ func (s *ScheduleStore) tickOne(ctx context.Context, ns, id, queue core.ID) (int
|
||||
backpressured := false
|
||||
for _, decision := range batch.Decisions {
|
||||
if err := s.saveOccurrence(ctx, tx, v, version, decision, now); err != nil {
|
||||
if errors.Is(err, fault.QueueFull) {
|
||||
if errors.Is(err, fault.QueueFull) || errors.Is(err, fault.ApplicationFull) {
|
||||
// 已提交本批较早的决定仍保留;满额的原触发点留待下次按冻结策略处理。
|
||||
t := decision.ScheduledAt
|
||||
next, backpressured = &t, true
|
||||
|
||||
@@ -37,7 +37,7 @@ func (s *TaskStore) CancelTask(ctx context.Context, token auth.Token, id core.ID
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
// 队列 → 任务 → 尝试 → 许可;取消不获取 Worker 会话锁。
|
||||
// 应用额度 → 队列 → 任务 → 尝试 → 许可;取消不获取 Worker 会话锁。
|
||||
if _, _, err := lockQueue(ctx, tx, token.NamespaceID, view.QueueID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -170,10 +170,7 @@ func (s *TaskStore) Submit(ctx context.Context, token auth.Token, key string, in
|
||||
out.Deduplicated = true
|
||||
return nil
|
||||
}
|
||||
var queue core.ID
|
||||
if err := tx.QueryRowContext(ctx, "SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", token.NamespaceID, in.QueueID).Scan(&queue); errors.Is(err, sql.ErrNoRows) {
|
||||
return fault.NotFound
|
||||
} else if err != nil {
|
||||
if _, _, err := lockQueue(ctx, tx, token.NamespaceID, in.QueueID); err != nil {
|
||||
return err
|
||||
}
|
||||
now, err := dbNow(ctx, tx)
|
||||
|
||||
@@ -46,6 +46,9 @@ func readExecutionTask(ctx context.Context, tx *sql.Tx, ns, id core.ID) (executi
|
||||
return row, err
|
||||
}
|
||||
func lockQueue(ctx context.Context, tx *sql.Tx, ns, id core.ID) (paused bool, max uint32, err error) {
|
||||
if _, err = lockApplicationResources(ctx, tx, ns, false); err != nil {
|
||||
return
|
||||
}
|
||||
err = tx.QueryRowContext(ctx, "SELECT paused,max_concurrency FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE", ns, id).Scan(&paused, &max)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = fault.NotFound
|
||||
@@ -232,6 +235,9 @@ func (s *ExecutionStore) Claim(ctx context.Context, token auth.Token, b worker.B
|
||||
if err := exactlyOne(changed, err); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := reserveApplicationAttempt(ctx, tx, token.NamespaceID); err != nil {
|
||||
return err
|
||||
}
|
||||
view, err := readTask(ctx, tx, token.NamespaceID, locked, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -417,6 +423,9 @@ func releasePermits(ctx context.Context, tx *sql.Tx, ns, attempt core.ID, now ti
|
||||
if len(permits) == 0 {
|
||||
return errors.New("active attempt has no permit to release")
|
||||
}
|
||||
if err := releaseApplicationAttempt(ctx, tx, ns, attempt); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range permits {
|
||||
result, err := tx.ExecContext(ctx, `UPDATE pulse_concurrency_limits SET used_permits=used_permits-1,revision=revision+1,updated_at=? WHERE namespace_id=? AND id=? AND used_permits>0`, now, ns, p.limit)
|
||||
if err := exactlyOne(result, err); err != nil {
|
||||
|
||||
@@ -41,10 +41,16 @@ func executionID(t *testing.T) core.ID {
|
||||
}
|
||||
return id
|
||||
}
|
||||
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB) *executionFixture {
|
||||
func newExecutionFixture(t *testing.T, ctx context.Context, db *sql.DB, application ...core.ID) *executionFixture {
|
||||
t.Helper()
|
||||
f := &executionFixture{t: t, ctx: ctx, db: db, tasks: service.New(NewTaskStore(db))}
|
||||
err := Bootstrap(ctx, db, BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}, func(v BootstrapResult) error { f.resources = v; return nil })
|
||||
in := BootstrapInput{Application: "worker-" + string(executionID(t)), Namespace: "default", Environment: "test", Queue: "work"}
|
||||
if len(application) > 0 {
|
||||
in.Application = ""
|
||||
in.ExistingApplicationID = application[0]
|
||||
in.Namespace = "ns-" + string(executionID(t))
|
||||
}
|
||||
err := Bootstrap(ctx, db, in, func(v BootstrapResult) error { f.resources = v; return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -59,7 +59,10 @@ func (s *ExecutionStore) Recover(ctx context.Context, limit int) (worker.Recover
|
||||
out.Examined++
|
||||
var changed pulse.TaskState
|
||||
err := WithTx(ctx, s.db, func(tx *sql.Tx) error {
|
||||
// 队列 → 任务 → 尝试 → 许可;忙碌资源留给下一批。
|
||||
// 应用额度 → 队列 → 任务 → 尝试 → 许可;忙碌资源留给下一批。
|
||||
if ok, err := lockApplicationResources(ctx, tx, c.ns, true); err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
var queue core.ID
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM pulse_queues WHERE namespace_id=? AND id=? FOR UPDATE SKIP LOCKED`, c.ns, c.queue).Scan(&queue)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
|
||||
Reference in New Issue
Block a user