feat: add bounded business state observability
This commit is contained in:
65
internal/observability/business.go
Normal file
65
internal/observability/business.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BusinessSnapshot 是存储层返回的固定基数业务快照。
|
||||
// Map 的键只由存储层固定状态集合产生，Registry 仍只向目录中的标签写值。
|
||||
type BusinessSnapshot struct {
|
||||
// TaskStates Pulse 任务状态的有界计数。
|
||||
TaskStates map[string]uint64
|
||||
// TaskOldestAgeSeconds 未完成任务中最早可处理记录的年龄。
|
||||
TaskOldestAgeSeconds float64
|
||||
// DeliveryStates 结果交付状态的有界计数。
|
||||
DeliveryStates map[string]uint64
|
||||
// DeliveryOldestAgeSeconds 待处理交付中最早可处理记录的年龄。
|
||||
DeliveryOldestAgeSeconds float64
|
||||
// WorkflowStates Weave 实例状态的有界计数。
|
||||
WorkflowStates map[string]uint64
|
||||
// WorkflowOldestAgeSeconds 未终结流程中最早更新时间的年龄。
|
||||
WorkflowOldestAgeSeconds float64
|
||||
// Truncated 是否至少有一类状态达到采样上限。
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
// RecordBusinessSnapshot records one bounded database sample. A failed sample
|
||||
// does not overwrite the last successful state, so stale data remains visible
|
||||
// together with an error counter and alertable last-success timestamp.
|
||||
func (r *Registry) RecordBusinessSnapshot(snapshot BusinessSnapshot, started time.Time, err error) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
outcome := "success"
|
||||
if err != nil {
|
||||
outcome = "error"
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
outcome = "timeout"
|
||||
}
|
||||
}
|
||||
r.AddCounter("orbit_business_snapshot_total", 1, Labels{"outcome": outcome})
|
||||
r.ObserveHistogram("orbit_business_snapshot_duration_seconds", time.Since(started).Seconds(), nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, state := range taskStates {
|
||||
r.SetGauge("orbit_pulse_tasks_state", float64(snapshot.TaskStates[state]), Labels{"state": state})
|
||||
}
|
||||
for _, state := range deliveryStates {
|
||||
r.SetGauge("orbit_result_deliveries_state", float64(snapshot.DeliveryStates[state]), Labels{"state": state})
|
||||
}
|
||||
for _, state := range workflowStates {
|
||||
r.SetGauge("orbit_weave_workflows_state", float64(snapshot.WorkflowStates[state]), Labels{"state": state})
|
||||
}
|
||||
r.SetGauge("orbit_pulse_tasks_oldest_age_seconds", snapshot.TaskOldestAgeSeconds, nil)
|
||||
r.SetGauge("orbit_result_delivery_oldest_age_seconds", snapshot.DeliveryOldestAgeSeconds, nil)
|
||||
r.SetGauge("orbit_weave_workflow_oldest_age_seconds", snapshot.WorkflowOldestAgeSeconds, nil)
|
||||
truncated := 0.0
|
||||
if snapshot.Truncated {
|
||||
truncated = 1
|
||||
}
|
||||
r.SetGauge("orbit_business_snapshot_truncated", truncated, nil)
|
||||
r.SetGauge("orbit_business_snapshot_last_success_timestamp_seconds", float64(time.Now().Unix()), nil)
|
||||
}
|
||||
@@ -6,6 +6,9 @@ var methods = []string{"GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS
|
||||
var routes = []string{"health_live", "health_ready", "worker_connect", "event_connect", "tasks", "queues", "workflows", "workflow_definitions", "schedules", "subscriptions", "application_resources", "other"}
|
||||
var statusClasses = []string{"1xx", "2xx", "3xx", "4xx", "5xx", "other"}
|
||||
var components = []string{"workflow", "schedule", "execution_recovery"}
|
||||
var taskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
var deliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
|
||||
var workflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
|
||||
// Every exported family is declared here, including each allowed label value.
|
||||
// Adding a metric therefore requires a code review; no user identifier is valid.
|
||||
@@ -33,5 +36,15 @@ func catalog() map[string]descriptor {
|
||||
}
|
||||
c["orbit_db_wait_count_total"] = descriptor{counter, "Cumulative waits for a MySQL connection", nil}
|
||||
c["orbit_db_wait_duration_seconds_total"] = descriptor{counter, "Cumulative seconds waiting for a MySQL connection", nil}
|
||||
c["orbit_pulse_tasks_state"] = descriptor{gauge, "Bounded count of Pulse tasks by durable state", map[string][]string{"state": taskStates}}
|
||||
c["orbit_pulse_tasks_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Pulse task", nil}
|
||||
c["orbit_result_deliveries_state"] = descriptor{gauge, "Bounded count of result deliveries by durable state", map[string][]string{"state": deliveryStates}}
|
||||
c["orbit_result_delivery_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled pending result delivery", nil}
|
||||
c["orbit_weave_workflows_state"] = descriptor{gauge, "Bounded count of Weave instances by durable state", map[string][]string{"state": workflowStates}}
|
||||
c["orbit_weave_workflow_oldest_age_seconds"] = descriptor{gauge, "Age of the oldest sampled unfinished Weave instance", nil}
|
||||
c["orbit_business_snapshot_total"] = descriptor{counter, "Completed bounded business-state snapshots by outcome", map[string][]string{"outcome": {"success", "error", "timeout"}}}
|
||||
c["orbit_business_snapshot_duration_seconds"] = descriptor{histogram, "Bounded business-state snapshot duration in seconds", nil}
|
||||
c["orbit_business_snapshot_last_success_timestamp_seconds"] = descriptor{gauge, "Unix time of the last successful bounded business-state snapshot", nil}
|
||||
c["orbit_business_snapshot_truncated"] = descriptor{gauge, "Whether a bounded business-state sample hit its per-state cap", nil}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -112,6 +112,42 @@ func TestRuntimeObservationDoesNotExposeErrors(t *testing.T) {
|
||||
disabled.RecordBatch("workflow", time.Now(), nil)
|
||||
disabled.RecordDatabase(sql.DBStats{})
|
||||
}
|
||||
|
||||
func TestBusinessSnapshotKeepsLastGoodStateOnFailure(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
started := time.Now().Add(-25 * time.Millisecond)
|
||||
r.RecordBusinessSnapshot(BusinessSnapshot{
|
||||
TaskStates: map[string]uint64{"WAITING": 3},
|
||||
TaskOldestAgeSeconds: 12.5,
|
||||
DeliveryStates: map[string]uint64{"PENDING": 2},
|
||||
DeliveryOldestAgeSeconds: 8.25,
|
||||
WorkflowStates: map[string]uint64{"RUNNING": 1},
|
||||
WorkflowOldestAgeSeconds: 4.75,
|
||||
Truncated: true,
|
||||
}, started, nil)
|
||||
first := r.Render()
|
||||
for _, want := range []string{
|
||||
`orbit_pulse_tasks_state{state="WAITING"} 3`,
|
||||
`orbit_result_deliveries_state{state="PENDING"} 2`,
|
||||
`orbit_weave_workflows_state{state="RUNNING"} 1`,
|
||||
"orbit_pulse_tasks_oldest_age_seconds 12.5",
|
||||
"orbit_business_snapshot_truncated 1",
|
||||
`orbit_business_snapshot_total{outcome="success"} 1`,
|
||||
} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("missing business sample %q", want)
|
||||
}
|
||||
}
|
||||
lastSuccess := r.families["orbit_business_snapshot_last_success_timestamp_seconds"].series[""].value.Load()
|
||||
r.RecordBusinessSnapshot(BusinessSnapshot{TaskStates: map[string]uint64{"WAITING": 99}}, time.Now(), context.DeadlineExceeded)
|
||||
second := r.Render()
|
||||
if !strings.Contains(second, `orbit_business_snapshot_total{outcome="timeout"} 1`) || !strings.Contains(second, `orbit_pulse_tasks_state{state="WAITING"} 3`) {
|
||||
t.Fatal("failed sample overwrote good state")
|
||||
}
|
||||
if got := r.families["orbit_business_snapshot_last_success_timestamp_seconds"].series[""].value.Load(); got != lastSuccess {
|
||||
t.Fatal("failed sample changed last success timestamp")
|
||||
}
|
||||
}
|
||||
func TestHTTPBoundedLabelsAndPanicPreserved(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
h := r.Middleware(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(202) }))
|
||||
|
||||
@@ -65,6 +65,9 @@ func TestIntegrationApplicationResourcesUpgrade(t *testing.T) {
|
||||
assertApplicationCounts(t, f, 4, 2, 1)
|
||||
// 还原后续新增表、字段和索引，得到真实 v9 结构；现有任务内容保持不变。
|
||||
f.exec(`ALTER TABLE orbit_events DROP INDEX ix_event_workflow_history,DROP COLUMN workflow_id`)
|
||||
f.exec(`ALTER TABLE pulse_tasks DROP INDEX ix_task_observe_state_available`)
|
||||
f.exec(`ALTER TABLE orbit_deliveries DROP INDEX ix_delivery_observe_state_available`)
|
||||
f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_observe_state_updated`)
|
||||
f.exec(`DROP TABLE weave_version_scopes`)
|
||||
f.exec(`ALTER TABLE weave_instances DROP INDEX ix_workflow_created,DROP INDEX ix_workflow_state_created,DROP INDEX ix_workflow_business_state_created,DROP INDEX ix_workflow_definition_created,DROP INDEX ix_workflow_definition_state_created,DROP INDEX ix_workflow_definition_business_created,DROP INDEX ix_workflow_definition_business_state_created`)
|
||||
f.exec(`ALTER TABLE weave_signals DROP INDEX ix_signal_history`)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- 为有界运行观测提供按状态和时间的前缀索引。
|
||||
-- 采集只读取固定状态的有限样本，不依赖外键或级联关系。
|
||||
ALTER TABLE pulse_tasks
|
||||
ADD KEY ix_task_observe_state_available (state, available_at, id);
|
||||
|
||||
ALTER TABLE orbit_deliveries
|
||||
ADD KEY ix_delivery_observe_state_available (state, available_at, id);
|
||||
|
||||
ALTER TABLE weave_instances
|
||||
ADD KEY ix_workflow_observe_state_updated (state, updated_at, id);
|
||||
@@ -89,7 +89,7 @@ func TestIntegrationNoForeignKeysUpgrade(t *testing.T) {
|
||||
indexes := func() []string {
|
||||
t.Helper()
|
||||
// 排除后续新增/替换索引；原有项在此逐项比较，v24/v25新增项由对应Query/Timeline升级测试核对。
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
rows, err := db.QueryContext(ctx, `SELECT CONCAT_WS(':',TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,COALESCE(SUB_PART,0),INDEX_TYPE,IS_VISIBLE) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME NOT IN ('orbit_application_resources','orbit_event_streams','orbit_result_events','weave_version_scopes') AND INDEX_NAME NOT IN ('ix_task_priority_dispatch','ix_delivery_subscription_due','ix_delivery_subscription_lease','ix_delivery_history','ix_delivery_state_history','uq_task_retry_parent','fk_pulse_tasks_3','ix_task_queue_business_created','ix_task_queue_business_state_created','uq_weave_task_result','ix_signal_expiry','ix_signal_activation','uq_wait_winner_signal','ix_task_workflow_owner','ix_signal_history','ix_event_workflow_history','ix_workflow_created','ix_workflow_state_created','ix_workflow_business_state_created','ix_workflow_definition_created','ix_workflow_definition_state_created','ix_workflow_definition_business_created','ix_workflow_definition_business_state_created','ix_task_observe_state_available','ix_delivery_observe_state_available','ix_workflow_observe_state_updated') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
126
internal/storage/mysql/operational_metrics.go
Normal file
126
internal/storage/mysql/operational_metrics.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
const operationalMetricSampleLimit = 10000
|
||||
|
||||
// OperationalSnapshot 是进程观测使用的有限数据库快照。
|
||||
// Counts 只包含固定状态的有界计数；达到上限时 Truncated 为 true，不能解释为精确总量。
|
||||
type OperationalSnapshot struct {
|
||||
// TaskStates 是 Pulse 任务按状态的有界计数。
|
||||
TaskStates map[string]uint64
|
||||
// TaskOldestAgeSeconds 是运行中任务中最早 available_at 的年龄。
|
||||
TaskOldestAgeSeconds float64
|
||||
// DeliveryStates 是结果交付按状态的有界计数。
|
||||
DeliveryStates map[string]uint64
|
||||
// DeliveryOldestAgeSeconds 是待处理交付中最早 available_at 的年龄。
|
||||
DeliveryOldestAgeSeconds float64
|
||||
// WorkflowStates 是 Weave 实例按状态的有界计数。
|
||||
WorkflowStates map[string]uint64
|
||||
// WorkflowOldestAgeSeconds 是未终结流程中最早 updated_at 的年龄。
|
||||
WorkflowOldestAgeSeconds float64
|
||||
// Truncated 表示至少一个状态达到有界采样上限。
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
var (
|
||||
operationalTaskStates = []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
operationalDeliveryStates = []string{"PENDING", "LEASED", "ACKED", "EXPIRED"}
|
||||
operationalWorkflowStates = []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"}
|
||||
)
|
||||
|
||||
// ReadOperationalSnapshot performs only bounded, indexed reads. It is intended
|
||||
// for a background sampler with a short context deadline, never for a request.
|
||||
func ReadOperationalSnapshot(ctx context.Context, db *sql.DB) (OperationalSnapshot, error) {
|
||||
if db == nil {
|
||||
return OperationalSnapshot{}, errors.New("database is required")
|
||||
}
|
||||
now, err := readDatabaseNow(ctx, db)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
tasks, taskTruncated, err := boundedStateCounts(ctx, db, "pulse_tasks", "ix_task_observe_state_available", operationalTaskStates)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
deliveries, deliveryTruncated, err := boundedStateCounts(ctx, db, "orbit_deliveries", "ix_delivery_observe_state_available", operationalDeliveryStates)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
workflows, workflowTruncated, err := boundedStateCounts(ctx, db, "weave_instances", "ix_workflow_observe_state_updated", operationalWorkflowStates)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
taskAge, err := oldestAge(ctx, db, "pulse_tasks", "ix_task_observe_state_available", []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW"}, "available_at", now)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
deliveryAge, err := oldestAge(ctx, db, "orbit_deliveries", "ix_delivery_observe_state_available", []string{"PENDING", "LEASED"}, "available_at", now)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
workflowAge, err := oldestAge(ctx, db, "weave_instances", "ix_workflow_observe_state_updated", []string{"RUNNING", "WAITING", "CANCELLING", "NEEDS_REVIEW"}, "updated_at", now)
|
||||
if err != nil {
|
||||
return OperationalSnapshot{}, err
|
||||
}
|
||||
return OperationalSnapshot{
|
||||
TaskStates: tasks, TaskOldestAgeSeconds: taskAge,
|
||||
DeliveryStates: deliveries, DeliveryOldestAgeSeconds: deliveryAge,
|
||||
WorkflowStates: workflows, WorkflowOldestAgeSeconds: workflowAge,
|
||||
Truncated: taskTruncated || deliveryTruncated || workflowTruncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func readDatabaseNow(ctx context.Context, db *sql.DB) (time.Time, error) {
|
||||
var now time.Time
|
||||
if err := db.QueryRowContext(ctx, "SELECT UTC_TIMESTAMP(6)").Scan(&now); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return now.UTC(), nil
|
||||
}
|
||||
|
||||
func boundedStateCounts(ctx context.Context, db *sql.DB, table, index string, states []string) (map[string]uint64, bool, error) {
|
||||
counts := make(map[string]uint64, len(states))
|
||||
truncated := false
|
||||
for _, state := range states {
|
||||
query := "SELECT COUNT(*) FROM (SELECT id FROM " + table + " FORCE INDEX (" + index + ") WHERE state=? LIMIT ?) AS bounded"
|
||||
var n uint64
|
||||
if err := db.QueryRowContext(ctx, query, state, operationalMetricSampleLimit+1).Scan(&n); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if n > operationalMetricSampleLimit {
|
||||
n = operationalMetricSampleLimit
|
||||
truncated = true
|
||||
}
|
||||
counts[state] = n
|
||||
}
|
||||
return counts, truncated, nil
|
||||
}
|
||||
|
||||
func oldestAge(ctx context.Context, db *sql.DB, table, index string, states []string, column string, now time.Time) (float64, error) {
|
||||
placeholders := "?"
|
||||
args := make([]any, 0, len(states))
|
||||
args = append(args, states[0])
|
||||
for _, state := range states[1:] {
|
||||
placeholders += ",?"
|
||||
args = append(args, state)
|
||||
}
|
||||
query := "SELECT " + column + " FROM " + table + " FORCE INDEX (" + index + ") WHERE state IN (" + placeholders + ") ORDER BY " + column + ",id LIMIT 1"
|
||||
var occurred time.Time
|
||||
if err := db.QueryRowContext(ctx, query, args...).Scan(&occurred); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
age := now.Sub(occurred.UTC()).Seconds()
|
||||
if age < 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return age, nil
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestIntegrationOperationalSnapshot verifies the bounded sampler against the
|
||||
// real MySQL grammar and the indexes installed by migration 000026.
|
||||
func TestIntegrationOperationalSnapshot(t *testing.T) {
|
||||
dsn := os.Getenv("ORBIT_TEST_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("set ORBIT_TEST_MYSQL_DSN to a disposable database, or run make integration")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
db, err := Open(ctx, dsn, 16, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := Migrate(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckSchema(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []struct {
|
||||
table string
|
||||
index string
|
||||
}{
|
||||
{"pulse_tasks", "ix_task_observe_state_available"},
|
||||
{"orbit_deliveries", "ix_delivery_observe_state_available"},
|
||||
{"weave_instances", "ix_workflow_observe_state_updated"},
|
||||
} {
|
||||
var n int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name=? AND index_name=?`, want.table, want.index).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n == 0 {
|
||||
t.Fatalf("missing operational index %s.%s", want.table, want.index)
|
||||
}
|
||||
}
|
||||
var foreignKeys int
|
||||
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&foreignKeys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if foreignKeys != 0 {
|
||||
t.Fatalf("operational migration introduced foreign keys: %d", foreignKeys)
|
||||
}
|
||||
|
||||
f := newExecutionFixture(t, ctx, db)
|
||||
f.submit(2)
|
||||
snapshot, err := ReadOperationalSnapshot(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := snapshot.TaskStates["WAITING"]; got < 1 {
|
||||
t.Fatalf("waiting task was not observed: %#v", snapshot.TaskStates)
|
||||
}
|
||||
if snapshot.TaskOldestAgeSeconds < 0 || snapshot.DeliveryOldestAgeSeconds < 0 || snapshot.WorkflowOldestAgeSeconds < 0 {
|
||||
t.Fatalf("negative age in snapshot: %#v", snapshot)
|
||||
}
|
||||
for _, state := range []string{"WAITING", "LEASED", "RUNNING", "NEEDS_REVIEW", "SUCCEEDED", "FAILED", "CANCELLED"} {
|
||||
if _, ok := snapshot.TaskStates[state]; !ok {
|
||||
t.Fatalf("missing fixed task state %q", state)
|
||||
}
|
||||
}
|
||||
if _, ok := snapshot.DeliveryStates["PENDING"]; !ok {
|
||||
t.Fatal("missing pending delivery state")
|
||||
}
|
||||
if _, ok := snapshot.WorkflowStates["RUNNING"]; !ok {
|
||||
t.Fatal("missing running workflow state")
|
||||
}
|
||||
|
||||
cancelled, cancelNow := context.WithCancel(ctx)
|
||||
cancelNow()
|
||||
if _, err := ReadOperationalSnapshot(cancelled, db); err == nil {
|
||||
t.Fatal("cancelled context unexpectedly completed snapshot")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user