feat: add automatic Worker runtime with lease recovery and bounded shutdown
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
// Package orbit 是 Orbit 的独立 Go SDK，提供健康、任务、队列与 Worker 底层协议。
|
||||
// WorkerConnection 不自动重连、续租或运行处理器；自动运行器与流程 SDK 仍在开发。
|
||||
// WorkerConnection 是底层接口；NewWorker/Run 自动管理处理器、续租、恢复与排空。
|
||||
// 流程 SDK 等后续能力按阶段继续开发。
|
||||
// SDK 不导入服务端模型、迁移或数据库依赖。
|
||||
package orbit
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// worker-protocol 演示一次文本规范化任务的真实执行与原消息重报。
|
||||
// 这是底层协议示例；持续运行、自动续租与优雅退出由后续 Worker 运行器提供。
|
||||
// 这是底层协议示例；持续运行、自动续租与优雅退出见 examples/worker 的运行器示例。
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
133
sdk/go/examples/worker/main.go
Normal file
133
sdk/go/examples/worker/main.go
Normal file
@@ -0,0 +1,133 @@
|
||||
// worker 演示自动 Worker：声明处理器，Run 管理连接/心跳/续租/恢复，取消后排空。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
func run() error {
|
||||
token, queue := os.Getenv("ORBIT_TOKEN"), os.Getenv("ORBIT_QUEUE_ID")
|
||||
if path := os.Getenv("ORBIT_CREDENTIALS_FILE"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read private credentials: %w", err)
|
||||
}
|
||||
var credentials struct {
|
||||
Token string `json:"token"`
|
||||
QueueID string `json:"queue_id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &credentials) != nil {
|
||||
return errors.New("invalid credentials file")
|
||||
}
|
||||
if token == "" {
|
||||
token = credentials.Token
|
||||
}
|
||||
if queue == "" {
|
||||
queue = credentials.QueueID
|
||||
}
|
||||
}
|
||||
client, err := orbit.NewClient(os.Getenv("ORBIT_URL"), orbit.Options{Token: token})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operation, err := orbit.NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler := "text.normalize.runtime." + operation
|
||||
w, err := orbit.NewWorker(client, orbit.WorkerOptions{QueueID: queue, Capacity: 2, ShutdownTimeout: 5 * time.Second, Handlers: []orbit.WorkerHandler{{Capability: orbit.WorkerCapability{Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: normalize}}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
// 示例先受理，再启动消费者；生产通常是独立的长驻 Worker 进程。
|
||||
var tasks []orbit.Task
|
||||
for i, text := range []string{" hello orbit ", " reusable worker ", " durable result "} {
|
||||
payload, _ := json.Marshal(struct {
|
||||
Text string `json:"text"`
|
||||
}{text})
|
||||
accepted, err := client.SubmitTask(ctx, fmt.Sprintf("runtime-%s-%d", operation, i), orbit.SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", Payload: payload, TaskTimeoutMS: 60000, Retry: orbit.RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: 10000}, RecoverySafety: orbit.RecoveryIdempotent})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tasks = append(tasks, accepted.Task)
|
||||
}
|
||||
workerCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(workerCtx) }()
|
||||
// 任一调用失败时也等待有界排空，不在 main 返回时静默丢掉执行中的 Worker。
|
||||
waited := false
|
||||
defer func() {
|
||||
stop()
|
||||
if !waited {
|
||||
<-done
|
||||
}
|
||||
}()
|
||||
for _, task := range tasks {
|
||||
for {
|
||||
current, err := client.GetTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.State == orbit.TaskSucceeded {
|
||||
fmt.Printf("task=%s state=%s result=%s\n", current.ID, current.State, current.Result)
|
||||
break
|
||||
}
|
||||
if current.State == orbit.TaskFailed || current.State == orbit.TaskNeedsReview {
|
||||
return fmt.Errorf("task %s requires attention: %s", current.ID, current.State)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
waited = true
|
||||
return fmt.Errorf("Worker stopped before completion: %v", err)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
stop()
|
||||
err = <-done
|
||||
waited = true
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats := w.Stats()
|
||||
if stats.Started != 3 || stats.ReportsConfirmed != 3 || stats.Running != 0 {
|
||||
return errors.New("unexpected Worker completion counts")
|
||||
}
|
||||
fmt.Printf("worker_drained=true started=%d confirmed=%d running=%d\n", stats.Started, stats.ReportsConfirmed, stats.Running)
|
||||
return nil
|
||||
}
|
||||
func normalize(ctx context.Context, execution orbit.WorkerExecution) (json.RawMessage, error) {
|
||||
var in struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal(execution.Task.Payload, &in); err != nil {
|
||||
return nil, &orbit.WorkerFailure{Code: "INVALID_TEXT"}
|
||||
}
|
||||
// 模拟可取消的短工作；真实 I/O 也必须使用传入的 ctx。
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
Text string `json:"text"`
|
||||
}{Text: strings.ToUpper(strings.TrimSpace(in.Text))})
|
||||
}
|
||||
130
sdk/go/worker_handler.go
Normal file
130
sdk/go/worker_handler.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type handlerCompletion struct {
|
||||
attemptID string
|
||||
result WorkerFinish
|
||||
}
|
||||
|
||||
func (w *Worker) invoke(ctx context.Context, handler WorkerHandler, execution WorkerExecution, ref WorkerAttempt, results chan<- handlerCompletion, runDone <-chan struct{}) {
|
||||
out := WorkerFinish{Ref: ref, Outcome: WorkerUnknown, ErrorCode: "HANDLER_ABORTED", Result: json.RawMessage(`{}`)}
|
||||
completed := false
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
out = unknownHandlerResult(ref, "HANDLER_PANIC")
|
||||
} else if !completed {
|
||||
out = unknownHandlerResult(ref, "HANDLER_ABORTED")
|
||||
}
|
||||
w.running.Add(-1)
|
||||
select {
|
||||
case results <- handlerCompletion{attemptID: ref.AttemptID, result: out}:
|
||||
case <-runDone:
|
||||
}
|
||||
}()
|
||||
if ctx.Err() != nil {
|
||||
out = unknownHandlerResult(ref, "LEASE_LOST")
|
||||
completed = true
|
||||
return
|
||||
}
|
||||
w.started.Add(1)
|
||||
result, err := handler.Handle(ctx, execution)
|
||||
if err == nil {
|
||||
result = append(json.RawMessage(nil), result...)
|
||||
if validHandlerResult(result) {
|
||||
out = WorkerFinish{Ref: ref, Outcome: WorkerSucceeded, Result: result}
|
||||
} else {
|
||||
out = unknownHandlerResult(ref, "INVALID_HANDLER_RESULT")
|
||||
}
|
||||
} else {
|
||||
var known *WorkerFailure
|
||||
if errors.As(err, &known) && known != nil && workerName(known.Code, 64) {
|
||||
details := append(json.RawMessage(nil), known.Details...)
|
||||
if len(details) == 0 {
|
||||
details = json.RawMessage(`{}`)
|
||||
}
|
||||
if validHandlerResult(details) {
|
||||
out = WorkerFinish{Ref: ref, Outcome: WorkerFailed, ErrorCode: known.Code, Result: details}
|
||||
} else {
|
||||
out = unknownHandlerResult(ref, "INVALID_HANDLER_RESULT")
|
||||
}
|
||||
} else if errors.Is(context.Cause(ctx), ErrExecutionLeaseLost) {
|
||||
out = unknownHandlerResult(ref, "LEASE_LOST")
|
||||
} else {
|
||||
out = unknownHandlerResult(ref, "HANDLER_ERROR")
|
||||
}
|
||||
}
|
||||
completed = true
|
||||
}
|
||||
func unknownHandlerResult(ref WorkerAttempt, code string) WorkerFinish {
|
||||
return WorkerFinish{Ref: ref, Outcome: WorkerUnknown, ErrorCode: code, Result: json.RawMessage(`{}`)}
|
||||
}
|
||||
|
||||
// validHandlerResult 提前拒绝服务端无法持久化的结果，避免失效报告无限重报。
|
||||
// 只检验数值范围；不把精确业务数字改写成 float64。
|
||||
func validHandlerResult(raw json.RawMessage) bool {
|
||||
if len(raw) == 0 || len(raw) > 256<<10 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
|
||||
return false
|
||||
}
|
||||
// 预留 task.finish 的信封/body 嵌套，避免合法结果单独校验后在帧校验阶段超深。
|
||||
frame := append([]byte(`{"body":{"result":`), raw...)
|
||||
frame = append(frame, []byte(`}}`)...)
|
||||
if validateWorkerJSON(frame) != nil {
|
||||
return false
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.UseNumber()
|
||||
for {
|
||||
value, err := d.Token()
|
||||
if err == io.EOF {
|
||||
return true
|
||||
}
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
number, ok := value.(json.Number)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
s := string(number)
|
||||
if len(s) > 256 {
|
||||
return false
|
||||
}
|
||||
if _, err := strconv.ParseFloat(s, 64); err != nil {
|
||||
return false
|
||||
}
|
||||
s = strings.TrimPrefix(s, "-")
|
||||
exponent := 0
|
||||
if i := strings.IndexAny(s, "eE"); i >= 0 {
|
||||
var err error
|
||||
exponent, err = strconv.Atoi(s[i+1:])
|
||||
if err != nil || exponent < -308 || exponent > 308 {
|
||||
return false
|
||||
}
|
||||
s = s[:i]
|
||||
}
|
||||
if i := strings.IndexByte(s, '.'); i >= 0 {
|
||||
exponent -= len(s) - i - 1
|
||||
s = s[:i] + s[i+1:]
|
||||
}
|
||||
s = strings.TrimLeft(s, "0")
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
for strings.HasSuffix(s, "0") {
|
||||
s = s[:len(s)-1]
|
||||
exponent++
|
||||
}
|
||||
if exponent < -308 || exponent > 308 || exponent+len(s)-1 > 308 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,17 +49,24 @@ func integrationWorkerClient(t *testing.T) (*Client, string, string) {
|
||||
// 只丢响应，不伪造任务状态，测试凭据不进入 URL 或输出。
|
||||
type workerFaultGateway struct {
|
||||
drop atomic.Value
|
||||
offline atomic.Bool
|
||||
mu sync.Mutex
|
||||
active map[*websocket.Conn]struct{}
|
||||
dropped chan string
|
||||
server *httptest.Server
|
||||
}
|
||||
|
||||
func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
t.Helper()
|
||||
g := &workerFaultGateway{dropped: make(chan string, 4)}
|
||||
g := &workerFaultGateway{dropped: make(chan string, 4), active: make(map[*websocket.Conn]struct{})}
|
||||
g.drop.Store("")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var handlers sync.WaitGroup
|
||||
g.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if g.offline.Load() {
|
||||
http.Error(w, "gateway offline", 503)
|
||||
return
|
||||
}
|
||||
handlers.Add(1)
|
||||
defer handlers.Done()
|
||||
bridge, stop := context.WithCancel(ctx)
|
||||
@@ -78,6 +85,13 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
return
|
||||
}
|
||||
defer down.CloseNow()
|
||||
g.mu.Lock()
|
||||
g.active[down] = struct{}{}
|
||||
g.mu.Unlock()
|
||||
defer func() { g.mu.Lock(); delete(g.active, down); g.mu.Unlock() }()
|
||||
if g.offline.Load() {
|
||||
return
|
||||
}
|
||||
down.SetReadLimit(512 << 10)
|
||||
var mu sync.Mutex
|
||||
var forwarding sync.WaitGroup
|
||||
@@ -141,6 +155,17 @@ func newWorkerFaultGateway(t *testing.T, upstream string) *workerFaultGateway {
|
||||
t.Cleanup(func() { cancel(); g.server.Close(); handlers.Wait() })
|
||||
return g
|
||||
}
|
||||
func (g *workerFaultGateway) disconnectActive() {
|
||||
g.mu.Lock()
|
||||
active := make([]*websocket.Conn, 0, len(g.active))
|
||||
for c := range g.active {
|
||||
active = append(active, c)
|
||||
}
|
||||
g.mu.Unlock()
|
||||
for _, c := range active {
|
||||
c.CloseNow()
|
||||
}
|
||||
}
|
||||
func (g *workerFaultGateway) expectDrop(t *testing.T, operation string) {
|
||||
t.Helper()
|
||||
select {
|
||||
|
||||
76
sdk/go/worker_lease_guard.go
Normal file
76
sdk/go/worker_lease_guard.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// executionGuard 独立于网络控制取消处理器，已到期/取消的上下文永不恢复。
|
||||
type executionGuard struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelCauseFunc
|
||||
mu sync.Mutex
|
||||
deadline time.Time
|
||||
wake chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newExecutionGuard(parent context.Context, lease WorkerLease) (*executionGuard, bool) {
|
||||
deadline, ok := lease.ExecutionDeadline()
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
g := &executionGuard{ctx: ctx, cancel: cancel, deadline: deadline, wake: make(chan struct{}, 1), done: make(chan struct{})}
|
||||
go g.watch()
|
||||
return g, true
|
||||
}
|
||||
func (g *executionGuard) watch() {
|
||||
defer close(g.done)
|
||||
timer := time.NewTimer(time.Until(g.until()))
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-g.ctx.Done():
|
||||
return
|
||||
case <-g.wake:
|
||||
case <-timer.C:
|
||||
}
|
||||
g.mu.Lock()
|
||||
remaining := time.Until(g.deadline)
|
||||
if remaining <= 0 {
|
||||
g.cancel(ErrExecutionLeaseLost)
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
timer.Reset(remaining)
|
||||
g.mu.Unlock()
|
||||
}
|
||||
}
|
||||
func (g *executionGuard) update(lease WorkerLease) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.ctx.Err() != nil || !time.Now().Before(g.deadline) {
|
||||
g.cancel(ErrExecutionLeaseLost)
|
||||
return false
|
||||
}
|
||||
deadline, ok := lease.ExecutionDeadline()
|
||||
if !lease.Active || lease.State != "RUNNING" {
|
||||
g.cancel(ErrExecutionLeaseLost)
|
||||
return false
|
||||
}
|
||||
// Lease 是新鲜核验，不是历史 Receipt。新会话可能缩短有效窗口，必须遵守。
|
||||
if !ok {
|
||||
g.cancel(ErrExecutionLeaseLost)
|
||||
return false
|
||||
}
|
||||
g.deadline = deadline
|
||||
select {
|
||||
case g.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return true
|
||||
}
|
||||
func (g *executionGuard) until() time.Time { g.mu.Lock(); defer g.mu.Unlock(); return g.deadline }
|
||||
func (g *executionGuard) stop(cause error) { g.cancel(cause); <-g.done }
|
||||
543
sdk/go/worker_runtime.go
Normal file
543
sdk/go/worker_runtime.go
Normal file
@@ -0,0 +1,543 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type runtimeAttempt struct {
|
||||
assignment WorkerAssignment
|
||||
startID, renewID, finishID string
|
||||
guard *executionGuard
|
||||
invoked, returned, lost bool
|
||||
result WorkerFinish
|
||||
nextTry, nextRenew time.Time
|
||||
retryDelay time.Duration
|
||||
}
|
||||
type runtimeCommand struct {
|
||||
kind string
|
||||
id string
|
||||
job *runtimeAttempt
|
||||
capacity WorkerCapacity
|
||||
conn *WorkerConnection
|
||||
}
|
||||
type runtimeReply struct {
|
||||
command *runtimeCommand
|
||||
result WorkerCommandResult
|
||||
welcome WorkerWelcome
|
||||
err error
|
||||
}
|
||||
type runtimeConnected struct {
|
||||
conn *WorkerConnection
|
||||
leases map[string]WorkerLease
|
||||
err error
|
||||
}
|
||||
|
||||
type workerRuntime struct {
|
||||
worker *Worker
|
||||
ctx context.Context
|
||||
jobs map[string]*runtimeAttempt
|
||||
order []string
|
||||
cursor int
|
||||
conn *WorkerConnection
|
||||
command *runtimeCommand
|
||||
connecting bool
|
||||
connectedOnce bool
|
||||
draining, drainConfirmed bool
|
||||
capacity uint32
|
||||
sequence uint64
|
||||
heartbeat, dialAt time.Time
|
||||
backoff time.Duration
|
||||
results chan handlerCompletion
|
||||
replies chan runtimeReply
|
||||
connections chan runtimeConnected
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// Run 持续消费直到 ctx 取消或不可恢复的认证/协议错误。取消后先排空并在
|
||||
// ShutdownTimeout 内收集在途结果；正常收尾返回 nil，超时返回 *WorkerRunError。
|
||||
// 已到期的处理器上下文不会因重连复活；每个已知尝试在本进程最多调用一次处理器。
|
||||
func (w *Worker) Run(parent context.Context) error {
|
||||
if !w.used.CompareAndSwap(false, true) {
|
||||
return ErrWorkerAlreadyRun
|
||||
}
|
||||
defer close(w.events)
|
||||
if parent.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.WithoutCancel(parent))
|
||||
var shutdownExpired atomic.Bool
|
||||
watchDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(watchDone)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-parent.Done():
|
||||
}
|
||||
timer := time.NewTimer(w.options.ShutdownTimeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
shutdownExpired.Store(true)
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
r := &workerRuntime{worker: w, ctx: ctx, jobs: make(map[string]*runtimeAttempt), backoff: w.options.ReconnectMinDelay, results: make(chan handlerCompletion, w.options.Capacity), replies: make(chan runtimeReply, 2), connections: make(chan runtimeConnected, 1)}
|
||||
defer func() {
|
||||
cancel()
|
||||
if r.conn != nil {
|
||||
r.conn.Close()
|
||||
}
|
||||
for _, j := range r.jobs {
|
||||
if j.guard != nil {
|
||||
j.guard.stop(context.Canceled)
|
||||
}
|
||||
}
|
||||
r.wg.Wait()
|
||||
// 连接可能在退出与交付结果之间成功，仍需关闭；不留下新会话连接。
|
||||
select {
|
||||
case c := <-r.connections:
|
||||
if c.conn != nil {
|
||||
c.conn.Close()
|
||||
}
|
||||
default:
|
||||
}
|
||||
<-watchDone
|
||||
}()
|
||||
tick := time.NewTicker(20 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
stop := parent.Done()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
cause := ctx.Err()
|
||||
if shutdownExpired.Load() {
|
||||
cause = ErrWorkerShutdownTimeout
|
||||
}
|
||||
return r.failure(cause)
|
||||
}
|
||||
if parent.Err() != nil && !r.draining {
|
||||
r.draining = true
|
||||
stop = nil
|
||||
w.emit(WorkerEvent{Type: "draining"})
|
||||
}
|
||||
r.observeExpired()
|
||||
if r.conn != nil && r.conn.Err() != nil {
|
||||
if err := r.disconnect(r.conn.Err()); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
// drain ACK 之前的派发已被底层读取器排进队列，收齐后才能宣布排空。
|
||||
if r.conn != nil {
|
||||
for len(r.conn.assignments) > 0 {
|
||||
if err := r.accept(<-r.conn.assignments); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.draining && len(r.jobs) == 0 && (r.drainConfirmed || !r.connectedOnce && !r.connecting) {
|
||||
return nil
|
||||
}
|
||||
if r.conn == nil && !r.connecting && !time.Now().Before(r.dialAt) {
|
||||
r.connect()
|
||||
}
|
||||
if r.conn != nil && r.command == nil {
|
||||
if err := r.dispatch(); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
var assignments <-chan WorkerAssignment
|
||||
var closed <-chan struct{}
|
||||
if r.conn != nil {
|
||||
assignments = r.conn.assignments
|
||||
closed = r.conn.Done()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-stop:
|
||||
stop = nil
|
||||
case <-tick.C:
|
||||
case <-closed:
|
||||
case a := <-assignments:
|
||||
if r.conn.Err() == nil {
|
||||
if err := r.accept(a); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
case done := <-r.results:
|
||||
if j := r.jobs[done.attemptID]; j != nil {
|
||||
j.returned = true
|
||||
j.result = done.result
|
||||
j.nextTry = time.Time{}
|
||||
}
|
||||
case reply := <-r.replies:
|
||||
if err := r.receive(reply); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
case connected := <-r.connections:
|
||||
if err := r.establish(connected); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *workerRuntime) failure(err error) error {
|
||||
return &WorkerRunError{Cause: err, Unconfirmed: len(r.jobs), Running: r.worker.running.Load()}
|
||||
}
|
||||
func (r *workerRuntime) event(j *runtimeAttempt, kind, code string) {
|
||||
r.worker.emit(WorkerEvent{Type: kind, TaskID: j.assignment.Task.ID, AttemptID: j.assignment.AttemptID, Code: code})
|
||||
}
|
||||
func (r *workerRuntime) accept(a WorkerAssignment) error {
|
||||
if j := r.jobs[a.AttemptID]; j != nil {
|
||||
if j.assignment.Attempt() != a.Attempt() {
|
||||
return ErrWorkerProtocol
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// 运行中失去授权但不退出的处理器仍占物理槽；允许接收一轮已在网络中的预占，
|
||||
// 不突破处理器容量，也不让待核对身份无限累积。
|
||||
if len(r.jobs) >= 2*int(r.worker.options.Capacity) {
|
||||
return ErrWorkerBackpressure
|
||||
}
|
||||
id, err := NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.jobs[a.AttemptID] = &runtimeAttempt{assignment: a, startID: id}
|
||||
r.order = append(r.order, a.AttemptID)
|
||||
return nil
|
||||
}
|
||||
func (r *workerRuntime) remove(j *runtimeAttempt) {
|
||||
if j.guard != nil {
|
||||
j.guard.stop(context.Canceled)
|
||||
}
|
||||
delete(r.jobs, j.assignment.AttemptID)
|
||||
for i, id := range r.order {
|
||||
if id == j.assignment.AttemptID {
|
||||
r.order = append(r.order[:i], r.order[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(r.order) > 0 {
|
||||
r.cursor %= len(r.order)
|
||||
} else {
|
||||
r.cursor = 0
|
||||
}
|
||||
}
|
||||
func (r *workerRuntime) observeExpired() {
|
||||
for _, j := range r.jobs {
|
||||
if j.guard != nil && j.guard.ctx.Err() != nil && !j.lost {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
}
|
||||
}
|
||||
func (r *workerRuntime) desiredCapacity() uint32 {
|
||||
if r.draining {
|
||||
return 0
|
||||
}
|
||||
ghosts := uint32(0)
|
||||
for _, j := range r.jobs {
|
||||
if j.returned || j.lost {
|
||||
ghosts++
|
||||
}
|
||||
}
|
||||
if ghosts >= r.worker.options.Capacity {
|
||||
return 0
|
||||
}
|
||||
return r.worker.options.Capacity - ghosts
|
||||
}
|
||||
|
||||
func (r *workerRuntime) connect() {
|
||||
r.connecting = true
|
||||
if r.connectedOnce {
|
||||
r.worker.reconnects.Add(1)
|
||||
r.worker.emit(WorkerEvent{Type: "reconnecting"})
|
||||
}
|
||||
refs := make([]WorkerAttempt, 0, len(r.jobs))
|
||||
for _, j := range r.jobs {
|
||||
refs = append(refs, j.assignment.Attempt())
|
||||
}
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
ctx, stop := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
defer stop()
|
||||
c, err := r.worker.client.DialWorker(ctx, r.worker.registration)
|
||||
result := runtimeConnected{conn: c, err: err, leases: make(map[string]WorkerLease, len(refs))}
|
||||
if err == nil {
|
||||
for _, ref := range refs {
|
||||
id, e := NewWorkerMessageID()
|
||||
if e != nil {
|
||||
result.err = e
|
||||
break
|
||||
}
|
||||
lease, e := c.Resume(ctx, id, ref)
|
||||
if e != nil {
|
||||
result.err = e
|
||||
break
|
||||
}
|
||||
result.leases[ref.AttemptID] = lease
|
||||
}
|
||||
}
|
||||
if result.err != nil && c != nil {
|
||||
c.Close()
|
||||
result.conn = nil
|
||||
}
|
||||
select {
|
||||
case r.connections <- result:
|
||||
case <-r.ctx.Done():
|
||||
if result.conn != nil {
|
||||
result.conn.Close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
func (r *workerRuntime) establish(c runtimeConnected) error {
|
||||
r.connecting = false
|
||||
if c.err != nil {
|
||||
if !workerRetryable(c.err) {
|
||||
return c.err
|
||||
}
|
||||
r.deferReconnect()
|
||||
return nil
|
||||
}
|
||||
r.conn = c.conn
|
||||
r.connectedOnce = true
|
||||
r.sequence = 0
|
||||
r.heartbeat = time.Time{}
|
||||
r.capacity = ^uint32(0)
|
||||
r.drainConfirmed = false
|
||||
r.draining = r.draining || c.conn.Welcome().Draining
|
||||
for id, lease := range c.leases {
|
||||
j := r.jobs[id]
|
||||
if j == nil {
|
||||
continue
|
||||
}
|
||||
if j.guard != nil {
|
||||
if !j.guard.update(lease) {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
j.nextRenew = time.Time{}
|
||||
}
|
||||
if !j.invoked && !lease.Active {
|
||||
r.event(j, "lease_lost", "NOT_STARTED")
|
||||
r.remove(j)
|
||||
}
|
||||
}
|
||||
r.worker.emit(WorkerEvent{Type: "connected"})
|
||||
return nil
|
||||
}
|
||||
func (r *workerRuntime) deferReconnect() {
|
||||
min := r.worker.options.ReconnectMinDelay
|
||||
r.dialAt = time.Now().Add(min + time.Duration(rand.Int64N(int64(r.backoff-min)+1)))
|
||||
r.backoff = minDuration(r.backoff*2, r.worker.options.ReconnectMaxDelay)
|
||||
}
|
||||
func (r *workerRuntime) disconnect(err error) error {
|
||||
if r.conn != nil {
|
||||
r.conn.Close()
|
||||
r.conn = nil
|
||||
}
|
||||
r.command = nil
|
||||
if !workerRetryable(err) {
|
||||
return err
|
||||
}
|
||||
r.deferReconnect()
|
||||
return nil
|
||||
}
|
||||
func workerRetryable(err error) bool {
|
||||
if errors.Is(err, ErrWorkerDisconnected) || errors.Is(err, ErrWorkerClosed) || errors.Is(err, ErrWorkerBackpressure) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var control *WorkerError
|
||||
if errors.As(err, &control) {
|
||||
return control.Code == "STALE_SESSION" || control.Code == "UNAVAILABLE"
|
||||
}
|
||||
var httpErr *HTTPError
|
||||
return errors.As(err, &httpErr) && (httpErr.StatusCode == 429 || httpErr.StatusCode >= 500)
|
||||
}
|
||||
func minDuration(a, b time.Duration) time.Duration {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (r *workerRuntime) dispatch() error {
|
||||
now := time.Now()
|
||||
capacity := r.desiredCapacity()
|
||||
if !now.Before(r.heartbeat) || capacity != r.capacity || r.draining && !r.drainConfirmed {
|
||||
id, err := NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.sequence++
|
||||
r.launch(&runtimeCommand{kind: "capacity", id: id, capacity: WorkerCapacity{Sequence: r.sequence, Capacity: capacity, Drain: r.draining}})
|
||||
return nil
|
||||
}
|
||||
// 结果优先释放占用；其后照顾临近续租，再批准新处理器。每类内部轮转。
|
||||
for _, kind := range []string{"finish", "renew", "start"} {
|
||||
for n := 0; n < len(r.order); n++ {
|
||||
i := (r.cursor + n) % len(r.order)
|
||||
j := r.jobs[r.order[i]]
|
||||
if now.Before(j.nextTry) {
|
||||
continue
|
||||
}
|
||||
var id *string
|
||||
switch kind {
|
||||
case "finish":
|
||||
if !j.returned {
|
||||
continue
|
||||
}
|
||||
id = &j.finishID
|
||||
case "renew":
|
||||
if !j.invoked || j.lost || j.returned || now.Before(j.nextRenew) {
|
||||
continue
|
||||
}
|
||||
id = &j.renewID
|
||||
case "start":
|
||||
if j.invoked || r.worker.running.Load() >= int64(r.worker.options.Capacity) {
|
||||
continue
|
||||
}
|
||||
id = &j.startID
|
||||
}
|
||||
if *id == "" {
|
||||
var err error
|
||||
*id, err = NewWorkerMessageID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
r.cursor = (i + 1) % len(r.order)
|
||||
r.launch(&runtimeCommand{kind: kind, id: *id, job: j})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (r *workerRuntime) launch(command *runtimeCommand) {
|
||||
command.conn = r.conn
|
||||
r.command = command
|
||||
// 操作所需值在 goroutine 启动前复制；运行器事件循环独占可变尝试状态。
|
||||
var ref WorkerAttempt
|
||||
var finish WorkerFinish
|
||||
if command.job != nil {
|
||||
ref = command.job.assignment.Attempt()
|
||||
finish = command.job.result
|
||||
}
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
reply := runtimeReply{command: command}
|
||||
switch command.kind {
|
||||
case "capacity":
|
||||
reply.welcome, reply.err = command.conn.Capacity(r.ctx, command.id, command.capacity)
|
||||
case "start":
|
||||
reply.result, reply.err = command.conn.Start(r.ctx, command.id, ref)
|
||||
case "renew":
|
||||
reply.result, reply.err = command.conn.Renew(r.ctx, command.id, ref)
|
||||
case "finish":
|
||||
reply.result, reply.err = command.conn.Finish(r.ctx, command.id, finish)
|
||||
}
|
||||
select {
|
||||
case r.replies <- reply:
|
||||
case <-r.ctx.Done():
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (r *workerRuntime) receive(reply runtimeReply) error {
|
||||
command := reply.command
|
||||
if r.command != command || command.conn != r.conn {
|
||||
return nil
|
||||
}
|
||||
r.command = nil
|
||||
j := command.job
|
||||
if reply.err != nil {
|
||||
var control *WorkerError
|
||||
if j != nil && errors.As(reply.err, &control) {
|
||||
switch control.Code {
|
||||
case "LEASE_LOST":
|
||||
if command.kind == "finish" {
|
||||
return reply.err
|
||||
}
|
||||
if j.guard != nil {
|
||||
j.guard.stop(ErrExecutionLeaseLost)
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", control.Code)
|
||||
} else {
|
||||
r.remove(j)
|
||||
}
|
||||
return nil
|
||||
case "EXECUTION_PAUSED", "UNAVAILABLE":
|
||||
if j.retryDelay == 0 {
|
||||
j.retryDelay = 200 * time.Millisecond
|
||||
} else {
|
||||
j.retryDelay = minDuration(j.retryDelay*2, 5*time.Second)
|
||||
}
|
||||
j.nextTry = time.Now().Add(j.retryDelay)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if workerRetryable(reply.err) {
|
||||
return r.disconnect(reply.err)
|
||||
}
|
||||
return reply.err
|
||||
}
|
||||
r.backoff = r.worker.options.ReconnectMinDelay
|
||||
if command.kind == "capacity" {
|
||||
r.capacity = command.capacity.Capacity
|
||||
r.drainConfirmed = command.capacity.Drain && reply.welcome.Draining
|
||||
r.heartbeat = time.Now().Add(minDuration(5*time.Second, reply.welcome.ExpiresAt.Sub(reply.welcome.ServerTime)/3))
|
||||
return nil
|
||||
}
|
||||
if command.kind == "finish" {
|
||||
if reply.result.Receipt.Disposition == "APPLIED" {
|
||||
r.worker.reports.Add(1)
|
||||
r.event(j, "result_applied", string(reply.result.Receipt.TaskState))
|
||||
} else {
|
||||
r.worker.evidence.Add(1)
|
||||
r.event(j, "result_evidence", string(reply.result.Receipt.TaskState))
|
||||
}
|
||||
r.remove(j)
|
||||
return nil
|
||||
}
|
||||
if command.kind == "start" {
|
||||
guard, ok := newExecutionGuard(r.ctx, reply.result.Lease)
|
||||
if !ok {
|
||||
r.event(j, "lease_lost", "START_NOT_AUTHORIZED")
|
||||
r.remove(j)
|
||||
return nil
|
||||
}
|
||||
j.guard = guard
|
||||
j.invoked = true
|
||||
r.worker.running.Add(1)
|
||||
task := j.assignment.Task
|
||||
task.Payload = append([]byte(nil), task.Payload...)
|
||||
task.Result = append([]byte(nil), task.Result...)
|
||||
execution := WorkerExecution{Task: task, AttemptID: j.assignment.AttemptID, Epoch: j.assignment.Epoch}
|
||||
handler := r.worker.handlers[WorkerCapability{Handler: task.Handler, HandlerVersion: task.HandlerVersion, PayloadVersion: task.PayloadVersion}]
|
||||
r.event(j, "handler_started", "")
|
||||
go r.worker.invoke(guard.ctx, handler, execution, j.assignment.Attempt(), r.results, r.ctx.Done())
|
||||
} else {
|
||||
r.worker.renewals.Add(1)
|
||||
j.renewID = ""
|
||||
if !j.guard.update(reply.result.Lease) {
|
||||
j.lost = true
|
||||
r.event(j, "lease_lost", "LEASE_LOST")
|
||||
}
|
||||
}
|
||||
j.nextTry = time.Time{}
|
||||
j.retryDelay = 0
|
||||
if !j.lost {
|
||||
j.nextRenew = time.Now().Add(max(20*time.Millisecond, time.Until(j.guard.until())/3))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
357
sdk/go/worker_runtime_integration_test.go
Normal file
357
sdk/go/worker_runtime_integration_test.go
Normal file
@@ -0,0 +1,357 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func runtimeTask(t *testing.T, ctx context.Context, c *Client, queue, handler string, timeout uint64) Task {
|
||||
t.Helper()
|
||||
out, err := c.SubmitTask(ctx, "runtime-"+workerTestID(t), SubmitTaskRequest{QueueID: queue, Handler: handler, HandlerVersion: "v1", PayloadVersion: "v1", Payload: json.RawMessage(`{"sequence":9007199254740993}`), TaskTimeoutMS: 120000, Retry: RetryPolicy{MaxAttempts: 1, AttemptTimeoutMS: timeout}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Task
|
||||
}
|
||||
func runtimeHandler(name string, handle func(context.Context, WorkerExecution) (json.RawMessage, error)) WorkerHandler {
|
||||
return WorkerHandler{Capability: WorkerCapability{Handler: name, HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: handle}
|
||||
}
|
||||
func waitRuntimeState(t *testing.T, ctx context.Context, c *Client, id string, state TaskState) Task {
|
||||
t.Helper()
|
||||
for {
|
||||
task, err := c.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.State == state {
|
||||
return task
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("task did not reach %s", state)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerRuntimeACKLossRenewAndDrain(t *testing.T) {
|
||||
api, queue, token := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second)
|
||||
defer cancel()
|
||||
g := newWorkerFaultGateway(t, api.baseURL)
|
||||
client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := "runtime.renew." + workerTestID(t)
|
||||
entered := make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
w, err := NewWorker(client, WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 45 * time.Second, Handlers: []WorkerHandler{runtimeHandler(name, func(work context.Context, e WorkerExecution) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
close(entered)
|
||||
// 必须跨过服务端最初的 30 秒租约；只有自动续租成功才能获得 APPLIED。
|
||||
select {
|
||||
case <-work.Done():
|
||||
return nil, work.Err()
|
||||
case <-time.After(32 * time.Second):
|
||||
return json.RawMessage(`{"sequence":9007199254740993,"done":true}`), nil
|
||||
}
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := runtimeTask(t, ctx, api, queue, name, 90000)
|
||||
g.drop.Store("task.start")
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(runCtx) }()
|
||||
g.expectDrop(t, "task.start")
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("Worker stopped before handler: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("handler did not start")
|
||||
}
|
||||
g.drop.Store("task.finish")
|
||||
stop() // 取消 Run 只进入排空，不能立即取消正在执行的函数。
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Worker did not finish within grace")
|
||||
}
|
||||
g.expectDrop(t, "task.finish")
|
||||
current := waitRuntimeState(t, ctx, api, task.ID, TaskSucceeded)
|
||||
stats := w.Stats()
|
||||
if calls.Load() != 1 || stats.Started != 1 || stats.Running != 0 || stats.ReportsConfirmed != 1 || stats.Reconnects < 2 || stats.RenewalsConfirmed < 2 || current.AttemptCount != 1 || string(current.Result) != `{"sequence":9007199254740993,"done":true}` {
|
||||
t.Fatalf("automatic execution invariant failed: calls=%d stats=%+v attempts=%d", calls.Load(), stats, current.AttemptCount)
|
||||
}
|
||||
// 仍在等待的同类工作不能被已排空的进程领取。
|
||||
waiting := runtimeTask(t, ctx, api, queue, name, 10000)
|
||||
observed, err := api.GetTask(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if observed.State != TaskWaiting {
|
||||
t.Fatal("drained Worker accepted another task")
|
||||
}
|
||||
if _, err := api.CancelTask(ctx, observed.ID, "cleanup-"+workerTestID(t), CancelTaskRequest{ExpectedRevision: observed.Revision, Reason: "finish runtime drain regression"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerRuntimeUnknownAndUncooperativeShutdown(t *testing.T) {
|
||||
api, queue, _ := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
handle func(context.Context, WorkerExecution) (json.RawMessage, error)
|
||||
state TaskState
|
||||
}{
|
||||
{"known", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
return nil, &WorkerFailure{Code: "INVALID_OBJECT"}
|
||||
}, TaskFailed},
|
||||
{"unknown", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
return nil, errors.New("external request timed out with private details")
|
||||
}, TaskNeedsReview},
|
||||
{"panic", func(context.Context, WorkerExecution) (json.RawMessage, error) { panic("private panic") }, TaskNeedsReview},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
name := "runtime." + test.name + "." + workerTestID(t)
|
||||
w, err := NewWorker(api, WorkerOptions{QueueID: queue, Capacity: 1, Handlers: []WorkerHandler{runtimeHandler(name, test.handle)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := runtimeTask(t, ctx, api, queue, name, 5000)
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
defer stop()
|
||||
waitRuntimeState(t, ctx, api, task.ID, test.state)
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("classification Worker did not stop")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("uncooperative", func(t *testing.T) {
|
||||
name := "runtime.uncooperative." + workerTestID(t)
|
||||
entered, release, returned := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
defer close(release)
|
||||
w, err := NewWorker(api, WorkerOptions{QueueID: queue, Capacity: 1, ShutdownTimeout: 250 * time.Millisecond, Handlers: []WorkerHandler{runtimeHandler(name, func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
close(returned)
|
||||
return json.RawMessage(`{}`), nil
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtimeTask(t, ctx, api, queue, name, 1500)
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
defer stop()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("handler did not enter")
|
||||
}
|
||||
began := time.Now()
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
var stopped *WorkerRunError
|
||||
if !errors.As(err, &stopped) || !errors.Is(err, ErrWorkerShutdownTimeout) || stopped.Running != 1 || stopped.Unconfirmed != 1 {
|
||||
t.Fatalf("uncooperative handler hidden: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("shutdown waited forever for handler")
|
||||
}
|
||||
if time.Since(began) > time.Second {
|
||||
t.Fatal("shutdown exceeded bounded grace")
|
||||
}
|
||||
select {
|
||||
case <-returned:
|
||||
t.Fatal("SDK claimed to kill uncooperative handler")
|
||||
default:
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerRuntimeLeaseLossCancelsAndDoesNotRevive(t *testing.T) {
|
||||
api, queue, _ := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
|
||||
defer cancel()
|
||||
name := "runtime.expired." + workerTestID(t)
|
||||
cause := make(chan error, 1)
|
||||
w, err := NewWorker(api, WorkerOptions{QueueID: queue, Capacity: 1, Handlers: []WorkerHandler{runtimeHandler(name, func(work context.Context, _ WorkerExecution) (json.RawMessage, error) {
|
||||
<-work.Done()
|
||||
cause <- context.Cause(work)
|
||||
return nil, work.Err()
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := runtimeTask(t, ctx, api, queue, name, 1400)
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
defer stop()
|
||||
select {
|
||||
case err := <-cause:
|
||||
if !errors.Is(err, ErrExecutionLeaseLost) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("handler did not observe lease cancellation")
|
||||
}
|
||||
waitRuntimeState(t, ctx, api, task.ID, TaskNeedsReview)
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Worker did not stop")
|
||||
}
|
||||
if w.Stats().Started != 1 {
|
||||
t.Fatal("expired handler was invoked again")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerRuntimeOutageCancelsAndReplaysEvidence(t *testing.T) {
|
||||
api, queue, token := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 18*time.Second)
|
||||
defer cancel()
|
||||
g := newWorkerFaultGateway(t, api.baseURL)
|
||||
client, err := NewClient(g.server.URL, Options{Token: token, HTTPClient: g.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := "runtime.outage." + workerTestID(t)
|
||||
entered, cancelled := make(chan struct{}), make(chan error, 1)
|
||||
w, err := NewWorker(client, WorkerOptions{QueueID: queue, Capacity: 1, ReconnectMaxDelay: time.Second, Handlers: []WorkerHandler{runtimeHandler(name, func(work context.Context, _ WorkerExecution) (json.RawMessage, error) {
|
||||
close(entered)
|
||||
<-work.Done()
|
||||
cancelled <- context.Cause(work)
|
||||
return nil, work.Err()
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := runtimeTask(t, ctx, api, queue, name, 2000)
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
defer stop()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("handler did not start")
|
||||
}
|
||||
g.offline.Store(true)
|
||||
g.disconnectActive()
|
||||
select {
|
||||
case cause := <-cancelled:
|
||||
if !errors.Is(cause, ErrExecutionLeaseLost) {
|
||||
t.Fatal(cause)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("disconnected handler ignored local expiry")
|
||||
}
|
||||
waitRuntimeState(t, ctx, api, task.ID, TaskNeedsReview)
|
||||
g.offline.Store(false)
|
||||
for w.Stats().EvidenceConfirmed != 1 {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Worker stopped before evidence replay: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal("pending result was not replayed")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Worker did not stop")
|
||||
}
|
||||
current, err := api.GetTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w.Stats().Started != 1 || current.AttemptCount != 1 || current.State != TaskNeedsReview || len(current.Result) != 0 {
|
||||
t.Fatal("outage revived handler or promoted late evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkerRuntimeConcurrency(t *testing.T) {
|
||||
api, queue, _ := integrationWorkerClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 18*time.Second)
|
||||
defer cancel()
|
||||
name := "runtime.concurrency." + workerTestID(t)
|
||||
var running, peak atomic.Int32
|
||||
w, err := NewWorker(api, WorkerOptions{QueueID: queue, Capacity: 2, Handlers: []WorkerHandler{runtimeHandler(name, func(work context.Context, _ WorkerExecution) (json.RawMessage, error) {
|
||||
value := running.Add(1)
|
||||
defer running.Add(-1)
|
||||
for old := peak.Load(); value > old; old = peak.Load() {
|
||||
if peak.CompareAndSwap(old, value) {
|
||||
break
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-work.Done():
|
||||
return nil, work.Err()
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
return json.RawMessage(`{"done":true}`), nil
|
||||
}
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var tasks []Task
|
||||
for range 6 {
|
||||
tasks = append(tasks, runtimeTask(t, ctx, api, queue, name, 10000))
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(run) }()
|
||||
defer stop()
|
||||
for _, task := range tasks {
|
||||
current := waitRuntimeState(t, ctx, api, task.ID, TaskSucceeded)
|
||||
if current.AttemptCount != 1 {
|
||||
t.Fatal("unexpected new attempt")
|
||||
}
|
||||
}
|
||||
stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Worker did not drain")
|
||||
}
|
||||
if peak.Load() != 2 || running.Load() != 0 || w.Stats().Started != 6 || w.Stats().ReportsConfirmed != 6 {
|
||||
t.Fatalf("concurrency invariant failed: peak=%d stats=%+v", peak.Load(), w.Stats())
|
||||
}
|
||||
}
|
||||
181
sdk/go/worker_runtime_test.go
Normal file
181
sdk/go/worker_runtime_test.go
Normal file
@@ -0,0 +1,181 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func guardLease(duration time.Duration) WorkerLease {
|
||||
return WorkerLease{Active: true, State: "RUNNING", deadline: time.Now().Add(duration)}
|
||||
}
|
||||
func TestExecutionGuardRenewsShortensAndNeverRevives(t *testing.T) {
|
||||
parent, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
g, ok := newExecutionGuard(parent, guardLease(80*time.Millisecond))
|
||||
if !ok {
|
||||
t.Fatal("missing guard")
|
||||
}
|
||||
defer g.stop(context.Canceled)
|
||||
if !g.update(guardLease(500 * time.Millisecond)) {
|
||||
t.Fatal("renew rejected")
|
||||
}
|
||||
select {
|
||||
case <-g.ctx.Done():
|
||||
t.Fatal("old lease cancelled a renewed handler")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
if !g.update(guardLease(35 * time.Millisecond)) {
|
||||
t.Fatal("new short session rejected")
|
||||
}
|
||||
select {
|
||||
case <-g.ctx.Done():
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("shorter session did not cancel handler")
|
||||
}
|
||||
if !errors.Is(context.Cause(g.ctx), ErrExecutionLeaseLost) {
|
||||
t.Fatal(context.Cause(g.ctx))
|
||||
}
|
||||
if g.update(guardLease(time.Minute)) {
|
||||
t.Fatal("expired handler revived")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerOutcomesAndAbort(t *testing.T) {
|
||||
ref := WorkerAttempt{TaskID: workerTestTask, AttemptID: workerTestAttempt, Epoch: 1}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
fn func(context.Context, WorkerExecution) (json.RawMessage, error)
|
||||
out WorkerOutcome
|
||||
code string
|
||||
}{
|
||||
{"success", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"n":9007199254740993}`), nil
|
||||
}, WorkerSucceeded, ""},
|
||||
{"known_failure", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
return nil, &WorkerFailure{Code: "INPUT_REJECTED", Details: json.RawMessage(`{"field":"name"}`)}
|
||||
}, WorkerFailed, "INPUT_REJECTED"},
|
||||
{"ordinary_error", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
return nil, errors.New("private error secret")
|
||||
}, WorkerUnknown, "HANDLER_ERROR"},
|
||||
{"panic", func(context.Context, WorkerExecution) (json.RawMessage, error) { panic("private panic secret") }, WorkerUnknown, "HANDLER_PANIC"},
|
||||
{"goexit", func(context.Context, WorkerExecution) (json.RawMessage, error) { runtime.Goexit(); return nil, nil }, WorkerUnknown, "HANDLER_ABORTED"},
|
||||
{"bad_json", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"n":1,"n":2}`), nil
|
||||
}, WorkerUnknown, "INVALID_HANDLER_RESULT"},
|
||||
{"number_range", func(context.Context, WorkerExecution) (json.RawMessage, error) { return json.RawMessage(`1e999`), nil }, WorkerUnknown, "INVALID_HANDLER_RESULT"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
w := &Worker{}
|
||||
w.running.Add(1)
|
||||
done := make(chan handlerCompletion, 1)
|
||||
go w.invoke(context.Background(), WorkerHandler{Handle: test.fn}, WorkerExecution{}, ref, done, make(chan struct{}))
|
||||
select {
|
||||
case got := <-done:
|
||||
if got.result.Outcome != test.out || got.result.ErrorCode != test.code || strings.Contains(string(got.result.Result), "secret") {
|
||||
t.Fatalf("unexpected classification: %s/%s", got.result.Outcome, got.result.ErrorCode)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("handler lost result")
|
||||
}
|
||||
if w.running.Load() != 0 || w.started.Load() != 1 {
|
||||
t.Fatal("handler counters leaked")
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, raw := range []string{`null`, `0.1e-308`, strings.Repeat("[", 31) + "1" + strings.Repeat("]", 31), strings.Repeat("9", 257)} {
|
||||
if validHandlerResult(json.RawMessage(raw)) {
|
||||
t.Fatal("unsupported result passed runtime validation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerOptionsAndSingleRun(t *testing.T) {
|
||||
c, _ := NewClient("http://127.0.0.1:1", Options{Token: "test-token"})
|
||||
opts := WorkerOptions{QueueID: workerTestQueue, Handlers: []WorkerHandler{{Capability: WorkerCapability{Handler: "test", HandlerVersion: "v1", PayloadVersion: "v1"}, Handle: func(context.Context, WorkerExecution) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }}}}
|
||||
w, err := NewWorker(c, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := w.Run(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Run(ctx); !errors.Is(err, ErrWorkerAlreadyRun) {
|
||||
t.Fatal("Worker identity reused after Run")
|
||||
}
|
||||
if _, open := <-w.Events(); open {
|
||||
t.Fatal("event channel not closed")
|
||||
}
|
||||
for _, invalid := range []WorkerOptions{{QueueID: workerTestQueue}, {QueueID: workerTestQueue, Handlers: append(opts.Handlers, opts.Handlers...)}, {QueueID: workerTestQueue, Handlers: opts.Handlers, ShutdownTimeout: -1}} {
|
||||
if _, err := NewWorker(c, invalid); err == nil {
|
||||
t.Fatal("invalid Worker options accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerShutdownBoundsPendingHandshake(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(entered); <-r.Context().Done() }))
|
||||
defer server.Close()
|
||||
c, _ := NewClient(server.URL, Options{Token: "test-token"})
|
||||
w, err := NewWorker(c, WorkerOptions{QueueID: workerTestQueue, ShutdownTimeout: 50 * time.Millisecond, Handlers: []WorkerHandler{runtimeHandler("test", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
t.Error("unexpected handler")
|
||||
return nil, nil
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(ctx) }()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("handshake did not start")
|
||||
}
|
||||
began := time.Now()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, ErrWorkerShutdownTimeout) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("shutdown blocked on pending handshake")
|
||||
}
|
||||
if time.Since(began) > 500*time.Millisecond {
|
||||
t.Fatal("handshake exceeded shutdown bound")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPermanentAuthenticationFailureStops(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "private upstream response", http.StatusUnauthorized)
|
||||
}))
|
||||
defer server.Close()
|
||||
c, _ := NewClient(server.URL, Options{Token: "test-token"})
|
||||
w, err := NewWorker(c, WorkerOptions{QueueID: workerTestQueue, Handlers: []WorkerHandler{runtimeHandler("test", func(context.Context, WorkerExecution) (json.RawMessage, error) {
|
||||
t.Error("unauthorized handler invoked")
|
||||
return nil, nil
|
||||
})}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
err = w.Run(ctx)
|
||||
var stopped *WorkerRunError
|
||||
var status *HTTPError
|
||||
if !errors.As(err, &stopped) || !errors.As(err, &status) || status.StatusCode != 401 || stopped.Unconfirmed != 0 || strings.Contains(err.Error(), "private") {
|
||||
t.Fatalf("unexpected authentication failure: %v", err)
|
||||
}
|
||||
}
|
||||
152
sdk/go/worker_runtime_types.go
Normal file
152
sdk/go/worker_runtime_types.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package orbit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrExecutionLeaseLost = errors.New("Worker execution lease expired or lost")
|
||||
var ErrWorkerShutdownTimeout = errors.New("Worker shutdown grace period expired")
|
||||
var ErrWorkerAlreadyRun = errors.New("Worker instance may only Run once")
|
||||
|
||||
// WorkerExecution 是交给业务处理器的输入，不包含实例或尝试秘密。
|
||||
type WorkerExecution struct {
|
||||
Task Task // Task 固定任务输入；业务副作用应使用稳定业务身份去重。
|
||||
AttemptID string // AttemptID 当前尝试，用于核查与追踪，不代替业务幂等键。
|
||||
Epoch uint64 // Epoch 执行代次；接入方资源可用它拒绝旧尝试写入。
|
||||
}
|
||||
|
||||
func (e WorkerExecution) String() string {
|
||||
return fmt.Sprintf("WorkerExecution{TaskID:%s AttemptID:%s Epoch:%d data:redacted}", e.Task.ID, e.AttemptID, e.Epoch)
|
||||
}
|
||||
func (e WorkerExecution) GoString() string { return e.String() }
|
||||
|
||||
// WorkerHandler 注册精确版本的业务函数；函数必须遵守 ctx 的取消。
|
||||
// 普通 error/panic 表示效果未知；明确的业务失败返回 *WorkerFailure。
|
||||
type WorkerHandler struct {
|
||||
Capability WorkerCapability // Capability 处理器和输入版本，整个 Worker 生命周期固定。
|
||||
Handle func(context.Context, WorkerExecution) (json.RawMessage, error) // Handle 返回非 null JSON；不得忽略租约取消继续发起副作用。
|
||||
}
|
||||
|
||||
// WorkerFailure 仅用于接入方能够确认的业务失败，不用于网络超时或未知外部效果。
|
||||
type WorkerFailure struct {
|
||||
Code string // Code 1–64 位稳定 ASCII 分类，禁止放原始异常正文。
|
||||
Details json.RawMessage // Details 可选非 null JSON，最多 256 KiB；空值使用空对象。
|
||||
}
|
||||
|
||||
func (e *WorkerFailure) Error() string { return "Worker handler reported a known failure" }
|
||||
|
||||
// WorkerOptions 配置自动执行器。一个实例绑定一个队列，只能 Run 一次。
|
||||
type WorkerOptions struct {
|
||||
QueueID string // QueueID 凭据有消费权限的队列。
|
||||
Capacity uint32 // Capacity 最大并发处理器数，默认 4，范围 1–256。
|
||||
Handlers []WorkerHandler // Handlers 1–32 种精确能力，不可重复，函数不得为空。
|
||||
ShutdownTimeout time.Duration // ShutdownTimeout 收到 Run context 取消后的收尾总时限，默认 30 秒，最多 5 分钟。
|
||||
ReconnectMinDelay time.Duration // ReconnectMinDelay 重连退避起点，默认 200 毫秒，至少 10 毫秒。
|
||||
ReconnectMaxDelay time.Duration // ReconnectMaxDelay 重连退避上限，默认 5 秒，最多 30 秒；采用区间内抖动。
|
||||
}
|
||||
|
||||
// WorkerEvent 是有界、可能丢弃的本地观察通知，不是持久化审计或结果订阅。
|
||||
type WorkerEvent struct {
|
||||
Type string // Type connected、reconnecting、handler_started、lease_lost、result_applied、result_evidence、draining。
|
||||
TaskID string // TaskID 涉及的任务；连接通知为空。
|
||||
AttemptID string // AttemptID 涉及的尝试；连接通知为空。
|
||||
Code string // Code 稳定错误/结果分类，不包含输入、结果或秘密。
|
||||
}
|
||||
|
||||
// WorkerStats 是当前进程的观测值，重启清零，不作为持久化成功率。
|
||||
type WorkerStats struct {
|
||||
Running int64 // Running 尚未返回的处理器数，包含不遵守取消的函数。
|
||||
Started uint64 // Started 本进程实际调用处理器的次数。
|
||||
RenewalsConfirmed uint64 // RenewalsConfirmed 收到当前续租确认的次数，不代表任务成功。
|
||||
ReportsConfirmed uint64 // ReportsConfirmed 收到 APPLIED 的结果报告数，包含 UNKNOWN。
|
||||
EvidenceConfirmed uint64 // EvidenceConfirmed 收到 EVIDENCE_ONLY 的报告数。
|
||||
Reconnects uint64 // Reconnects 首次连接之后发起重连的次数。
|
||||
DroppedEvents uint64 // DroppedEvents 本地通知缓冲满时丢弃的条数。
|
||||
}
|
||||
|
||||
// WorkerRunError 表示执行器未能正常收尾；不能据此认为外部效果已撤销。
|
||||
type WorkerRunError struct {
|
||||
Cause error // Cause 连接/权限/协议错误或收尾超时，支持 errors.Is/As。
|
||||
Unconfirmed int // Unconfirmed 尚未完成控制核对的已知尝试数。
|
||||
Running int64 // Running 返回时仍未退出的处理器数；Go 无法强制杀死 goroutine。
|
||||
}
|
||||
|
||||
func (e *WorkerRunError) Error() string {
|
||||
return fmt.Sprintf("Worker stopped with %d unconfirmed attempts and %d running handlers: %v", e.Unconfirmed, e.Running, e.Cause)
|
||||
}
|
||||
func (e *WorkerRunError) Unwrap() error { return e.Cause }
|
||||
|
||||
// Worker 自动管理注册、心跳、续租、重连和有界排空；任务权威状态仍在服务端。
|
||||
type Worker struct {
|
||||
client *Client
|
||||
options WorkerOptions
|
||||
registration WorkerRegistration
|
||||
handlers map[WorkerCapability]WorkerHandler
|
||||
used atomic.Bool
|
||||
events chan WorkerEvent
|
||||
running atomic.Int64
|
||||
started atomic.Uint64
|
||||
renewals atomic.Uint64
|
||||
reports atomic.Uint64
|
||||
evidence atomic.Uint64
|
||||
reconnects atomic.Uint64
|
||||
dropped atomic.Uint64
|
||||
}
|
||||
|
||||
func NewWorker(client *Client, options WorkerOptions) (*Worker, error) {
|
||||
if client == nil || client.token == "" {
|
||||
return nil, errors.New("authenticated Orbit client is required")
|
||||
}
|
||||
if options.Capacity == 0 {
|
||||
options.Capacity = 4
|
||||
}
|
||||
if options.ShutdownTimeout == 0 {
|
||||
options.ShutdownTimeout = 30 * time.Second
|
||||
}
|
||||
if options.ReconnectMinDelay == 0 {
|
||||
options.ReconnectMinDelay = 200 * time.Millisecond
|
||||
}
|
||||
if options.ReconnectMaxDelay == 0 {
|
||||
options.ReconnectMaxDelay = 5 * time.Second
|
||||
}
|
||||
if options.ShutdownTimeout < time.Millisecond || options.ShutdownTimeout > 5*time.Minute || options.ReconnectMinDelay < 10*time.Millisecond || options.ReconnectMaxDelay < options.ReconnectMinDelay || options.ReconnectMaxDelay > 30*time.Second {
|
||||
return nil, errors.New("invalid Worker shutdown or reconnect duration")
|
||||
}
|
||||
capabilities := make([]WorkerCapability, 0, len(options.Handlers))
|
||||
handlers := make(map[WorkerCapability]WorkerHandler, len(options.Handlers))
|
||||
for _, h := range options.Handlers {
|
||||
if h.Handle == nil {
|
||||
return nil, errors.New("Worker handler function is required")
|
||||
}
|
||||
capabilities = append(capabilities, h.Capability)
|
||||
handlers[h.Capability] = h
|
||||
}
|
||||
r, err := NewWorkerRegistration(options.QueueID, options.Capacity, capabilities)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options.Handlers = append([]WorkerHandler(nil), options.Handlers...)
|
||||
return &Worker{client: client, options: options, registration: r, handlers: handlers, events: make(chan WorkerEvent, 128)}, nil
|
||||
}
|
||||
|
||||
// Events 返回只读通知流；无需读取也不会阻塞执行，Run 返回时关闭。
|
||||
func (w *Worker) Events() <-chan WorkerEvent { return w.events }
|
||||
func (w *Worker) Stats() WorkerStats {
|
||||
return WorkerStats{Running: w.running.Load(), Started: w.started.Load(), RenewalsConfirmed: w.renewals.Load(), ReportsConfirmed: w.reports.Load(), EvidenceConfirmed: w.evidence.Load(), Reconnects: w.reconnects.Load(), DroppedEvents: w.dropped.Load()}
|
||||
}
|
||||
func (w *Worker) emit(event WorkerEvent) {
|
||||
select {
|
||||
case w.events <- event:
|
||||
default:
|
||||
w.dropped.Add(1)
|
||||
}
|
||||
}
|
||||
func (w *Worker) String() string {
|
||||
return fmt.Sprintf("Worker{QueueID:%s Capacity:%d secrets:redacted}", w.options.QueueID, w.options.Capacity)
|
||||
}
|
||||
func (w *Worker) GoString() string { return w.String() }
|
||||
Reference in New Issue
Block a user