feat: add independent platform modules and multi-provider storage
This commit is contained in:
129
resilience/breaker.go
Normal file
129
resilience/breaker.go
Normal file
@@ -0,0 +1,129 @@
|
||||
// Package resilience provides per-dependency circuit breaking and concurrency isolation.
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrOpen = errors.New("resilience: circuit open")
|
||||
var ErrBusy = errors.New("resilience: concurrency limit reached")
|
||||
|
||||
type Config struct {
|
||||
FailureThreshold int `json:"failureThreshold"`
|
||||
MaxConcurrent int `json:"maxConcurrent"`
|
||||
OpenTimeout time.Duration `json:"openTimeout"`
|
||||
IsFailure func(error) bool `json:"-"`
|
||||
}
|
||||
type Breaker struct {
|
||||
mu sync.Mutex
|
||||
config Config
|
||||
active, failures int
|
||||
state string
|
||||
until time.Time
|
||||
generation uint64
|
||||
now func() time.Time
|
||||
}
|
||||
type Snapshot struct {
|
||||
State string `json:"state"`
|
||||
Active int `json:"active"`
|
||||
Failures int `json:"failures"`
|
||||
}
|
||||
|
||||
func New(cfg Config) (*Breaker, error) {
|
||||
if cfg.FailureThreshold < 0 || cfg.MaxConcurrent < 0 || cfg.OpenTimeout < 0 {
|
||||
return nil, errors.New("resilience: invalid configuration")
|
||||
}
|
||||
if cfg.FailureThreshold == 0 {
|
||||
cfg.FailureThreshold = 5
|
||||
}
|
||||
if cfg.MaxConcurrent == 0 {
|
||||
cfg.MaxConcurrent = 32
|
||||
}
|
||||
if cfg.OpenTimeout == 0 {
|
||||
cfg.OpenTimeout = 5 * time.Second
|
||||
}
|
||||
if cfg.IsFailure == nil {
|
||||
cfg.IsFailure = func(err error) bool { return err != nil && !errors.Is(err, context.Canceled) }
|
||||
}
|
||||
return &Breaker{config: cfg, state: "closed", now: time.Now}, nil
|
||||
}
|
||||
|
||||
// Acquire returns a completion callback; it must be called exactly once.
|
||||
// A half-open circuit admits one probe. Old-generation results cannot close it.
|
||||
func (b *Breaker) Acquire(ctx context.Context) (func(bool), error) {
|
||||
if ctx == nil {
|
||||
return nil, errors.New("resilience: nil context")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.state == "open" {
|
||||
if b.now().Before(b.until) {
|
||||
return nil, ErrOpen
|
||||
}
|
||||
b.state = "halfOpen"
|
||||
b.generation++
|
||||
}
|
||||
if b.state == "halfOpen" && b.active > 0 {
|
||||
return nil, ErrOpen
|
||||
}
|
||||
if b.active >= b.config.MaxConcurrent {
|
||||
return nil, ErrBusy
|
||||
}
|
||||
b.active++
|
||||
generation := b.generation
|
||||
var once sync.Once
|
||||
return func(success bool) {
|
||||
once.Do(func() {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.active--
|
||||
if generation != b.generation {
|
||||
return
|
||||
}
|
||||
if success {
|
||||
b.failures = 0
|
||||
if b.state == "halfOpen" {
|
||||
b.state = "closed"
|
||||
b.generation++
|
||||
}
|
||||
return
|
||||
}
|
||||
b.failures++
|
||||
if b.state == "halfOpen" || b.failures >= b.config.FailureThreshold {
|
||||
b.state = "open"
|
||||
b.until = b.now().Add(b.config.OpenTimeout)
|
||||
b.generation++
|
||||
}
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
func (b *Breaker) Do(ctx context.Context, work func(context.Context) error) (err error) {
|
||||
if work == nil {
|
||||
return errors.New("resilience: nil work")
|
||||
}
|
||||
done, err := b.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
complete := false
|
||||
defer func() {
|
||||
if !complete {
|
||||
done(false)
|
||||
}
|
||||
}()
|
||||
err = work(ctx)
|
||||
done(!b.config.IsFailure(err))
|
||||
complete = true
|
||||
return err
|
||||
}
|
||||
func (b *Breaker) Snapshot() Snapshot {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return Snapshot{State: b.state, Active: b.active, Failures: b.failures}
|
||||
}
|
||||
75
resilience/breaker_test.go
Normal file
75
resilience/breaker_test.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
func TestBreakerProbeAndLateResult(t *testing.T) {
|
||||
b, _ := New(Config{FailureThreshold: 1, MaxConcurrent: 2, OpenTimeout: time.Second})
|
||||
now := time.Now()
|
||||
b.now = func() time.Time { return now }
|
||||
ctx := context.Background()
|
||||
first, _ := b.Acquire(ctx)
|
||||
late, _ := b.Acquire(ctx)
|
||||
first(false)
|
||||
late(true)
|
||||
if b.Snapshot().State != "open" {
|
||||
t.Fatal(b.Snapshot())
|
||||
}
|
||||
if _, err := b.Acquire(ctx); !errors.Is(err, ErrOpen) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(time.Second)
|
||||
probe, err := b.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = b.Acquire(ctx); !errors.Is(err, ErrOpen) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe(true)
|
||||
probe(false)
|
||||
if s := b.Snapshot(); s.State != "closed" || s.Active != 0 {
|
||||
t.Fatal(s)
|
||||
}
|
||||
}
|
||||
func TestTransportHoldsSlotAndReleasesPanic(t *testing.T) {
|
||||
b, _ := New(Config{MaxConcurrent: 1})
|
||||
tr := Transport{Breaker: b, Base: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader("hello"))}, nil
|
||||
})}
|
||||
req, _ := http.NewRequest("GET", "http://example.test", nil)
|
||||
r, err := tr.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = tr.RoundTrip(req); !errors.Is(err, ErrBusy) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
if b.Snapshot().Active != 0 {
|
||||
t.Fatal(b.Snapshot())
|
||||
}
|
||||
tr.Base = roundTripFunc(func(*http.Request) (*http.Response, error) { panic("fixture") })
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Error("panic lost")
|
||||
}
|
||||
}()
|
||||
tr.RoundTrip(req)
|
||||
}()
|
||||
if b.Snapshot().Active != 0 {
|
||||
t.Fatal(b.Snapshot())
|
||||
}
|
||||
}
|
||||
67
resilience/http.go
Normal file
67
resilience/http.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Transport isolates one logical upstream. Create separate Breakers for
|
||||
// different dependencies. Slots remain occupied until body Close/EOF.
|
||||
type Transport struct {
|
||||
Base http.RoundTripper
|
||||
Breaker *Breaker
|
||||
}
|
||||
|
||||
func (t Transport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
if t.Breaker == nil {
|
||||
return nil, errors.New("resilience: breaker required")
|
||||
}
|
||||
done, err := t.Breaker.Acquire(r.Context())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if v := recover(); v != nil {
|
||||
done(false)
|
||||
panic(v)
|
||||
}
|
||||
}()
|
||||
base := t.Base
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
response, err := base.RoundTrip(r)
|
||||
if err != nil {
|
||||
done(!t.Breaker.config.IsFailure(err))
|
||||
if response != nil && response.Body != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if response == nil || response.Body == nil {
|
||||
done(false)
|
||||
return nil, errors.New("resilience: invalid transport response")
|
||||
}
|
||||
response.Body = &breakerBody{ReadCloser: response.Body, done: done, success: response.StatusCode < 500}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
type breakerBody struct {
|
||||
io.ReadCloser
|
||||
done func(bool)
|
||||
success bool
|
||||
}
|
||||
|
||||
func (b *breakerBody) Read(p []byte) (int, error) {
|
||||
n, err := b.ReadCloser.Read(p)
|
||||
if err != nil {
|
||||
b.done(b.success && err == io.EOF)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
func (b *breakerBody) Close() error {
|
||||
err := b.ReadCloser.Close()
|
||||
b.done(b.success && err == nil)
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user