252 lines
7.3 KiB
Go
252 lines
7.3 KiB
Go
package main
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"net/http/httputil"
|
||
"net/url"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"runtime"
|
||
"strings"
|
||
"sync/atomic"
|
||
"syscall"
|
||
"testing"
|
||
"time"
|
||
|
||
outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql"
|
||
orbit "git.yuntuo.hk/k2safe/orbit/sdk/go"
|
||
"github.com/go-sql-driver/mysql"
|
||
)
|
||
|
||
func TestIntegrationHTTPServiceRestartAndResults(t *testing.T) {
|
||
dsn, base, path := os.Getenv("ORBIT_TEST_HTTP_DSN"), os.Getenv("ORBIT_TEST_HTTP_URL"), os.Getenv("ORBIT_TEST_HTTP_CREDENTIALS_FILE")
|
||
if dsn == "" || base == "" || path == "" {
|
||
t.Skip("requires disposable business database and Orbit process")
|
||
}
|
||
cfg, err := mysql.ParseDSN(dsn)
|
||
if err != nil || cfg.DBName != "orbit_http_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") || !strings.HasPrefix(base, "http://127.0.0.1:") {
|
||
t.Fatal("requires isolated loopback environment")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second)
|
||
defer cancel()
|
||
db, err := outbox.Open(ctx, dsn)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer db.Close()
|
||
binary := filepath.Join(t.TempDir(), "server")
|
||
build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-race", "-o", binary, ".")
|
||
if raw, err := build.CombinedOutput(); err != nil {
|
||
t.Fatalf("build failed %v %s", err, raw)
|
||
}
|
||
env := []string{}
|
||
for _, v := range os.Environ() {
|
||
if !strings.HasPrefix(v, "ORBIT_") && !strings.HasPrefix(v, "APP_") {
|
||
env = append(env, v)
|
||
}
|
||
}
|
||
env = append(env, "APP_MYSQL_DSN="+dsn, "APP_HTTP_ADDR=127.0.0.1:0", "ORBIT_URL="+base, "ORBIT_CREDENTIALS_FILE="+path)
|
||
call := func(args ...string) string {
|
||
t.Helper()
|
||
cmd := exec.CommandContext(ctx, binary, args...)
|
||
cmd.Env = env
|
||
raw, err := cmd.CombinedOutput()
|
||
if err != nil {
|
||
t.Fatalf("command failed: %v %s", err, raw)
|
||
}
|
||
return strings.TrimSpace(string(raw))
|
||
}
|
||
call("init")
|
||
sub := call("provision")
|
||
if call("provision") != sub {
|
||
t.Fatal("provision changed subscription")
|
||
}
|
||
env = append(env, "ORBIT_SUBSCRIPTION_ID="+sub)
|
||
target, _ := url.Parse(base)
|
||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||
var offline atomic.Bool
|
||
offline.Store(true)
|
||
gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if offline.Load() {
|
||
http.Error(w, "offline", 503)
|
||
return
|
||
}
|
||
proxy.ServeHTTP(w, r)
|
||
}))
|
||
defer gateway.Close()
|
||
for i, v := range env {
|
||
if strings.HasPrefix(v, "ORBIT_URL=") {
|
||
env[i] = "ORBIT_URL=" + gateway.URL
|
||
}
|
||
}
|
||
type running struct {
|
||
cmd *exec.Cmd
|
||
done chan error
|
||
url string
|
||
stderr *bytes.Buffer
|
||
}
|
||
start := func() *running {
|
||
t.Helper()
|
||
cmd := exec.CommandContext(ctx, binary, "serve")
|
||
cmd.Env = env
|
||
stdout, err := cmd.StdoutPipe()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
stderr := new(bytes.Buffer)
|
||
cmd.Stderr = stderr
|
||
if err = cmd.Start(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
address := make(chan string, 1)
|
||
go func() {
|
||
scanner := bufio.NewScanner(stdout)
|
||
for scanner.Scan() {
|
||
if value, ok := strings.CutPrefix(scanner.Text(), "http_listening="); ok {
|
||
address <- "http://" + value
|
||
}
|
||
}
|
||
close(address)
|
||
}()
|
||
done := make(chan error, 1)
|
||
go func() { done <- cmd.Wait(); close(done) }()
|
||
run := &running{cmd: cmd, done: done, stderr: stderr}
|
||
t.Cleanup(func() { cmd.Process.Kill(); <-done })
|
||
select {
|
||
case run.url = <-address:
|
||
if run.url == "" {
|
||
<-done
|
||
t.Fatalf("startup failed: %s", stderr.String())
|
||
}
|
||
case <-ctx.Done():
|
||
t.Fatal("startup timeout")
|
||
}
|
||
return run
|
||
}
|
||
httpClient := &http.Client{Timeout: 5 * time.Second}
|
||
get := func(base, path string) map[string]any {
|
||
t.Helper()
|
||
r, err := httpClient.Get(base + path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer r.Body.Close()
|
||
if r.StatusCode != 200 {
|
||
t.Fatalf("GET %s: %d", path, r.StatusCode)
|
||
}
|
||
var v map[string]any
|
||
if json.NewDecoder(r.Body).Decode(&v) != nil {
|
||
t.Fatal("invalid JSON")
|
||
}
|
||
return v
|
||
}
|
||
id, err := orbit.NewEventMessageID()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
payload := []byte(`{"operation_id":"` + id + `","source_ref":"objects/http-demo.txt"}`)
|
||
post := func(base string, body []byte, status int) map[string]any {
|
||
t.Helper()
|
||
r, err := httpClient.Post(base+"/jobs", "application/json", bytes.NewReader(body))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer r.Body.Close()
|
||
if r.StatusCode != status {
|
||
raw, _ := io.ReadAll(r.Body)
|
||
t.Fatalf("POST: %d %s", r.StatusCode, raw)
|
||
}
|
||
var out map[string]any
|
||
if status == 202 {
|
||
json.NewDecoder(r.Body).Decode(&out)
|
||
}
|
||
return out
|
||
}
|
||
wait := func(check func() bool) {
|
||
t.Helper()
|
||
for !check() {
|
||
select {
|
||
case <-ctx.Done():
|
||
t.Fatal("condition timeout")
|
||
case <-time.After(50 * time.Millisecond):
|
||
}
|
||
}
|
||
}
|
||
first := start()
|
||
wait(func() bool {
|
||
r, err := httpClient.Get(first.url + "/health/ready")
|
||
if err != nil {
|
||
return false
|
||
}
|
||
r.Body.Close()
|
||
return r.StatusCode == 200
|
||
})
|
||
if v := post(first.url, payload, 202); v["local_committed"] != true || v["duplicate"] != false {
|
||
t.Fatal(v)
|
||
}
|
||
wait(func() bool {
|
||
var n int
|
||
err := db.QueryRowContext(ctx, `SELECT send_count FROM sender_outbox WHERE operation_id=?`, id).Scan(&n)
|
||
return err == nil && n > 0
|
||
})
|
||
first.cmd.Process.Kill()
|
||
if err := <-first.done; err == nil {
|
||
t.Fatal("expected real SIGKILL")
|
||
}
|
||
offline.Store(false)
|
||
second := start()
|
||
wait(func() bool { return get(second.url, "/jobs/"+id)["execution"] == "SUCCEEDED" })
|
||
if post(second.url, payload, 202)["duplicate"] != true {
|
||
t.Fatal("original operation duplicated after restart")
|
||
}
|
||
post(second.url, bytes.ReplaceAll(payload, []byte("http-demo"), []byte("changed")), 409)
|
||
view := get(second.url, "/jobs/"+id)
|
||
result, ok := view["result"].(map[string]any)
|
||
if !ok || result["normalized_ref"] != "OBJECTS/HTTP-DEMO.TXT" {
|
||
t.Fatal(view)
|
||
}
|
||
var jobs, inboxRows, fks int
|
||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sender_jobs WHERE operation_id=?`, id).Scan(&jobs); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM receiver_inbox WHERE queue_id=(SELECT queue_id FROM sender_outbox WHERE operation_id=?)`, id).Scan(&inboxRows); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fks); err != nil || jobs != 1 || inboxRows != 1 || fks != 0 {
|
||
t.Fatal(jobs, inboxRows, fks, err)
|
||
}
|
||
// 先看到平台持久ACK，再发SIGTERM，验证通知不是只停留在业务内存或数据库。
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var creds config
|
||
if json.Unmarshal(raw, &creds) != nil {
|
||
t.Fatal("credentials")
|
||
}
|
||
api, err := orbit.NewClient(base, orbit.Options{Token: creds.Token})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
wait(func() bool { v, err := api.GetSubscription(ctx, sub); return err == nil && v.PendingCount == 0 })
|
||
if err = second.cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
select {
|
||
case err = <-second.done:
|
||
if err != nil {
|
||
t.Fatalf("SIGTERM failed %v %s", err, second.stderr.String())
|
||
}
|
||
case <-ctx.Done():
|
||
t.Fatal("shutdown timeout")
|
||
}
|
||
t.Log("Orbit unavailable: HTTP transaction committed; real SIGKILL/restart recovered original operation; Worker + persistent Inbox + ACK completed once; conflict rejected; SIGTERM exited cleanly; zero foreign keys")
|
||
}
|