package main import ( "bufio" "bytes" "context" "encoding/json" "io" "net/http" "net/http/httptest" "net/http/httputil" "net/url" "os" "os/exec" "path/filepath" "runtime" "strings" "sync/atomic" "syscall" "testing" "time" outbox "git.yuntuo.hk/k2safe/orbit/adapters/outbox/mysql" orbit "git.yuntuo.hk/k2safe/orbit/sdk/go" "github.com/go-sql-driver/mysql" ) func TestIntegrationHTTPServiceRestartAndResults(t *testing.T) { dsn, base, path := os.Getenv("ORBIT_TEST_HTTP_DSN"), os.Getenv("ORBIT_TEST_HTTP_URL"), os.Getenv("ORBIT_TEST_HTTP_CREDENTIALS_FILE") if dsn == "" || base == "" || path == "" { t.Skip("requires disposable business database and Orbit process") } cfg, err := mysql.ParseDSN(dsn) if err != nil || cfg.DBName != "orbit_http_test" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") || !strings.HasPrefix(base, "http://127.0.0.1:") { t.Fatal("requires isolated loopback environment") } ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second) defer cancel() db, err := outbox.Open(ctx, dsn) if err != nil { t.Fatal(err) } defer db.Close() binary := filepath.Join(t.TempDir(), "server") build := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-race", "-o", binary, ".") if raw, err := build.CombinedOutput(); err != nil { t.Fatalf("build failed %v %s", err, raw) } env := []string{} for _, v := range os.Environ() { if !strings.HasPrefix(v, "ORBIT_") && !strings.HasPrefix(v, "APP_") { env = append(env, v) } } env = append(env, "APP_MYSQL_DSN="+dsn, "APP_HTTP_ADDR=127.0.0.1:0", "ORBIT_URL="+base, "ORBIT_CREDENTIALS_FILE="+path) call := func(args ...string) string { t.Helper() cmd := exec.CommandContext(ctx, binary, args...) cmd.Env = env raw, err := cmd.CombinedOutput() if err != nil { t.Fatalf("command failed: %v %s", err, raw) } return strings.TrimSpace(string(raw)) } call("init") sub := call("provision") if call("provision") != sub { t.Fatal("provision changed subscription") } env = append(env, "ORBIT_SUBSCRIPTION_ID="+sub) target, _ := url.Parse(base) proxy := httputil.NewSingleHostReverseProxy(target) var offline atomic.Bool offline.Store(true) gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if offline.Load() { http.Error(w, "offline", 503) return } proxy.ServeHTTP(w, r) })) defer gateway.Close() for i, v := range env { if strings.HasPrefix(v, "ORBIT_URL=") { env[i] = "ORBIT_URL=" + gateway.URL } } type running struct { cmd *exec.Cmd done chan error url string stderr *bytes.Buffer } start := func() *running { t.Helper() cmd := exec.CommandContext(ctx, binary, "serve") cmd.Env = env stdout, err := cmd.StdoutPipe() if err != nil { t.Fatal(err) } stderr := new(bytes.Buffer) cmd.Stderr = stderr if err = cmd.Start(); err != nil { t.Fatal(err) } address := make(chan string, 1) go func() { scanner := bufio.NewScanner(stdout) for scanner.Scan() { if value, ok := strings.CutPrefix(scanner.Text(), "http_listening="); ok { address <- "http://" + value } } close(address) }() done := make(chan error, 1) go func() { done <- cmd.Wait(); close(done) }() run := &running{cmd: cmd, done: done, stderr: stderr} t.Cleanup(func() { cmd.Process.Kill(); <-done }) select { case run.url = <-address: if run.url == "" { <-done t.Fatalf("startup failed: %s", stderr.String()) } case <-ctx.Done(): t.Fatal("startup timeout") } return run } httpClient := &http.Client{Timeout: 5 * time.Second} get := func(base, path string) map[string]any { t.Helper() r, err := httpClient.Get(base + path) if err != nil { t.Fatal(err) } defer r.Body.Close() if r.StatusCode != 200 { t.Fatalf("GET %s: %d", path, r.StatusCode) } var v map[string]any if json.NewDecoder(r.Body).Decode(&v) != nil { t.Fatal("invalid JSON") } return v } id, err := orbit.NewEventMessageID() if err != nil { t.Fatal(err) } payload := []byte(`{"operation_id":"` + id + `","source_ref":"objects/http-demo.txt"}`) post := func(base string, body []byte, status int) map[string]any { t.Helper() r, err := httpClient.Post(base+"/jobs", "application/json", bytes.NewReader(body)) if err != nil { t.Fatal(err) } defer r.Body.Close() if r.StatusCode != status { raw, _ := io.ReadAll(r.Body) t.Fatalf("POST: %d %s", r.StatusCode, raw) } var out map[string]any if status == 202 { json.NewDecoder(r.Body).Decode(&out) } return out } wait := func(check func() bool) { t.Helper() for !check() { select { case <-ctx.Done(): t.Fatal("condition timeout") case <-time.After(50 * time.Millisecond): } } } first := start() wait(func() bool { r, err := httpClient.Get(first.url + "/health/ready") if err != nil { return false } r.Body.Close() return r.StatusCode == 200 }) if v := post(first.url, payload, 202); v["local_committed"] != true || v["duplicate"] != false { t.Fatal(v) } wait(func() bool { var n int err := db.QueryRowContext(ctx, `SELECT send_count FROM sender_outbox WHERE operation_id=?`, id).Scan(&n) return err == nil && n > 0 }) first.cmd.Process.Kill() if err := <-first.done; err == nil { t.Fatal("expected real SIGKILL") } offline.Store(false) second := start() wait(func() bool { return get(second.url, "/jobs/"+id)["execution"] == "SUCCEEDED" }) if post(second.url, payload, 202)["duplicate"] != true { t.Fatal("original operation duplicated after restart") } post(second.url, bytes.ReplaceAll(payload, []byte("http-demo"), []byte("changed")), 409) view := get(second.url, "/jobs/"+id) result, ok := view["result"].(map[string]any) if !ok || result["normalized_ref"] != "OBJECTS/HTTP-DEMO.TXT" { t.Fatal(view) } var jobs, inboxRows, fks int if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sender_jobs WHERE operation_id=?`, id).Scan(&jobs); err != nil { t.Fatal(err) } if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM receiver_inbox WHERE queue_id=(SELECT queue_id FROM sender_outbox WHERE operation_id=?)`, id).Scan(&inboxRows); err != nil { t.Fatal(err) } if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE()`).Scan(&fks); err != nil || jobs != 1 || inboxRows != 1 || fks != 0 { t.Fatal(jobs, inboxRows, fks, err) } // 先看到平台持久ACK,再发SIGTERM,验证通知不是只停留在业务内存或数据库。 raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var creds config if json.Unmarshal(raw, &creds) != nil { t.Fatal("credentials") } api, err := orbit.NewClient(base, orbit.Options{Token: creds.Token}) if err != nil { t.Fatal(err) } wait(func() bool { v, err := api.GetSubscription(ctx, sub); return err == nil && v.PendingCount == 0 }) if err = second.cmd.Process.Signal(syscall.SIGTERM); err != nil { t.Fatal(err) } select { case err = <-second.done: if err != nil { t.Fatalf("SIGTERM failed %v %s", err, second.stderr.String()) } case <-ctx.Done(): t.Fatal("shutdown timeout") } t.Log("Orbit unavailable: HTTP transaction committed; real SIGKILL/restart recovered original operation; Worker + persistent Inbox + ACK completed once; conflict rejected; SIGTERM exited cleanly; zero foreign keys") }