feat: add reliable event consumer SDK and process recovery tests

sdk/go/event_process_integration_test.go

@@ -0,0 +1,103 @@
+package orbit
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestIntegrationEventProcessLeaseRecovery(t *testing.T) {
+	a := startOrbitProcess(t)
+	_, _, token := integrationWorkerClient(t)
+	api := processClient(t, a, token)
+	ctx, stop := context.WithTimeout(context.Background(), 45*time.Second)
+	defer stop()
+	sub := integrationEventSubscription(t, ctx, api, 3)
+	_ = integrationCancelledEvent(t, ctx, api, sub)
+	firstConn := eventDialIntegration(t, ctx, api, sub, 1)
+	first, err := firstConn.Next(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.kill(t)
+	select {
+	case <-firstConn.Done():
+	case <-time.After(3 * time.Second):
+		t.Fatal("dead server connection remained live")
+	}
+	b := startOrbitProcess(t)
+	other := processClient(t, b, token)
+	secondConn := eventDialIntegration(t, ctx, other, sub, 1)
+	second, err := secondConn.Next(ctx)
+	if err != nil || first.Event.ID != second.Event.ID || first.Delivery.ID != second.Delivery.ID || second.Delivery.LeaseEpoch != 2 || second.Delivery.Attempts != 2 {
+		t.Fatal("node restart did not recover original delivery", err)
+	}
+	_, err = secondConn.Ack(ctx, workerTestID(t), first.Proof())
+	var remote *EventError
+	if !errors.As(err, &remote) || remote.Code != "DELIVERY_LEASE_LOST" {
+		t.Fatal("dead node proof acknowledged newer lease", err)
+	}
+	if _, err = secondConn.Ack(ctx, workerTestID(t), second.Proof()); err != nil {
+		t.Fatal(err)
+	}
+	current, err := other.GetSubscription(ctx, sub.ID)
+	if err != nil || current.PendingCount != 0 {
+		t.Fatal("recovered acknowledgement not persisted", err)
+	}
+	t.Log("SIGKILL confirmed by Wait; new process recovered original delivery after natural lease expiry, rejected old proof and committed new ACK")
+}
+func TestIntegrationEventProcessAckCommitResponseLoss(t *testing.T) {
+	a := startOrbitProcess(t)
+	_, _, token := integrationWorkerClient(t)
+	api := processClient(t, a, token)
+	ctx, stop := context.WithTimeout(context.Background(), 20*time.Second)
+	defer stop()
+	sub := integrationEventSubscription(t, ctx, api, 3)
+	_ = integrationCancelledEvent(t, ctx, api, sub)
+	gateway := newProtocolFaultGateway(t, a.url, "/v1/events/connect", eventSubprotocol)
+	gateway.drop.Store("event.ack")
+	gateway.dropOffline.Store(true)
+	client, err := NewClient(gateway.server.URL, Options{Token: token, HTTPClient: gateway.server.Client()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var calls atomic.Int64
+	consumer, err := NewEventConsumer(client, EventConsumerOptions{SubscriptionID: sub.ID, Capacity: 1, ShutdownTimeout: time.Second, Handle: func(context.Context, ResultEvent) error { calls.Add(1); return nil }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	run, cancel := context.WithCancel(ctx)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- consumer.Run(run) }()
+	awaitProcessDrop(t, ctx, gateway, "event.ack")
+	before, err := api.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
+	if err != nil || len(before.Items) != 1 || before.Items[0].AcknowledgedAt == nil {
+		t.Fatal("fault occurred before durable ACK", err)
+	}
+	a.kill(t)
+	b := startOrbitProcess(t)
+	other := processClient(t, b, token)
+	gateway.target.Store(b.url)
+	gateway.offline.Store(false)
+	for consumer.Stats().Acknowledged == 0 {
+		select {
+		case <-ctx.Done():
+			t.Fatal("consumer failed to reconcile ACK on new process")
+		case err := <-done:
+			t.Fatal("consumer stopped", err)
+		case <-time.After(20 * time.Millisecond):
+		}
+	}
+	cancel()
+	if err := awaitConsumerStop(t, done); err != nil {
+		t.Fatal(err)
+	}
+	after, err := other.ListEventDeliveries(ctx, sub.ID, DeliveryPageRequest{State: DeliveryAcked})
+	if err != nil || len(after.Items) != 1 || after.Items[0].Attempts != 1 || after.Items[0].AcknowledgedAt == nil || !after.Items[0].AcknowledgedAt.Equal(*before.Items[0].AcknowledgedAt) || calls.Load() != 1 {
+		t.Fatal("ACK receipt changed or committed handler reran", err)
+	}
+	t.Log("ACK commit verified before response loss and SIGKILL; replacement process returned original receipt without another handler or delivery attempt")
+}