go-omni/transport/http2/soak_test.go

package http2

import (
	"context"
	"crypto/tls"
	"io"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	xhttp2 "golang.org/x/net/http2"
)

// TestTLSSoak is short by default so it remains part of normal package
// verification. Linux acceptance runs set GO_OMNI_HTTP2_SOAK_DURATION=30m.
func TestTLSSoak(t *testing.T) {
	started := time.Now()
	duration := soakDuration(t)
	logSoakResources(t, started, "before", 0, 0, Stats{})

	certificate, roots := testCertificate(t)
	server := New(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		switch request.URL.Path {
		case "/slow":
			timer := time.NewTimer(10 * time.Millisecond)
			defer timer.Stop()
			select {
			case <-timer.C:
			case <-request.Context().Done():
				return
			}
		case "/body":
			if _, err := io.Copy(io.Discard, request.Body); err != nil {
				http.Error(writer, err.Error(), http.StatusBadRequest)
				return
			}
		}
		writer.Header().Set("X-Soak-Request-Id", request.Header.Get("X-Soak-Request-Id"))
		_, _ = io.WriteString(writer, request.URL.Path)
	}))
	server.MaxConnections = 8
	server.MaxConcurrentRequests = 128
	server.MaxConcurrentStreams = 128
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	done := make(chan error, 1)
	go func() {
		done <- server.ServeTLS(context.Background(), listener, &tls.Config{
			Certificates: []tls.Certificate{certificate},
			MinVersion:   tls.VersionTLS12,
		})
	}()

	transport := &xhttp2.Transport{
		TLSClientConfig: &tls.Config{
			RootCAs:    roots,
			ServerName: "localhost",
			MinVersion: tls.VersionTLS12,
			NextProtos: []string{"h2"},
		},
		StrictMaxConcurrentStreams: true,
	}
	client := &http.Client{Transport: transport, Timeout: 3 * time.Second}

	var requests, failures atomic.Uint64
	var firstFailure string
	var failureOnce sync.Once
	recordFailure := func(message string) {
		failures.Add(1)
		failureOnce.Do(func() { firstFailure = message })
	}
	monitorStop, monitorDone := make(chan struct{}), make(chan struct{})
	go func() {
		defer close(monitorDone)
		ticker := time.NewTicker(15 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				logSoakResources(t, started, "running", requests.Load(), failures.Load(), server.Stats())
			case <-monitorStop:
				return
			}
		}
	}()
	defer func() {
		close(monitorStop)
		<-monitorDone
	}()
	deadline := time.Now().Add(duration)
	var workers sync.WaitGroup
	for worker := 0; worker < 128; worker++ {
		workers.Add(1)
		go func(worker int) {
			defer workers.Done()
			for sequence := uint64(0); time.Now().Before(deadline); sequence++ {
				path := "/health"
				method := http.MethodGet
				var body io.Reader
				if worker%32 == 0 {
					path = "/slow"
				}
				if worker%64 == 0 {
					path = "/body"
					method = http.MethodPost
					body = &repeatingReader{remaining: 64 << 10}
				}
				requestContext, cancelRequest := context.WithTimeout(context.Background(), 2*time.Second)
				request, err := http.NewRequestWithContext(requestContext, method, "https://"+listener.Addr().String()+path, body)
				if err != nil {
					cancelRequest()
					recordFailure(err.Error())
					continue
				}
				requestID := strconv.Itoa(worker) + ":" + strconv.FormatUint(sequence, 10)
				request.Header.Set("X-Soak-Request-Id", requestID)
				response, err := client.Do(request)
				if err != nil {
					cancelRequest()
					recordFailure(err.Error())
					continue
				}
				responseBody, readErr := io.ReadAll(io.LimitReader(response.Body, 1024))
				response.Body.Close()
				cancelRequest()
				if readErr != nil {
					recordFailure(readErr.Error())
					continue
				}
				if response.StatusCode != http.StatusOK || response.ProtoMajor != 2 || string(responseBody) != path || response.Header.Get("X-Soak-Request-Id") != requestID {
					recordFailure("unexpected status, protocol, body or request ID: " + response.Status)
					continue
				}
				requests.Add(1)
			}
		}(worker)
	}
	workers.Wait()
	transport.CloseIdleConnections()

	shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	shutdownErr := server.Shutdown(shutdownContext)
	cancel()
	if shutdownErr != nil {
		t.Fatalf("TLS soak shutdown: %v", shutdownErr)
	}
	if err := <-done; err != nil {
		t.Fatalf("TLS soak serve: %v", err)
	}
	logSoakResources(t, started, "after", requests.Load(), failures.Load(), server.Stats())
	if failures.Load() != 0 || requests.Load() == 0 {
		t.Fatalf("TLS soak requests=%d failures=%d firstFailure=%q stats=%+v", requests.Load(), failures.Load(), firstFailure, server.Stats())
	}
	if stats := server.Stats(); stats.ActiveConnections != 0 || stats.ActiveRequests != 0 || stats.ProtocolErrors != 0 || stats.RejectedRequests != 0 {
		t.Fatalf("TLS soak protocol errors, unexpected rejection or unreclaimed resources: %+v", stats)
	}
}

func soakDuration(t *testing.T) time.Duration {
	t.Helper()
	if value := os.Getenv("GO_OMNI_HTTP2_SOAK_DURATION"); value != "" {
		duration, err := time.ParseDuration(value)
		if err != nil || duration <= 0 {
			t.Fatalf("invalid GO_OMNI_HTTP2_SOAK_DURATION %q: %v", value, err)
		}
		return duration
	}
	return 3 * time.Second
}

// These measurements cover the client and server in the same test process.
// RSS includes race runtime allocations that do not appear in Go heap stats.
func logSoakResources(t *testing.T, started time.Time, phase string, requests, failures uint64, stats Stats) {
	t.Helper()
	var memory runtime.MemStats
	runtime.ReadMemStats(&memory)
	rssBytes, fileDescriptors := int64(-1), -1
	if data, err := os.ReadFile("/proc/self/statm"); err == nil {
		if fields := strings.Fields(string(data)); len(fields) >= 2 {
			if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
				rssBytes = pages * int64(os.Getpagesize())
			}
		}
	}
	if entries, err := os.ReadDir("/proc/self/fd"); err == nil {
		fileDescriptors = len(entries)
	}
	t.Logf("phase=%s elapsed=%s requests=%d failures=%d heapBytes=%d rssBytes=%d goroutines=%d fileDescriptors=%d stats=%+v",
		phase, time.Since(started).Round(time.Millisecond), requests, failures,
		memory.HeapAlloc, rssBytes, runtime.NumGoroutine(), fileDescriptors, stats)
}

type repeatingReader struct {
	remaining int
}

func (r *repeatingReader) Read(buffer []byte) (int, error) {
	if r.remaining == 0 {
		return 0, io.EOF
	}
	if len(buffer) > r.remaining {
		buffer = buffer[:r.remaining]
	}
	for index := range buffer {
		buffer[index] = 'x'
	}
	r.remaining -= len(buffer)
	return len(buffer), nil
}

func (r *repeatingReader) Close() error { return nil }