Phase 4: Production readiness - graceful shutdown, health checks, load testing, pprof

- Graceful shutdown: SIGINT/SIGTERM handling with 5s timeout
- Readiness probe (/readyz): pings database before reporting ready
- Connection pool: documented MaxOpenConns(1) for SQLite single-writer constraint
- Load test tool: cmd/loadtest with configurable concurrency/total requests
- Shell script: scripts/load-test.sh for quick load testing
- pprof endpoints: /debug/pprof/* when HATCH_DEBUG env var is set

Co-Authored-By: Paperclip <noreply@paperclip.ing>

cmd/hatch/main.go

@@ -1,10 +1,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	"github.com/elfoundation/hatch/internal/handler"
 	"github.com/elfoundation/hatch/internal/store"
@@ -30,12 +34,36 @@ func main() {
 	defer repo.Close()
 
 	h := handler.New(repo)
+	h.Debug = os.Getenv("HATCH_DEBUG") != ""
 	r := chi.NewRouter()
 	h.RegisterRoutes(r)
 
 	addr := fmt.Sprintf(":%s", port)
+	srv := &http.Server{
+		Addr:    addr,
+		Handler: r,
+	}
+
+	// Start server in a goroutine.
+	go func() {
 		log.Printf("hatch starting on %s", addr)
-	if err := http.ListenAndServe(addr, r); err != nil {
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
 			log.Fatalf("hatch server error: %v", err)
 		}
+	}()
+
+	// Wait for interrupt signal for graceful shutdown.
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+	<-quit
+	log.Println("shutting down server...")
+
+	// Create a deadline for shutdown.
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(ctx); err != nil {
+		log.Fatalf("server forced to shutdown: %v", err)
+	}
+
+	log.Println("server exited")
 }

cmd/loadtest/main.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+func main() {
+	url := flag.String("url", "http://localhost:8080/healthz", "URL to test")
+	concurrency := flag.Int("c", 10, "concurrent workers")
+	total := flag.Int("n", 1000, "total requests")
+	flag.Parse()
+
+	var (
+		successCount int64
+		failCount    int64
+		totalLatency int64 // microseconds
+		minLatency   int64 = 1<<63 - 1
+		maxLatency   int64
+	)
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+		Transport: &http.Transport{
+			MaxIdleConns:        *concurrency,
+			MaxIdleConnsPerHost: *concurrency,
+			IdleConnTimeout:     90 * time.Second,
+		},
+	}
+
+	var wg sync.WaitGroup
+	requests := make(chan int, *total)
+	for i := 0; i < *total; i++ {
+		requests <- i
+	}
+	close(requests)
+
+	start := time.Now()
+
+	for i := 0; i < *concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for range requests {
+				reqStart := time.Now()
+				resp, err := client.Get(*url)
+				latency := time.Since(reqStart).Microseconds()
+				if err != nil {
+					atomic.AddInt64(&failCount, 1)
+					continue
+				}
+				io.Copy(io.Discard, resp.Body)
+				resp.Body.Close()
+				if resp.StatusCode == http.StatusOK {
+					atomic.AddInt64(&successCount, 1)
+				} else {
+					atomic.AddInt64(&failCount, 1)
+				}
+				atomic.AddInt64(&totalLatency, latency)
+				// Update min/max using atomic operations for simplicity
+				for {
+					old := atomic.LoadInt64(&minLatency)
+					if latency >= old || atomic.CompareAndSwapInt64(&minLatency, old, latency) {
+						break
+					}
+				}
+				for {
+					old := atomic.LoadInt64(&maxLatency)
+					if latency <= old || atomic.CompareAndSwapInt64(&maxLatency, old, latency) {
+						break
+					}
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	duration := time.Since(start)
+
+	fmt.Printf("Load test completed\n")
+	fmt.Printf("URL: %s\n", *url)
+	fmt.Printf("Total requests: %d\n", *total)
+	fmt.Printf("Concurrency: %d\n", *concurrency)
+	fmt.Printf("Duration: %v\n", duration)
+	fmt.Printf("Success: %d\n", atomic.LoadInt64(&successCount))
+	fmt.Printf("Failed: %d\n", atomic.LoadInt64(&failCount))
+	if *total > 0 {
+		avgLatency := float64(atomic.LoadInt64(&totalLatency)) / float64(*total)
+		fmt.Printf("Average latency: %.2f ms\n", avgLatency/1000)
+		fmt.Printf("Min latency: %.2f ms\n", float64(atomic.LoadInt64(&minLatency))/1000)
+		fmt.Printf("Max latency: %.2f ms\n", float64(atomic.LoadInt64(&maxLatency))/1000)
+		fmt.Printf("Requests/sec: %.2f\n", float64(*total)/duration.Seconds())
+	}
+}

internal/handler/handler.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"net/http/pprof"
 	"strings"
 
 	"github.com/elfoundation/hatch/internal/store"
@@ -14,6 +15,7 @@ import (
 // Handler groups all HTTP handlers and their shared dependencies.
 type Handler struct {
 	Repo  store.Repository
+	Debug bool
 }
 
 // New creates a new Handler with the given store.
@@ -28,6 +30,18 @@ func (h *Handler) RegisterRoutes(r chi.Router) {
 
 	// Health check.
 	r.Get("/healthz", Healthz)
+	r.Get("/readyz", h.Readyz)
+
+	// Debug endpoints (only when enabled).
+	if h.Debug {
+		r.Route("/debug/pprof", func(r chi.Router) {
+			r.HandleFunc("/", pprof.Index)
+			r.HandleFunc("/cmdline", pprof.Cmdline)
+			r.HandleFunc("/profile", pprof.Profile)
+			r.HandleFunc("/symbol", pprof.Symbol)
+			r.HandleFunc("/trace", pprof.Trace)
+		})
+	}
 
 	// JSON API v1 routes.
 	h.RegisterV1Routes(r)
@@ -120,6 +134,20 @@ func Healthz(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("ok\n"))
 }
 
+// Readyz returns 200 OK if the service is ready to handle requests.
+// It checks database connectivity.
+func (h *Handler) Readyz(w http.ResponseWriter, r *http.Request) {
+	if err := h.Repo.Ping(r.Context()); err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		json.NewEncoder(w).Encode(map[string]string{"error": "database not ready"})
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("ok\n"))
+}
+
 // writeError writes a JSON error response.
 func writeError(w http.ResponseWriter, status int, msg string) {
 	w.Header().Set("Content-Type", "application/json")

internal/store/db.go

@@ -24,7 +24,13 @@ func Open(dbPath string) (Repository, error) {
 	if err != nil {
 		return nil, fmt.Errorf("store: open %s: %w", dbPath, err)
 	}
+	// SQLite supports only one writer at a time. Using MaxOpenConns(1)
+	// prevents "database is locked" errors under concurrent writes.
+	// For read-heavy workloads with WAL mode, this can be increased,
+	// but the single-writer constraint remains.
 	conn.SetMaxOpenConns(1)
+	conn.SetMaxIdleConns(1)
+	conn.SetConnMaxLifetime(0) // No limit; SQLite connections are lightweight.
 	if err := migrate(conn); err != nil {
 		conn.Close()
 		return nil, fmt.Errorf("store: migrate: %w", err)

internal/store/models.go

@@ -38,6 +38,7 @@ type Repository interface {
 	GetMock(ctx context.Context, endpointID string) (*MockConfig, error)
 	SetMock(ctx context.Context, mock *MockConfig) error
 	Close() error
+	Ping(ctx context.Context) error
 }
 
 var _ Repository = (*sqliteRepo)(nil)

internal/store/sqlite_repo.go

@@ -139,4 +139,8 @@ func (r *sqliteRepo) SearchRequests(ctx context.Context, endpointID string, quer
 
 func (r *sqliteRepo) Close() error { return r.db.Close() }
 
+func (r *sqliteRepo) Ping(ctx context.Context) error {
+	return r.db.PingContext(ctx)
+}
+
 func utcNow() string { return time.Now().UTC().Format("2006-01-02T15:04:05.000Z07:00") }

internal/testutil/fake_repo.go

@@ -91,6 +91,8 @@ func (f *FakeRepository) SetMock(_ context.Context, mock *store.MockConfig) erro
 
 func (f *FakeRepository) Close() error { return nil }
 
+func (f *FakeRepository) Ping(_ context.Context) error { return nil }
+
 type notFoundError struct{}
 
 func (e *notFoundError) Error() string { return "not found" }

scripts/load-test.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Load test script for Hatch server.
+# Assumes the server is already running at http://localhost:8080
+# or at the URL specified by HATCH_URL.
+
+HATCH_URL="${HATCH_URL:-http://localhost:8080}"
+CONCURRENCY="${CONCURRENCY:-10}"
+TOTAL="${TOTAL:-1000}"
+
+echo "Running load test against $HATCH_URL"
+echo "Concurrency: $CONCURRENCY, Total requests: $TOTAL"
+
+# Build the load test binary if not present.
+if [ ! -f ./loadtest ]; then
+    echo "Building load test binary..."
+    go build -o ./loadtest ./cmd/loadtest
+fi
+
+# Run the load test.
+./loadtest -url "$HATCH_URL/healthz" -c "$CONCURRENCY" -n "$TOTAL"
+
+echo "Load test completed."