diff --git a/README.md b/README.md index d7c89c0..38c1bf1 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ curl http://localhost:8080// - Optional `Access-Control-Allow-Origin` header (`-cors-origin`) for simple CORS use cases - Structured logging via `log/slog` with text or JSON output (`-log-format`) - `/_health` endpoint for liveness/readiness probes +- Graceful shutdown on SIGTERM/SIGINT with configurable drain windows (`-shutdown-delay`, `-shutdown-idle-grace-period`, `-shutdown-timeout`) for zero-downtime deploys ## Installation @@ -93,6 +94,12 @@ Usage of gcsproxy: Minimum log level: debug, info, warn, or error. (default "info") -not-found string Object served with HTTP 404 for unmatched routes. + -shutdown-delay duration + Delay after SIGTERM/SIGINT during which requests are served normally but responses carry Connection: close. + -shutdown-idle-grace-period duration + After the listener closes, keep idle connections open for up to this long; ends early once no connections remain. + -shutdown-timeout duration + Max wait for in-flight requests after the idle grace period; 0 waits indefinitely. (default 30s) -spa SPA fallback: serve -i from the bucket root with HTTP 200 for unmatched routes. -v Show access log. @@ -232,7 +239,30 @@ Edge cases: ### Health check -`/_health` returns `200 OK` with the body `OK`. It does not call GCS and is safe to use as a Kubernetes/Cloud Run liveness or readiness probe. +`/_health` returns `200 OK` with the body `OK`. It does not call GCS and is safe to use as a Kubernetes/Cloud Run liveness or readiness probe. It keeps returning `200` during graceful shutdown, so a liveness probe won't kill the process mid-drain. + +### Graceful shutdown + +On SIGTERM or SIGINT, gcsproxy shuts down in four phases designed to eliminate 502/503s and connection resets during rolling deploys: + +1. **Drain** (`-shutdown-delay`): nothing stops — new connections are accepted and requests are served normally, but every response carries `Connection: close`, telling keep-alive clients (e.g. an nginx `upstream keepalive` pool) to retire the connection after use. +2. **Stop accepting**: the listener closes; new TCP connections are refused. Existing connections keep working. +3. **Idle grace** (`-shutdown-idle-grace-period`): existing idle connections stay open, because a client may send a request at the exact moment the server would close one (that race is what causes resets). Requests arriving on them are still served, each response again marked `Connection: close`. This phase ends as soon as no connections remain open, so it never waits longer than necessary. +4. **Close** (`-shutdown-timeout`): remaining idle connections are closed and gcsproxy waits for in-flight requests to finish — at most `-shutdown-timeout` (default `30s`; `0` waits indefinitely) — then exits 0. + +`-shutdown-delay` and `-shutdown-idle-grace-period` default to `0s`, which still gives a basic graceful shutdown (in-flight requests complete before exit). Recommended production values: + +``` +gcsproxy -shutdown-delay 5s -shutdown-idle-grace-period 5s -shutdown-timeout 30s +``` + +Sizing guidance: + +- Set `-shutdown-idle-grace-period` above your clients' idle-connection reuse timeout (nginx upstream `keepalive_timeout`, Go's `Transport.IdleConnTimeout`, etc.) so every pooled connection is either used once more (and cleanly closed) or closed by the client before the server closes it. +- Make sure your supervisor's kill timeout (systemd `TimeoutStopSec`, Kubernetes `terminationGracePeriodSeconds`) exceeds `shutdown-delay + shutdown-idle-grace-period + shutdown-timeout`, otherwise the process is SIGKILLed mid-drain. +- A second SIGTERM/SIGINT terminates the process immediately. + +Note that even with the default values, SIGTERM now waits for in-flight requests (up to `-shutdown-timeout`) instead of exiting immediately; supervisors backstop this with SIGKILL after their kill timeout. ### Authentication @@ -275,8 +305,10 @@ Wants=network-online.target [Service] Type=simple -ExecStart=/opt/gcsproxy/gcsproxy -v +ExecStart=/opt/gcsproxy/gcsproxy -v -shutdown-delay 5s -shutdown-idle-grace-period 5s -shutdown-timeout 30s Restart=on-failure +# Must exceed shutdown-delay + shutdown-idle-grace-period + shutdown-timeout. +TimeoutStopSec=45 [Install] WantedBy=multi-user.target diff --git a/gcsproxy.service b/gcsproxy.service index 4f736c0..bd2473b 100644 --- a/gcsproxy.service +++ b/gcsproxy.service @@ -3,8 +3,10 @@ Description=gcsproxy [Service] Type=simple -ExecStart=/opt/gcsproxy/gcsproxy -v +ExecStart=/opt/gcsproxy/gcsproxy -v -shutdown-timeout 30s ExecStop=/bin/kill -SIGTERM $MAINPID +# Must exceed shutdown-delay + shutdown-idle-grace-period + shutdown-timeout. +TimeoutStopSec=45 [Install] WantedBy = multi-user.target \ No newline at end of file diff --git a/main.go b/main.go index e172701..349cf8f 100644 --- a/main.go +++ b/main.go @@ -7,11 +7,16 @@ import ( "fmt" "io" "log/slog" + "net" "net/http" "net/url" "os" + "os/signal" "strconv" "strings" + "sync" + "sync/atomic" + "syscall" "time" "cloud.google.com/go/auth/credentials" @@ -21,16 +26,20 @@ import ( ) type Server struct { - addr string - client *storage.Client - defaultIndex string - walkUpIndex bool - sourceBucket string - spa bool - notFoundPath string - contentLength bool - corsOrigin string - verbose bool + addr string + client *storage.Client + defaultIndex string + walkUpIndex bool + sourceBucket string + spa bool + notFoundPath string + contentLength bool + corsOrigin string + verbose bool + shutdownDelay time.Duration + idleGracePeriod time.Duration + shutdownTimeout time.Duration + draining atomic.Bool } func main() { @@ -47,6 +56,9 @@ func main() { logLevel = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") contentLength = flag.Bool("content-length", false, "Send the Content-Length header (disables chunked transfer).") corsOrigin = flag.String("cors-origin", "", "Value for the Access-Control-Allow-Origin header.") + shutdownDelay = flag.Duration("shutdown-delay", 0, "Delay after SIGTERM/SIGINT during which requests are served normally but responses carry Connection: close.") + idleGracePeriod = flag.Duration("shutdown-idle-grace-period", 0, "After the listener closes, keep idle connections open for up to this long; ends early once no connections remain.") + shutdownTimeout = flag.Duration("shutdown-timeout", 30*time.Second, "Max wait for in-flight requests after the idle grace period; 0 waits indefinitely.") ) flag.Parse() @@ -74,6 +86,9 @@ func main() { if *spa && *notFoundPath != "" { fatal("-spa and -not-found are mutually exclusive") } + if *shutdownDelay < 0 || *idleGracePeriod < 0 || *shutdownTimeout < 0 { + fatal("-shutdown-delay, -shutdown-idle-grace-period and -shutdown-timeout must not be negative") + } ctx := context.Background() var opts []option.ClientOption @@ -91,28 +106,158 @@ func main() { if err != nil { fatal("failed to create client", "err", err) } + defer client.Close() s := &Server{ - addr: *bind, - client: client, - defaultIndex: *defaultIndex, - walkUpIndex: *walkUpIndex, - sourceBucket: *sourceBucket, - spa: *spa, - notFoundPath: *notFoundPath, - contentLength: *contentLength, - corsOrigin: *corsOrigin, - verbose: *verbose, - } - - if err := s.ListenAndServe(); err != nil { + addr: *bind, + client: client, + defaultIndex: *defaultIndex, + walkUpIndex: *walkUpIndex, + sourceBucket: *sourceBucket, + spa: *spa, + notFoundPath: *notFoundPath, + contentLength: *contentLength, + corsOrigin: *corsOrigin, + verbose: *verbose, + shutdownDelay: *shutdownDelay, + idleGracePeriod: *idleGracePeriod, + shutdownTimeout: *shutdownTimeout, + } + + sigCtx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer stop() + // Unregister after the first signal so a second SIGTERM/SIGINT kills the + // process immediately via the default disposition. + context.AfterFunc(sigCtx, stop) + + if err := s.ListenAndServe(sigCtx); err != nil { fatal("server exited", "err", err) } } -func (s *Server) ListenAndServe() error { - slog.Info("listening", "addr", s.addr) - return http.ListenAndServe(s.addr, s.handler()) +func (s *Server) ListenAndServe(ctx context.Context) error { + ln, err := net.Listen("tcp", s.addr) + if err != nil { + return err + } + slog.Info("listening", "addr", ln.Addr().String()) + return s.serve(ctx, ln, s.handler()) +} + +// serve runs h on ln until ctx is canceled, then executes the phased graceful +// shutdown: drain (Connection: close) -> close listener -> idle grace -> +// Shutdown. Idle connections stay open until the idle grace period elapses so +// clients never race a new request against a server-side close. +func (s *Server) serve(ctx context.Context, ln net.Listener, h http.Handler) error { + tracker := &connTracker{} + srv := &http.Server{Handler: s.drainingHandler(h), ConnState: tracker.connState} + + errc := make(chan error, 1) + go func() { errc <- srv.Serve(ln) }() + + select { + case err := <-errc: + return err + case <-ctx.Done(): + } + + // Phase 1: keep serving; every new response tells clients to stop reusing. + s.draining.Store(true) + slog.Info("shutdown: draining", "delay", s.shutdownDelay) + time.Sleep(s.shutdownDelay) + + // Phase 2: stop accepting new connections. Existing conns keep working. + slog.Info("shutdown: closing listener") + ln.Close() + // Wait for Serve to return so it untracks its listener; otherwise Shutdown + // below can report a spurious double-close error. + if err := <-errc; err != nil && !errors.Is(err, net.ErrClosed) { + slog.Warn("shutdown: serve loop exited with error", "err", err) + } + + // Phase 3: leave idle connections open; requests on them are still served. + // Skip the wait when no connections remain, and end it early once the + // last one closes. + slog.Info("shutdown: waiting before closing idle connections", "grace", s.idleGracePeriod, "open", tracker.open()) + select { + case <-tracker.noneOpen(): + slog.Info("shutdown: no connections remain") + case <-time.After(s.idleGracePeriod): + } + + // Phase 4: close idle connections, wait for in-flight requests. + slog.Info("shutdown: closing idle connections", "timeout", s.shutdownTimeout) + sctx := context.Background() + if s.shutdownTimeout > 0 { + var cancel context.CancelFunc + sctx, cancel = context.WithTimeout(sctx, s.shutdownTimeout) + defer cancel() + } + if err := srv.Shutdown(sctx); err != nil { + srv.Close() + return fmt.Errorf("graceful shutdown incomplete: %w", err) + } + slog.Info("shutdown: complete") + return nil +} + +// connTracker counts open server connections via http.Server.ConnState so +// the shutdown sequence can stop waiting as soon as none remain. Active +// connections count too: a response whose headers predate the drain flag +// carries no Connection: close, so its connection can still go idle and be +// reused by the client. +type connTracker struct { + mu sync.Mutex + count int + waiter chan struct{} +} + +func (ct *connTracker) connState(_ net.Conn, state http.ConnState) { + ct.mu.Lock() + defer ct.mu.Unlock() + switch state { + case http.StateNew: + ct.count++ + case http.StateHijacked, http.StateClosed: + ct.count-- + if ct.count == 0 && ct.waiter != nil { + close(ct.waiter) + ct.waiter = nil + } + } +} + +func (ct *connTracker) open() int { + ct.mu.Lock() + defer ct.mu.Unlock() + return ct.count +} + +// noneOpen returns a channel that is closed once no connections remain open. +// Only valid after the listener stopped accepting new connections, since the +// count never rises again from zero. +func (ct *connTracker) noneOpen() <-chan struct{} { + ct.mu.Lock() + defer ct.mu.Unlock() + ch := make(chan struct{}) + if ct.count == 0 { + close(ch) + } else { + ct.waiter = ch + } + return ch +} + +// drainingHandler marks every response written after shutdown begins with +// Connection: close, so keep-alive clients stop reusing the connection. +// net/http then closes the connection after the response completes. +func (s *Server) drainingHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.draining.Load() { + w.Header().Set("Connection", "close") + } + next.ServeHTTP(w, r) + }) } func (s *Server) handler() http.Handler { diff --git a/main_test.go b/main_test.go index fd3cc6a..a80ffac 100644 --- a/main_test.go +++ b/main_test.go @@ -1,17 +1,23 @@ package main import ( + "bufio" "bytes" + "context" "encoding/json" "errors" "io" "log/slog" + "net" "net/http" "net/http/httptest" "net/http/httputil" "net/url" + "os" + "os/exec" "strconv" "strings" + "syscall" "testing" "time" @@ -1555,3 +1561,522 @@ func TestProxy_HEAD_GzippedObject(t *testing.T) { } }) } + +// startShutdownServer runs s.serve with handler h on an ephemeral port. +// Canceling the returned func stands in for SIGTERM; the returned channel +// receives serve's return value. +func startShutdownServer(t *testing.T, s *Server, h http.Handler) (string, context.CancelFunc, <-chan error) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.serve(ctx, ln, h) }() + t.Cleanup(cancel) + return ln.Addr().String(), cancel, done +} + +func waitServeDone(t *testing.T, done <-chan error) { + t.Helper() + select { + case err := <-done: + if err != nil { + t.Fatalf("serve returned %v, want nil", err) + } + case <-time.After(30 * time.Second): + t.Fatal("server did not shut down") + } +} + +// waitListenerClosed polls until new TCP connections are refused, proving +// the listener is closed. +func waitListenerClosed(t *testing.T, addr string) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + conn, err := net.Dial("tcp", addr) + if err != nil { + return + } + conn.Close() + if time.Now().After(deadline) { + t.Fatal("listener never closed") + } + time.Sleep(5 * time.Millisecond) + } +} + +func getHealth(t *testing.T, addr string) *http.Response { + t.Helper() + res, err := http.Get("http://" + addr + "/_health") + if err != nil { + t.Fatalf("GET /_health: %v", err) + } + if _, err := io.Copy(io.Discard, res.Body); err != nil { + t.Fatalf("read /_health body: %v", err) + } + res.Body.Close() + if res.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", res.StatusCode, http.StatusOK) + } + return res +} + +func TestShutdown_DrainWindowSendsConnectionClose(t *testing.T) { + s := newTestServer(t, nil) + // Far longer than the test runs, so the drain window is still open when + // the assertions execute no matter how slow the runner is. The serve + // goroutine is deliberately abandoned mid-drain; the test binary's exit + // reaps it, and shutdown completion is covered by the other tests. + s.shutdownDelay = 10 * time.Minute + addr, cancel, _ := startShutdownServer(t, s, s.handler()) + + if res := getHealth(t, addr); res.Close { + t.Fatal("response before shutdown carries Connection: close") + } + + cancel() + // Every drain-window response must be 200 (getHealth asserts that) and + // gain Connection: close once draining is visible. + deadline := time.Now().Add(30 * time.Second) + for !getHealth(t, addr).Close { + if time.Now().After(deadline) { + t.Fatal("drain-window responses never gained Connection: close") + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestShutdown_IdleConnServedAfterListenerClose(t *testing.T) { + s := newTestServer(t, nil) + // Far longer than the test runs: the idle connection must only be closed + // by the early exit after it closes, never by elapsed time. The test + // still finishes immediately because serving the idle connection closes + // it, which ends the idle-grace wait. + s.idleGracePeriod = 10 * time.Minute + addr, cancel, done := startShutdownServer(t, s, s.handler()) + + conn, br := dialIdleConn(t, addr) + + cancel() + waitListenerClosed(t, addr) // also asserts brand-new conns are refused + + // A request on the idle connection during the idle grace period must + // still be served, with Connection: close. + if _, err := io.WriteString(conn, "GET /_health HTTP/1.1\r\nHost: gcsproxy.test\r\n\r\n"); err != nil { + t.Fatalf("write request on idle conn: %v", err) + } + res, err := http.ReadResponse(br, nil) + if err != nil { + t.Fatalf("read response on idle conn: %v", err) + } + if _, err := io.Copy(io.Discard, res.Body); err != nil { + t.Fatalf("read response body: %v", err) + } + res.Body.Close() + if res.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", res.StatusCode, http.StatusOK) + } + if !res.Close { + t.Fatal("response after listener close lacks Connection: close") + } + + // After that response the server closes the connection cleanly, and with + // no connections left the idle-grace wait ends early. + conn.SetReadDeadline(time.Now().Add(30 * time.Second)) + if _, err := br.ReadByte(); err != io.EOF { + t.Fatalf("read after final response = %v, want io.EOF", err) + } + waitServeDone(t, done) +} + +func TestShutdown_IdleGraceSkippedWhenNoConnections(t *testing.T) { + s := newTestServer(t, nil) + s.idleGracePeriod = 10 * time.Minute // must be skipped: no connections exist + _, cancel, done := startShutdownServer(t, s, s.handler()) + + cancel() + waitServeDone(t, done) // fails if the wait is not skipped +} + +// dialIdleConn opens a raw keep-alive connection and completes one request on +// it, leaving it idle like a proxy pool member. +func dialIdleConn(t *testing.T, addr string) (net.Conn, *bufio.Reader) { + t.Helper() + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatalf("net.Dial: %v", err) + } + t.Cleanup(func() { conn.Close() }) + br := bufio.NewReader(conn) + if _, err := io.WriteString(conn, "GET /_health HTTP/1.1\r\nHost: gcsproxy.test\r\n\r\n"); err != nil { + t.Fatalf("write request: %v", err) + } + res, err := http.ReadResponse(br, nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + if _, err := io.Copy(io.Discard, res.Body); err != nil { + t.Fatalf("read response body: %v", err) + } + res.Body.Close() + if res.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", res.StatusCode, http.StatusOK) + } + if res.Close { + t.Fatal("response before shutdown carries Connection: close") + } + return conn, br +} + +func TestShutdown_IdleGraceEndsWhenIdleConnsClose(t *testing.T) { + s := newTestServer(t, nil) + s.idleGracePeriod = 10 * time.Minute // must end early: client closes its conn + addr, cancel, done := startShutdownServer(t, s, s.handler()) + + conn, _ := dialIdleConn(t, addr) + + cancel() + waitListenerClosed(t, addr) + + // The idle connection is still open, so shutdown must keep waiting. + select { + case err := <-done: + t.Fatalf("serve returned %v while an idle connection was open", err) + case <-time.After(100 * time.Millisecond): + } + + conn.Close() + waitServeDone(t, done) // fails if the wait doesn't end early +} + +func TestShutdown_IdleGraceWaitsForAllIdleConns(t *testing.T) { + s := newTestServer(t, nil) + s.idleGracePeriod = 10 * time.Minute + addr, cancel, done := startShutdownServer(t, s, s.handler()) + + conn1, _ := dialIdleConn(t, addr) + conn2, _ := dialIdleConn(t, addr) + + cancel() + waitListenerClosed(t, addr) + + // Closing one of two idle connections must not end the wait. + conn1.Close() + select { + case err := <-done: + t.Fatalf("serve returned %v while an idle connection was open", err) + case <-time.After(100 * time.Millisecond): + } + + conn2.Close() + waitServeDone(t, done) +} + +func TestShutdown_IdleConnClosedAfterIdleGraceElapses(t *testing.T) { + s := newTestServer(t, nil) + s.idleGracePeriod = 100 * time.Millisecond + addr, cancel, done := startShutdownServer(t, s, s.handler()) + + conn, br := dialIdleConn(t, addr) + + cancel() + + // The client never uses or closes the idle connection, so once the + // idle grace period elapses the server closes it. + conn.SetReadDeadline(time.Now().Add(30 * time.Second)) + if _, err := br.ReadByte(); err != io.EOF { + t.Fatalf("read on abandoned idle conn = %v, want io.EOF", err) + } + waitServeDone(t, done) +} + +func TestShutdown_CompletesAndRefusesNewConnections(t *testing.T) { + run := func(t *testing.T, delay, idleGrace time.Duration) { + s := newTestServer(t, nil) + s.shutdownDelay = delay + s.idleGracePeriod = idleGrace + addr, cancel, done := startShutdownServer(t, s, s.handler()) + + getHealth(t, addr) + cancel() + waitServeDone(t, done) + + if conn, err := net.Dial("tcp", addr); err == nil { + conn.Close() + t.Fatal("dial succeeded after shutdown completed") + } + } + t.Run("short delays", func(t *testing.T) { run(t, 20*time.Millisecond, 20*time.Millisecond) }) + t.Run("zero delays", func(t *testing.T) { run(t, 0, 0) }) +} + +func TestShutdown_WaitsForInFlightRequest(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-release + io.WriteString(w, "done") + }) + s := &Server{shutdownDelay: 20 * time.Millisecond, idleGracePeriod: 20 * time.Millisecond} + addr, cancel, done := startShutdownServer(t, s, h) + + type result struct { + status int + body string + err error + } + resc := make(chan result, 1) + go func() { + res, err := http.Get("http://" + addr + "/") + if err != nil { + resc <- result{err: err} + return + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + resc <- result{status: res.StatusCode, body: string(body), err: err} + }() + + <-started + cancel() + waitListenerClosed(t, addr) + + // Shutdown must keep waiting while the request is in flight. + select { + case err := <-done: + t.Fatalf("serve returned %v before the in-flight request completed", err) + case <-time.After(100 * time.Millisecond): + } + + close(release) + r := <-resc + if r.err != nil { + t.Fatalf("in-flight request failed: %v", r.err) + } + if r.status != http.StatusOK || r.body != "done" { + t.Fatalf("in-flight request = %d %q, want %d %q", r.status, r.body, http.StatusOK, "done") + } + waitServeDone(t, done) +} + +func TestShutdown_TimeoutCompletesInTime(t *testing.T) { + s := newTestServer(t, nil) + s.shutdownTimeout = 10 * time.Minute // generous: must still exit cleanly + addr, cancel, done := startShutdownServer(t, s, s.handler()) + + getHealth(t, addr) + cancel() + waitServeDone(t, done) +} + +func TestShutdown_TimeoutForcesClose(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) // unblock the handler goroutine + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-release + }) + s := &Server{shutdownTimeout: 100 * time.Millisecond} + addr, cancel, done := startShutdownServer(t, s, h) + + errc := make(chan error, 1) + go func() { + res, err := http.Get("http://" + addr + "/") + if err == nil { + io.Copy(io.Discard, res.Body) + res.Body.Close() + } + errc <- err + }() + + <-started + cancel() + + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "graceful shutdown incomplete") { + t.Fatalf("serve returned %v, want graceful shutdown incomplete error", err) + } + case <-time.After(30 * time.Second): + t.Fatal("server did not shut down after the timeout") + } + // The stuck request's connection was force-closed without a response. + if err := <-errc; err == nil { + t.Fatal("stuck in-flight request succeeded, want connection error") + } +} + +// --- ListenAndServe / serve error-path tests --- + +func TestListenAndServe_CleanShutdown(t *testing.T) { + s := &Server{addr: "127.0.0.1:0"} + ctx, cancel := context.WithCancel(context.Background()) + cancel() // shutdown begins immediately; all phases have zero durations + if err := s.ListenAndServe(ctx); err != nil { + t.Fatalf("ListenAndServe = %v, want nil", err) + } +} + +func TestListenAndServe_BindError(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen: %v", err) + } + defer ln.Close() + + s := &Server{addr: ln.Addr().String()} + if err := s.ListenAndServe(context.Background()); err == nil { + t.Fatal("ListenAndServe on an occupied port succeeded, want error") + } +} + +func TestServe_ListenerErrorBeforeShutdown(t *testing.T) { + s := &Server{} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen: %v", err) + } + done := make(chan error, 1) + go func() { done <- s.serve(context.Background(), ln, s.handler()) }() + + // Killing the listener without canceling ctx makes srv.Serve fail; serve + // must return that error instead of waiting for shutdown. + ln.Close() + select { + case err := <-done: + if !errors.Is(err, net.ErrClosed) { + t.Fatalf("serve returned %v, want net.ErrClosed", err) + } + case <-time.After(30 * time.Second): + t.Fatal("serve did not return after listener close") + } +} + +// --- main() subprocess tests (shutdown flags and signal handling) --- + +// TestMain lets the test binary impersonate the gcsproxy binary: with +// GCSPROXY_RUN_MAIN=1 it runs main() on the test binary's arguments, so +// subprocess tests can exercise main's flag validation and startup errors +// for real, including exit codes. +func TestMain(m *testing.M) { + if os.Getenv("GCSPROXY_RUN_MAIN") == "1" { + main() + return + } + os.Exit(m.Run()) +} + +// runMain re-executes the test binary as gcsproxy and returns its stderr and +// exit code. STORAGE_EMULATOR_HOST lets storage.NewClient succeed without +// credentials; nothing ever connects to it. +func runMain(t *testing.T, args ...string) (string, int) { + t.Helper() + cmd := exec.Command(os.Args[0], args...) + cmd.Env = append(os.Environ(), "GCSPROXY_RUN_MAIN=1", "STORAGE_EMULATOR_HOST=127.0.0.1:1") + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Run() + if err == nil { + return stderr.String(), 0 + } + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Fatalf("running %v: %v", args, err) + } + return stderr.String(), ee.ExitCode() +} + +func TestMain_NegativeDurationValidation(t *testing.T) { + cases := []struct { + name string + args []string + }{ + {"negative shutdown delay", []string{"-shutdown-delay", "-1s"}}, + {"negative idle grace period", []string{"-shutdown-idle-grace-period", "-1s"}}, + {"negative shutdown timeout", []string{"-shutdown-timeout", "-1s"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stderr, code := runMain(t, tc.args...) + if code != 1 { + t.Errorf("exit code = %d, want 1 (stderr=%q)", code, stderr) + } + if !strings.Contains(stderr, "must not be negative") { + t.Errorf("stderr %q does not contain %q", stderr, "must not be negative") + } + }) + } +} + +func TestMain_BindFailure(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen: %v", err) + } + defer ln.Close() + + stderr, code := runMain(t, "-b", ln.Addr().String()) + if code != 1 { + t.Errorf("exit code = %d, want 1 (stderr=%q)", code, stderr) + } + if !strings.Contains(stderr, "server exited") { + t.Errorf("stderr %q does not contain %q", stderr, "server exited") + } +} + +func TestMain_ServesAndShutsDownOnSIGTERM(t *testing.T) { + cmd := exec.Command(os.Args[0], "-b", "127.0.0.1:0", "-log-format", "text") + // STORAGE_EMULATOR_HOST makes storage.NewClient skip credential lookup; + // nothing ever connects to it because only /_health is requested. + cmd.Env = append(os.Environ(), "GCSPROXY_RUN_MAIN=1", "STORAGE_EMULATOR_HOST=127.0.0.1:1") + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatalf("StderrPipe: %v", err) + } + if err := cmd.Start(); err != nil { + t.Fatalf("starting gcsproxy: %v", err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + + // The bound port is only known from the "listening" log line. + var addr string + scanner := bufio.NewScanner(stderr) + // addr must be checked before Scan: with the operands swapped, finding the + // address would still block on one more line the child never writes. + for addr == "" && scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, "msg=listening") { + continue + } + for _, f := range strings.Fields(line) { + if v, ok := strings.CutPrefix(f, "addr="); ok { + addr = v + } + } + } + if addr == "" { + t.Fatalf("no listening line on stderr (scanner err: %v)", scanner.Err()) + } + go io.Copy(io.Discard, stderr) // keep draining shutdown logs + + getHealth(t, addr) + + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatalf("SIGTERM: %v", err) + } + waited := make(chan error, 1) + go func() { waited <- cmd.Wait() }() + select { + case err := <-waited: + if err != nil { + t.Fatalf("gcsproxy exited with %v, want exit code 0", err) + } + case <-time.After(30 * time.Second): + t.Fatal("gcsproxy did not exit after SIGTERM") + } +}