diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index 0f73ecfea..c9b314924 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -557,9 +557,11 @@ func notifyDetachedTunnelTrackersOfUpstreamRemoval(upstream string, self *tunnel // Connection deletion unregisters empty trackers. Do not hold the // registry lock while acquiring tracker locks or closing connections. + // One deadline for every tracker; see closeConnectionsLocked. + deadline := time.Now().Add(closeControlWriteTimeout) var err error for _, tunnel := range trackers { - if closeErr := tunnel.closeConnectionsForUpstream(upstream); closeErr != nil && tunnel == self && err == nil { + if closeErr := tunnel.closeConnectionsForUpstream(upstream, deadline); closeErr != nil && tunnel == self && err == nil { err = closeErr } } @@ -1242,6 +1244,16 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe // http.Transport reads a newer value to dial a new connection when that new // value is updated by another reverse proxy handler, typically forward_auth. ctx := context.WithValue(req.Context(), dialInfoCtxKey, di) + // Record the upstream socket for the websocket close message, since an + // upgraded HTTP/1 body hides it; see upstreamWriteCloser. + if isWebsocket(req) { + upstreamConn := new(atomic.Pointer[net.Conn]) + trace.GotConn = func(info httptrace.GotConnInfo) { + conn := info.Conn + upstreamConn.Store(&conn) + } + ctx = context.WithValue(ctx, upstreamConnCtxKey, upstreamConn) + } req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) // do the round-trip diff --git a/modules/caddyhttp/reverseproxy/streaming.go b/modules/caddyhttp/reverseproxy/streaming.go index 125e329be..b702f13fb 100644 --- a/modules/caddyhttp/reverseproxy/streaming.go +++ b/modules/caddyhttp/reverseproxy/streaming.go @@ -29,6 +29,7 @@ import ( "net" "net/http" "sync" + "sync/atomic" "time" "unsafe" @@ -59,6 +60,38 @@ func (rwc extendedConnectReadWriteCloser) Write(p []byte) (n int, err error) { return n, nil } +// SetWriteDeadline bounds writes to the extended CONNECT stream, so a peer +// that stopped reading cannot block the close message forever. +func (rwc extendedConnectReadWriteCloser) SetWriteDeadline(deadline time.Time) error { + //nolint:bodyclose + return http.NewResponseController(rwc.ResponseWriter).SetWriteDeadline(deadline) +} + +// upstreamConnCtxKey is the context key for the *atomic.Pointer[net.Conn] +// recording the connection a websocket upgrade's round trip used. +const upstreamConnCtxKey caddy.CtxKey = "reverse_proxy.upstream_conn" + +// upstreamWriteCloser returns where to write the close message to the +// upstream. The write must be bounded by a deadline, but for HTTP/1 +// upgrades net/http hands the upstream socket back inside a body type that +// hides its deadline methods and writes straight through to the socket, so +// the close message goes to the socket recorded during the round trip +// instead. An HTTP/1 connection is dedicated to this request; HTTP/2 and +// HTTP/3 share theirs between streams, so their bodies are used as they are. +func upstreamWriteCloser(req *http.Request, res *http.Response, backConn io.ReadWriteCloser) io.ReadWriteCloser { + if res.ProtoMajor != 1 { + return backConn + } + recorded, ok := req.Context().Value(upstreamConnCtxKey).(*atomic.Pointer[net.Conn]) + if !ok { + return backConn + } + if uc := recorded.Load(); uc != nil { + return *uc + } + return backConn +} + func (h *Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWriter, req *http.Request, res *http.Response, upstreamAddr string) { // Until the tunnel takes ownership, every setup failure must close the // backend body and any hijacked client connection. net/http no longer @@ -208,16 +241,16 @@ func (h *Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWrit // need to know whether the connection is considered the server or the // client side of the proxy. Note that gracefulClose must not capture h, // since the tunnel may outlive the handler instance. - gracefulClose := func(conn io.ReadWriteCloser, isClient bool) func() error { + gracefulClose := func(conn io.ReadWriteCloser, isClient bool) func(time.Time) error { if isWebsocket(req) { - return func() error { - return writeCloseControl(conn, isClient) + return func(deadline time.Time) error { + return writeCloseControl(conn, isClient, deadline) } } return nil } deleteFrontConn := tunnel.registerConnection(conn, gracefulClose(conn, false), detached, upstreamAddr) - deleteBackConn := tunnel.registerConnection(backConn, gracefulClose(backConn, true), detached, upstreamAddr) + deleteBackConn := tunnel.registerConnection(backConn, gracefulClose(upstreamWriteCloser(req, res, backConn), true), detached, upstreamAddr) if h.streamLogsSkipHandshake() { caddyhttp.SetVar(req.Context(), caddyhttp.LogSkipVar, true) } @@ -521,7 +554,7 @@ func (h Handler) copyBuffer(dst io.Writer, src io.Reader, buf []byte, logger *za // closed when the upstream is removed from the config, not on every reload. type openConnection struct { conn io.ReadWriteCloser - gracefulClose func() error + gracefulClose func(deadline time.Time) error detached bool upstream string } @@ -549,7 +582,7 @@ func newTunnelTracker(logger *zap.Logger, closeDelay time.Duration) *tunnelTrack // registerConnection stores conn in the tracking map. The caller must invoke // the returned del func when the connection is done. -func (ts *tunnelTracker) registerConnection(conn io.ReadWriteCloser, gracefulClose func() error, detached bool, upstream string) (del func()) { +func (ts *tunnelTracker) registerConnection(conn io.ReadWriteCloser, gracefulClose func(deadline time.Time) error, detached bool, upstream string) (del func()) { ts.mu.Lock() ts.connections[conn] = openConnection{conn, gracefulClose, detached, upstream} ts.mu.Unlock() @@ -574,16 +607,27 @@ func (ts *tunnelTracker) registerConnection(conn io.ReadWriteCloser, gracefulClo // closeAttachedConnections closes all tracked attached connections. func (ts *tunnelTracker) closeAttachedConnections() error { - var err error ts.mu.Lock() defer ts.mu.Unlock() + // detached connections are only closed when the upstream is gone from the config + return ts.closeConnectionsLocked(time.Now().Add(closeControlWriteTimeout), func(oc openConnection) bool { + return !oc.detached + }) +} + +// closeConnectionsLocked sends a close message to, then closes, every +// tracked connection that match selects. The caller must hold mu. Every +// graceful close shares one absolute deadline: the loop is serial and holds +// the lock, so per-connection timeouts would add up across stalled peers. +// The first error is returned. +func (ts *tunnelTracker) closeConnectionsLocked(deadline time.Time, match func(openConnection) bool) error { + var err error for _, oc := range ts.connections { - // detached connections are only closed when the upstream is gone from the config - if oc.detached { + if !match(oc) { continue } if oc.gracefulClose != nil { - if gracefulErr := oc.gracefulClose(); gracefulErr != nil && err == nil { + if gracefulErr := oc.gracefulClose(deadline); gracefulErr != nil && err == nil { err = gracefulErr } } @@ -635,7 +679,24 @@ func (ts *tunnelTracker) cleanupAttachedConnections() error { // from his simple implementation of this I was able to learn from at: // github.com/pascaldekloe/websocket. Further work for handling masking // taken from github.com/gorilla/websocket. -func writeCloseControl(conn io.Writer, isClient bool) error { +// +// The write must finish by deadline: a peer that stopped reading leaves the +// tunnel's copy goroutine stuck writing, and the close message would wait +// behind it forever while the caller holds the tunnel tracker lock. The +// connection is closed right after, so the deadline needs no reset. +// Connections that cannot set a write deadline are skipped instead. +func writeCloseControl(conn io.Writer, isClient bool, deadline time.Time) error { + dc, ok := conn.(interface{ SetWriteDeadline(time.Time) error }) + if !ok { + return nil + } + if err := dc.SetWriteDeadline(deadline); err != nil { + if errors.Is(err, http.ErrNotSupported) { + return nil + } + return err + } + // Sources: // https://github.com/pascaldekloe/websocket/blob/32050af67a5d/websocket.go#L119 // https://github.com/gorilla/websocket/blob/v1.5.0/conn.go#L413 @@ -685,6 +746,11 @@ func writeCloseControl(conn io.Writer, isClient bool) error { return err } +// closeControlWriteTimeout is how long one batch of graceful closes waits +// for peers to accept their close messages before giving up on them. It is +// a variable so tests can shorten it. +var closeControlWriteTimeout = 5 * time.Second + // Copied from https://github.com/gorilla/websocket/blob/v1.5.0/mask.go func maskBytes(key [4]byte, pos int, b []byte) int { // Mask one byte at a time for small buffers. @@ -744,27 +810,17 @@ func isWebsocket(r *http.Request) bool { // closeConnectionsForUpstream closes detached connections to the given // upstream on stopped trackers. Attached connections keep their close delay. -func (ts *tunnelTracker) closeConnectionsForUpstream(addr string) error { - var err error +// The deadline is shared so one upstream removal is bounded across every +// tracker it notifies. +func (ts *tunnelTracker) closeConnectionsForUpstream(addr string, deadline time.Time) error { ts.mu.Lock() defer ts.mu.Unlock() if !ts.stopped { return nil } - for _, oc := range ts.connections { - if !oc.detached || oc.upstream != addr { - continue - } - if oc.gracefulClose != nil { - if gracefulErr := oc.gracefulClose(); gracefulErr != nil && err == nil { - err = gracefulErr - } - } - if closeErr := oc.conn.Close(); closeErr != nil && err == nil { - err = closeErr - } - } - return err + return ts.closeConnectionsLocked(deadline, func(oc openConnection) bool { + return oc.detached && oc.upstream == addr + }) } type maxLatencyWriter struct { diff --git a/modules/caddyhttp/reverseproxy/streaming_test.go b/modules/caddyhttp/reverseproxy/streaming_test.go index 5b239a64d..83b74b2a3 100644 --- a/modules/caddyhttp/reverseproxy/streaming_test.go +++ b/modules/caddyhttp/reverseproxy/streaming_test.go @@ -10,9 +10,11 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "runtime" "strings" "sync" + "sync/atomic" "testing" "time" @@ -441,88 +443,95 @@ func TestHandlerUpgradedStreamHalfClose(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - srvc := make(chan *net.TCPConn, 1) - - backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - conn, _, err := http.NewResponseController(w).Hijack() - if err != nil { - t.Errorf("backend hijack failed: %v", err) - return - } - tcp, ok := conn.(*net.TCPConn) - if !ok { - conn.Close() - t.Errorf("backend conn is %T, want *net.TCPConn", conn) - return - } - if _, err := io.WriteString(tcp, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil { - tcp.Close() - t.Errorf("backend upgrade write failed: %v", err) - return - } - srvc <- tcp - })) - defer backend.Close() - - h := minimalHandler(0, &Upstream{ - Host: new(Host), - Dial: backend.Listener.Addr().String(), - }) - h.tunnelTracker = newTunnelTracker(caddy.Log(), 0) - initReverseProxyMetrics(h, prometheus.NewRegistry()) - - frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Isolate half-close propagation from request cancellation which has - // its own backend-close path. - r = r.WithContext(context.WithoutCancel(r.Context())) - _ = h.ServeHTTP(w, prepareTestRequest(r), caddyhttp.HandlerFunc(func(http.ResponseWriter, *http.Request) error { - return nil - })) - })) - defer frontend.Close() - - frontendURL, err := url.Parse(frontend.URL) - if err != nil { - t.Fatalf("failed to parse frontend URL: %v", err) - } - addr, err := net.ResolveTCPAddr("tcp", frontendURL.Host) - if err != nil { - t.Fatalf("failed to resolve TCP address: %v", err) - } - cli, err := net.DialTCP("tcp", nil, addr) - if err != nil { - t.Fatalf("failed to dial frontend: %v", err) - } - defer cli.Close() - - req, _ := http.NewRequest(http.MethodGet, frontend.URL, nil) - req.Header.Set("Connection", "Upgrade") - req.Header.Set("Upgrade", "websocket") - if err := req.Write(cli); err != nil { - t.Fatalf("failed to write upgrade request: %v", err) - } - - resp, err := http.ReadResponse(bufio.NewReader(cli), &http.Request{Method: http.MethodGet}) - if err != nil { - t.Fatalf("failed to read upgrade response: %v", err) - } - if resp.StatusCode != http.StatusSwitchingProtocols { - t.Fatalf("status = %d, want 101", resp.StatusCode) - } - - var srv *net.TCPConn - select { - case srv = <-srvc: - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for the backend connection") - } - defer srv.Close() - + _, cli, srv := newUpgradedTunnel(t) test(t, cli, srv) }) } } +// newUpgradedTunnel proxies a websocket upgrade from a raw client connection +// through a Handler to a hijacking backend and returns both ends of the +// tunnel. The Handler's tunnel tracker holds the proxied connections. +func newUpgradedTunnel(t *testing.T) (h *Handler, cli, srv *net.TCPConn) { + t.Helper() + srvc := make(chan *net.TCPConn, 1) + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, _, err := http.NewResponseController(w).Hijack() + if err != nil { + t.Errorf("backend hijack failed: %v", err) + return + } + tcp, ok := conn.(*net.TCPConn) + if !ok { + conn.Close() + t.Errorf("backend conn is %T, want *net.TCPConn", conn) + return + } + if _, err := io.WriteString(tcp, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil { + tcp.Close() + t.Errorf("backend upgrade write failed: %v", err) + return + } + srvc <- tcp + })) + t.Cleanup(backend.Close) + + h = minimalHandler(0, &Upstream{ + Host: new(Host), + Dial: backend.Listener.Addr().String(), + }) + h.tunnelTracker = newTunnelTracker(caddy.Log(), 0) + initReverseProxyMetrics(h, prometheus.NewRegistry()) + + frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Isolate the tunnel from request cancellation, which has its own + // backend-close path. + r = r.WithContext(context.WithoutCancel(r.Context())) + _ = h.ServeHTTP(w, prepareTestRequest(r), caddyhttp.HandlerFunc(func(http.ResponseWriter, *http.Request) error { + return nil + })) + })) + t.Cleanup(frontend.Close) + + frontendURL, err := url.Parse(frontend.URL) + if err != nil { + t.Fatalf("failed to parse frontend URL: %v", err) + } + addr, err := net.ResolveTCPAddr("tcp", frontendURL.Host) + if err != nil { + t.Fatalf("failed to resolve TCP address: %v", err) + } + cli, err = net.DialTCP("tcp", nil, addr) + if err != nil { + t.Fatalf("failed to dial frontend: %v", err) + } + t.Cleanup(func() { cli.Close() }) + + req, _ := http.NewRequest(http.MethodGet, frontend.URL, nil) + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + if err := req.Write(cli); err != nil { + t.Fatalf("failed to write upgrade request: %v", err) + } + + resp, err := http.ReadResponse(bufio.NewReader(cli), &http.Request{Method: http.MethodGet}) + if err != nil { + t.Fatalf("failed to read upgrade response: %v", err) + } + if resp.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("status = %d, want 101", resp.StatusCode) + } + + select { + case srv = <-srvc: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the backend connection") + } + t.Cleanup(func() { srv.Close() }) + return h, cli, srv +} + // A response carrying the Incremental header field (RFC 10036) must be // forwarded without buffering, whatever its Content-Type and Content-Length. func TestFlushIntervalIncremental(t *testing.T) { @@ -617,7 +626,7 @@ func TestDetachedTrackerRemovalAllowsConcurrentConnectionDeletion(t *testing.T) } deleted := make(chan struct{}) conn := newTrackingReadWriteCloser() - del := ts.registerConnection(conn, func() error { + del := ts.registerConnection(conn, func(time.Time) error { go func() { deleteOther() close(deleted) @@ -644,6 +653,228 @@ func TestDetachedTrackerRemovalAllowsConcurrentConnectionDeletion(t *testing.T) } } +// newStalledWebsocketPeer returns one end of a pipe whose other end never +// reads, with a goroutine stuck writing to it the way a tunnel's copy loop +// would be. net.Pipe is unbuffered and lets only one write through at a +// time, so a close message written to conn waits behind that stuck write, +// just as it does on a socket whose peer stopped reading. +func newStalledWebsocketPeer(t *testing.T) net.Conn { + t.Helper() + conn, peer := net.Pipe() + t.Cleanup(func() { + conn.Close() + peer.Close() + }) + go func() { _, _ = conn.Write([]byte("stalled")) }() + return conn +} + +// shortenCloseControlWriteTimeout makes a stalled peer time out quickly for +// the rest of the test. +func shortenCloseControlWriteTimeout(t *testing.T) { + t.Helper() + saved := closeControlWriteTimeout + closeControlWriteTimeout = 10 * time.Millisecond + t.Cleanup(func() { closeControlWriteTimeout = saved }) +} + +func TestHandlerCleanupDoesNotBlockOnStalledWebsocketPeer(t *testing.T) { + conn := newStalledWebsocketPeer(t) + + ts := newTunnelTracker(caddy.Log(), 0) + shortenCloseControlWriteTimeout(t) + ts.registerConnection(conn, func(deadline time.Time) error { + return writeCloseControl(conn, false, deadline) + }, false, "upstream") + + cleanupDone := make(chan error, 1) + go func() { cleanupDone <- ts.cleanupAttachedConnections() }() + + select { + case err := <-cleanupDone: + if !errors.Is(err, os.ErrDeadlineExceeded) { + t.Errorf("cleanup = %v, want %v", err, os.ErrDeadlineExceeded) + } + case <-time.After(10 * time.Second): + t.Fatal("cleanup blocked on a websocket peer that stopped reading") + } +} + +func TestHandlerCleanupUsesOneDeadlinePerBatch(t *testing.T) { + // The close loop is serial and holds the tracker lock, so a deadline per + // connection would let every stalled peer add another full timeout. + ts := newTunnelTracker(caddy.Log(), 0) + shortenCloseControlWriteTimeout(t) + var deadlines []time.Time + for range 2 { + conn := newStalledWebsocketPeer(t) + ts.registerConnection(conn, func(deadline time.Time) error { + deadlines = append(deadlines, deadline) + return writeCloseControl(conn, false, deadline) + }, false, "upstream") + } + + _ = ts.cleanupAttachedConnections() + if len(deadlines) != 2 { + t.Fatalf("got %d graceful closes, want 2", len(deadlines)) + } + if !deadlines[0].Equal(deadlines[1]) { + t.Errorf("graceful closes got deadlines %v and %v, want one shared deadline", deadlines[0], deadlines[1]) + } +} + +func TestUpstreamRemovalUsesOneDeadlineAcrossTrackers(t *testing.T) { + // Removing an upstream notifies every stopped tracker in turn, and each + // close loop holds its own lock, so a deadline per tracker would let + // every tracker with a stalled peer add another full timeout. + const upstream = "upstream-removal-one-deadline" + shortenCloseControlWriteTimeout(t) + var deadlines []time.Time + var trackers []*tunnelTracker + for range 2 { + ts := newTunnelTracker(caddy.Log(), 0) + registerDetachedTunnelTrackers(ts) + t.Cleanup(func() { unregisterDetachedTunnelTrackers(ts) }) + conn := newStalledWebsocketPeer(t) + ts.registerConnection(conn, func(deadline time.Time) error { + deadlines = append(deadlines, deadline) + return writeCloseControl(conn, false, deadline) + }, true, upstream) + if err := ts.cleanupAttachedConnections(); err != nil { + t.Fatal(err) + } + trackers = append(trackers, ts) + } + + _ = notifyDetachedTunnelTrackersOfUpstreamRemoval(upstream, trackers[0]) + if len(deadlines) != 2 { + t.Fatalf("got %d graceful closes, want 2", len(deadlines)) + } + if !deadlines[0].Equal(deadlines[1]) { + t.Errorf("trackers got deadlines %v and %v, want one shared deadline", deadlines[0], deadlines[1]) + } +} + +func TestWriteCloseControlSkipsConnectionsWithoutWriteDeadline(t *testing.T) { + // A write that cannot be bounded could block forever behind a stalled + // peer, so the best-effort close message is skipped instead. + var buf bytes.Buffer + if err := writeCloseControl(&buf, false, time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + if buf.Len() != 0 { + t.Errorf("close message %q was written to a connection without write deadlines", buf.Bytes()) + } +} + +func TestExtendedConnectReadWriteCloserSetWriteDeadline(t *testing.T) { + rw := &writeDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + rwc := extendedConnectReadWriteCloser{io.NopCloser(strings.NewReader("")), rw} + deadline := time.Now().Add(time.Second) + if err := rwc.SetWriteDeadline(deadline); err != nil { + t.Fatal(err) + } + if !rw.writeDeadline.Equal(deadline) { + t.Errorf("write deadline = %v, want %v", rw.writeDeadline, deadline) + } + + // Writers without deadline support report ErrNotSupported, which + // writeCloseControl treats as a reason to skip the close message. + plain := extendedConnectReadWriteCloser{io.NopCloser(strings.NewReader("")), httptest.NewRecorder()} + if err := plain.SetWriteDeadline(deadline); !errors.Is(err, http.ErrNotSupported) { + t.Errorf("SetWriteDeadline on a plain recorder = %v, want ErrNotSupported", err) + } +} + +type writeDeadlineRecorder struct { + *httptest.ResponseRecorder + writeDeadline time.Time +} + +func (r *writeDeadlineRecorder) SetWriteDeadline(deadline time.Time) error { + r.writeDeadline = deadline + return nil +} + +func TestHandlerUpgradeBoundsUpstreamCloseMessage(t *testing.T) { + switch runtime.GOOS { + case "plan9", "js", "wasip1": + t.Skipf("not supported on %s", runtime.GOOS) + } + + // net/http hands upgraded HTTP/1 bodies back inside a type that hides the + // socket's deadline methods, so without the connection recorded during + // the round trip the close message to the upstream would be skipped. + h, cli, srv := newUpgradedTunnel(t) + if err := cli.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } + if err := srv.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } + + // Data reaching the backend proves both connections are registered. + if _, err := io.WriteString(cli, "hello"); err != nil { + t.Fatal(err) + } + hello := make([]byte, len("hello")) + if _, err := io.ReadFull(srv, hello); err != nil || string(hello) != "hello" { + t.Fatalf("backend read %q, %v; want %q", hello, err, "hello") + } + + // Send the close messages without closing anything. Cleanup closes the + // connections in map order, and closing the client side first can tear + // down the tunnel before the upstream gets its close message. + deadline := time.Now().Add(5 * time.Second) + h.tunnelTracker.mu.Lock() + for _, oc := range h.tunnelTracker.connections { + if err := oc.gracefulClose(deadline); err != nil { + t.Errorf("graceful close = %v", err) + } + } + h.tunnelTracker.mu.Unlock() + frame := make([]byte, 2) + if _, err := io.ReadFull(srv, frame); err != nil { + t.Fatalf("backend did not receive a close message: %v", err) + } + if frame[0] != 0x88 || frame[1]&0x80 == 0 { + t.Fatalf("backend received %x, want a masked close frame", frame) + } +} + +func TestUpstreamWriteCloser(t *testing.T) { + // The HTTP/1 path, including the recorded socket, is covered end to end + // by TestHandlerUpgradeBoundsUpstreamCloseMessage. HTTP/2 and HTTP/3 share + // their connection between streams, so the close message must never go + // to a recorded socket for them. + body, _ := net.Pipe() + socket, _ := net.Pipe() + for _, tc := range []struct { + name string + protoMajor int + recorded net.Conn + want io.ReadWriteCloser + }{ + {name: "http/1 with recorded socket", protoMajor: 1, recorded: socket, want: socket}, + {name: "http/1 without recorded socket", protoMajor: 1, want: body}, + {name: "http/2", protoMajor: 2, recorded: socket, want: body}, + {name: "http/3", protoMajor: 3, recorded: socket, want: body}, + } { + t.Run(tc.name, func(t *testing.T) { + recorded := new(atomic.Pointer[net.Conn]) + if tc.recorded != nil { + recorded.Store(&tc.recorded) + } + req := httptest.NewRequest(http.MethodGet, "/", nil) + req = req.WithContext(context.WithValue(req.Context(), upstreamConnCtxKey, recorded)) + res := &http.Response{ProtoMajor: tc.protoMajor} + if got := upstreamWriteCloser(req, res, body); got != tc.want { + t.Errorf("upstreamWriteCloser() = %v, want %v", got, tc.want) + } + }) + } +} + // opaqueUpgradeWriter supports hijacking but hides the recorder from the // detachment helper, as a middleware wrapper without Unwrap would do. type opaqueUpgradeWriter struct {