reverseproxy: set a write deadline on websocket close messages (#8309)
* reverseproxy: set a write deadline on websocket close messages Cleanup sends a Close control message to every attached websocket while holding the tunnel tracker lock, during a config reload. If the peer has stopped reading, the tunnel's copy goroutine is blocked writing to it and the close message waits behind that write forever. The reload never finishes and holds the config lock, so every later /load and /config request hangs until Caddy is restarted. Setting a write deadline before the close message unblocks both writes. The connection is closed right after, so the deadline is never cleared. * Add bot disclosure * reverseproxy: make close control timeout a const, harden stalled-peer test * Delete BOT_DISCLOSURE.md Signed-off-by: Francis Lavoie <lavofr@gmail.com> * reverseproxy: bound each batch of websocket close messages by one deadline The close loops are serial and hold the tunnel tracker lock, so a per-connection write timeout let N stalled peers hold a reload for N times the timeout. Compute one absolute deadline per batch instead, and skip the best-effort close frame on connections that cannot set a write deadline rather than risk the hang. Extended CONNECT streams gain SetWriteDeadline through http.ResponseController. For HTTP/1 upgrades, net/http returns the upstream socket inside a body that hides its deadline methods, so the upstream close message was never bounded; record the connection from the request's ClientTrace and write the close message to that socket. Restores the TODO about being the exclusive writer, per review. * reverseproxy: make websocket close tests deterministic Per review, the stalled-peer tests use net.Pipe instead of real TCP: it is unbuffered and lets one write through at a time, so a close message waits behind a stuck copy write without large allocations or sleeps. The batch test checks that graceful closes share one deadline instead of timing cleanup. The HTTP/1 upstream close test now runs a real upgrade through ServeHTTP and the transport, so it fails if the GotConn hook is removed. The choice of upstream close writer moves into upstreamCloseWriter, with a table test for the HTTP/2 and HTTP/3 case. * reverseproxy: record the upstream conn with atomic.Pointer Per review, store the connection from the GotConn hook in an atomic.Pointer[net.Conn] instead of a mutex-guarded holder type. Rename upstreamCloseWriter to upstreamWriteCloser. It returns io.ReadWriteCloser, since both the body and the recorded socket are one, so gracefulClose keeps its original io.ReadWriteCloser parameter. * reverseproxy: share one close deadline across detached trackers Per review, an upstream removal notifies every stopped tracker in turn, and each close loop holds its own lock, so a deadline per tracker let N trackers with stalled peers hold a reload for N times the timeout. Compute one absolute deadline in notifyDetachedTunnelTrackersOfUpstreamRemoval and pass it through to closeConnectionsForUpstream. Add a test with two stopped trackers holding stalled net.Pipe peers that asserts both graceful closes receive the same deadline. * reverseproxy: simplify the websocket close deadline change Fold the two tunnel close loops into closeConnectionsLocked, so the shared-deadline rule and its rationale live in one place. Replace the per-tracker closeWriteTimeout field, which production never varied, with a package variable that tests shorten. Record the upstream socket only for websocket requests, since only their graceful close reads it, and store a copy of the connection rather than the address of the hook's parameter. Use the package's caddy.CtxKey convention for the context key. Share the upgrade tunnel fixture between the half-close test and the upstream close message test.
This commit is contained in:
parent
0da7b08e81
commit
b3d67c861d
3 changed files with 405 additions and 106 deletions
|
|
@ -557,9 +557,11 @@ func notifyDetachedTunnelTrackersOfUpstreamRemoval(upstream string, self *tunnel
|
|||
|
||||
// Connection deletion unregisters empty trackers. Do not hold the
|
||||
// registry lock while acquiring tracker locks or closing connections.
|
||||
// One deadline for every tracker; see closeConnectionsLocked.
|
||||
deadline := time.Now().Add(closeControlWriteTimeout)
|
||||
var err error
|
||||
for _, tunnel := range trackers {
|
||||
if closeErr := tunnel.closeConnectionsForUpstream(upstream); closeErr != nil && tunnel == self && err == nil {
|
||||
if closeErr := tunnel.closeConnectionsForUpstream(upstream, deadline); closeErr != nil && tunnel == self && err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
|
|
@ -1242,6 +1244,16 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
|
|||
// http.Transport reads a newer value to dial a new connection when that new
|
||||
// value is updated by another reverse proxy handler, typically forward_auth.
|
||||
ctx := context.WithValue(req.Context(), dialInfoCtxKey, di)
|
||||
// Record the upstream socket for the websocket close message, since an
|
||||
// upgraded HTTP/1 body hides it; see upstreamWriteCloser.
|
||||
if isWebsocket(req) {
|
||||
upstreamConn := new(atomic.Pointer[net.Conn])
|
||||
trace.GotConn = func(info httptrace.GotConnInfo) {
|
||||
conn := info.Conn
|
||||
upstreamConn.Store(&conn)
|
||||
}
|
||||
ctx = context.WithValue(ctx, upstreamConnCtxKey, upstreamConn)
|
||||
}
|
||||
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
|
||||
|
||||
// do the round-trip
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
|
|
@ -59,6 +60,38 @@ func (rwc extendedConnectReadWriteCloser) Write(p []byte) (n int, err error) {
|
|||
return n, nil
|
||||
}
|
||||
|
||||
// SetWriteDeadline bounds writes to the extended CONNECT stream, so a peer
|
||||
// that stopped reading cannot block the close message forever.
|
||||
func (rwc extendedConnectReadWriteCloser) SetWriteDeadline(deadline time.Time) error {
|
||||
//nolint:bodyclose
|
||||
return http.NewResponseController(rwc.ResponseWriter).SetWriteDeadline(deadline)
|
||||
}
|
||||
|
||||
// upstreamConnCtxKey is the context key for the *atomic.Pointer[net.Conn]
|
||||
// recording the connection a websocket upgrade's round trip used.
|
||||
const upstreamConnCtxKey caddy.CtxKey = "reverse_proxy.upstream_conn"
|
||||
|
||||
// upstreamWriteCloser returns where to write the close message to the
|
||||
// upstream. The write must be bounded by a deadline, but for HTTP/1
|
||||
// upgrades net/http hands the upstream socket back inside a body type that
|
||||
// hides its deadline methods and writes straight through to the socket, so
|
||||
// the close message goes to the socket recorded during the round trip
|
||||
// instead. An HTTP/1 connection is dedicated to this request; HTTP/2 and
|
||||
// HTTP/3 share theirs between streams, so their bodies are used as they are.
|
||||
func upstreamWriteCloser(req *http.Request, res *http.Response, backConn io.ReadWriteCloser) io.ReadWriteCloser {
|
||||
if res.ProtoMajor != 1 {
|
||||
return backConn
|
||||
}
|
||||
recorded, ok := req.Context().Value(upstreamConnCtxKey).(*atomic.Pointer[net.Conn])
|
||||
if !ok {
|
||||
return backConn
|
||||
}
|
||||
if uc := recorded.Load(); uc != nil {
|
||||
return *uc
|
||||
}
|
||||
return backConn
|
||||
}
|
||||
|
||||
func (h *Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWriter, req *http.Request, res *http.Response, upstreamAddr string) {
|
||||
// Until the tunnel takes ownership, every setup failure must close the
|
||||
// backend body and any hijacked client connection. net/http no longer
|
||||
|
|
@ -208,16 +241,16 @@ func (h *Handler) handleUpgradeResponse(logger *zap.Logger, rw http.ResponseWrit
|
|||
// need to know whether the connection is considered the server or the
|
||||
// client side of the proxy. Note that gracefulClose must not capture h,
|
||||
// since the tunnel may outlive the handler instance.
|
||||
gracefulClose := func(conn io.ReadWriteCloser, isClient bool) func() error {
|
||||
gracefulClose := func(conn io.ReadWriteCloser, isClient bool) func(time.Time) error {
|
||||
if isWebsocket(req) {
|
||||
return func() error {
|
||||
return writeCloseControl(conn, isClient)
|
||||
return func(deadline time.Time) error {
|
||||
return writeCloseControl(conn, isClient, deadline)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
deleteFrontConn := tunnel.registerConnection(conn, gracefulClose(conn, false), detached, upstreamAddr)
|
||||
deleteBackConn := tunnel.registerConnection(backConn, gracefulClose(backConn, true), detached, upstreamAddr)
|
||||
deleteBackConn := tunnel.registerConnection(backConn, gracefulClose(upstreamWriteCloser(req, res, backConn), true), detached, upstreamAddr)
|
||||
if h.streamLogsSkipHandshake() {
|
||||
caddyhttp.SetVar(req.Context(), caddyhttp.LogSkipVar, true)
|
||||
}
|
||||
|
|
@ -521,7 +554,7 @@ func (h Handler) copyBuffer(dst io.Writer, src io.Reader, buf []byte, logger *za
|
|||
// closed when the upstream is removed from the config, not on every reload.
|
||||
type openConnection struct {
|
||||
conn io.ReadWriteCloser
|
||||
gracefulClose func() error
|
||||
gracefulClose func(deadline time.Time) error
|
||||
detached bool
|
||||
upstream string
|
||||
}
|
||||
|
|
@ -549,7 +582,7 @@ func newTunnelTracker(logger *zap.Logger, closeDelay time.Duration) *tunnelTrack
|
|||
|
||||
// registerConnection stores conn in the tracking map. The caller must invoke
|
||||
// the returned del func when the connection is done.
|
||||
func (ts *tunnelTracker) registerConnection(conn io.ReadWriteCloser, gracefulClose func() error, detached bool, upstream string) (del func()) {
|
||||
func (ts *tunnelTracker) registerConnection(conn io.ReadWriteCloser, gracefulClose func(deadline time.Time) error, detached bool, upstream string) (del func()) {
|
||||
ts.mu.Lock()
|
||||
ts.connections[conn] = openConnection{conn, gracefulClose, detached, upstream}
|
||||
ts.mu.Unlock()
|
||||
|
|
@ -574,16 +607,27 @@ func (ts *tunnelTracker) registerConnection(conn io.ReadWriteCloser, gracefulClo
|
|||
|
||||
// closeAttachedConnections closes all tracked attached connections.
|
||||
func (ts *tunnelTracker) closeAttachedConnections() error {
|
||||
var err error
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
// detached connections are only closed when the upstream is gone from the config
|
||||
return ts.closeConnectionsLocked(time.Now().Add(closeControlWriteTimeout), func(oc openConnection) bool {
|
||||
return !oc.detached
|
||||
})
|
||||
}
|
||||
|
||||
// closeConnectionsLocked sends a close message to, then closes, every
|
||||
// tracked connection that match selects. The caller must hold mu. Every
|
||||
// graceful close shares one absolute deadline: the loop is serial and holds
|
||||
// the lock, so per-connection timeouts would add up across stalled peers.
|
||||
// The first error is returned.
|
||||
func (ts *tunnelTracker) closeConnectionsLocked(deadline time.Time, match func(openConnection) bool) error {
|
||||
var err error
|
||||
for _, oc := range ts.connections {
|
||||
// detached connections are only closed when the upstream is gone from the config
|
||||
if oc.detached {
|
||||
if !match(oc) {
|
||||
continue
|
||||
}
|
||||
if oc.gracefulClose != nil {
|
||||
if gracefulErr := oc.gracefulClose(); gracefulErr != nil && err == nil {
|
||||
if gracefulErr := oc.gracefulClose(deadline); gracefulErr != nil && err == nil {
|
||||
err = gracefulErr
|
||||
}
|
||||
}
|
||||
|
|
@ -635,7 +679,24 @@ func (ts *tunnelTracker) cleanupAttachedConnections() error {
|
|||
// from his simple implementation of this I was able to learn from at:
|
||||
// github.com/pascaldekloe/websocket. Further work for handling masking
|
||||
// taken from github.com/gorilla/websocket.
|
||||
func writeCloseControl(conn io.Writer, isClient bool) error {
|
||||
//
|
||||
// The write must finish by deadline: a peer that stopped reading leaves the
|
||||
// tunnel's copy goroutine stuck writing, and the close message would wait
|
||||
// behind it forever while the caller holds the tunnel tracker lock. The
|
||||
// connection is closed right after, so the deadline needs no reset.
|
||||
// Connections that cannot set a write deadline are skipped instead.
|
||||
func writeCloseControl(conn io.Writer, isClient bool, deadline time.Time) error {
|
||||
dc, ok := conn.(interface{ SetWriteDeadline(time.Time) error })
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := dc.SetWriteDeadline(deadline); err != nil {
|
||||
if errors.Is(err, http.ErrNotSupported) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Sources:
|
||||
// https://github.com/pascaldekloe/websocket/blob/32050af67a5d/websocket.go#L119
|
||||
// https://github.com/gorilla/websocket/blob/v1.5.0/conn.go#L413
|
||||
|
|
@ -685,6 +746,11 @@ func writeCloseControl(conn io.Writer, isClient bool) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// closeControlWriteTimeout is how long one batch of graceful closes waits
|
||||
// for peers to accept their close messages before giving up on them. It is
|
||||
// a variable so tests can shorten it.
|
||||
var closeControlWriteTimeout = 5 * time.Second
|
||||
|
||||
// Copied from https://github.com/gorilla/websocket/blob/v1.5.0/mask.go
|
||||
func maskBytes(key [4]byte, pos int, b []byte) int {
|
||||
// Mask one byte at a time for small buffers.
|
||||
|
|
@ -744,27 +810,17 @@ func isWebsocket(r *http.Request) bool {
|
|||
|
||||
// closeConnectionsForUpstream closes detached connections to the given
|
||||
// upstream on stopped trackers. Attached connections keep their close delay.
|
||||
func (ts *tunnelTracker) closeConnectionsForUpstream(addr string) error {
|
||||
var err error
|
||||
// The deadline is shared so one upstream removal is bounded across every
|
||||
// tracker it notifies.
|
||||
func (ts *tunnelTracker) closeConnectionsForUpstream(addr string, deadline time.Time) error {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
if !ts.stopped {
|
||||
return nil
|
||||
}
|
||||
for _, oc := range ts.connections {
|
||||
if !oc.detached || oc.upstream != addr {
|
||||
continue
|
||||
}
|
||||
if oc.gracefulClose != nil {
|
||||
if gracefulErr := oc.gracefulClose(); gracefulErr != nil && err == nil {
|
||||
err = gracefulErr
|
||||
}
|
||||
}
|
||||
if closeErr := oc.conn.Close(); closeErr != nil && err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
return ts.closeConnectionsLocked(deadline, func(oc openConnection) bool {
|
||||
return oc.detached && oc.upstream == addr
|
||||
})
|
||||
}
|
||||
|
||||
type maxLatencyWriter struct {
|
||||
|
|
|
|||
|
|
@ -10,9 +10,11 @@ import (
|
|||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -441,88 +443,95 @@ func TestHandlerUpgradedStreamHalfClose(t *testing.T) {
|
|||
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
srvc := make(chan *net.TCPConn, 1)
|
||||
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, _, err := http.NewResponseController(w).Hijack()
|
||||
if err != nil {
|
||||
t.Errorf("backend hijack failed: %v", err)
|
||||
return
|
||||
}
|
||||
tcp, ok := conn.(*net.TCPConn)
|
||||
if !ok {
|
||||
conn.Close()
|
||||
t.Errorf("backend conn is %T, want *net.TCPConn", conn)
|
||||
return
|
||||
}
|
||||
if _, err := io.WriteString(tcp, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil {
|
||||
tcp.Close()
|
||||
t.Errorf("backend upgrade write failed: %v", err)
|
||||
return
|
||||
}
|
||||
srvc <- tcp
|
||||
}))
|
||||
defer backend.Close()
|
||||
|
||||
h := minimalHandler(0, &Upstream{
|
||||
Host: new(Host),
|
||||
Dial: backend.Listener.Addr().String(),
|
||||
})
|
||||
h.tunnelTracker = newTunnelTracker(caddy.Log(), 0)
|
||||
initReverseProxyMetrics(h, prometheus.NewRegistry())
|
||||
|
||||
frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Isolate half-close propagation from request cancellation which has
|
||||
// its own backend-close path.
|
||||
r = r.WithContext(context.WithoutCancel(r.Context()))
|
||||
_ = h.ServeHTTP(w, prepareTestRequest(r), caddyhttp.HandlerFunc(func(http.ResponseWriter, *http.Request) error {
|
||||
return nil
|
||||
}))
|
||||
}))
|
||||
defer frontend.Close()
|
||||
|
||||
frontendURL, err := url.Parse(frontend.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse frontend URL: %v", err)
|
||||
}
|
||||
addr, err := net.ResolveTCPAddr("tcp", frontendURL.Host)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to resolve TCP address: %v", err)
|
||||
}
|
||||
cli, err := net.DialTCP("tcp", nil, addr)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to dial frontend: %v", err)
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
req, _ := http.NewRequest(http.MethodGet, frontend.URL, nil)
|
||||
req.Header.Set("Connection", "Upgrade")
|
||||
req.Header.Set("Upgrade", "websocket")
|
||||
if err := req.Write(cli); err != nil {
|
||||
t.Fatalf("failed to write upgrade request: %v", err)
|
||||
}
|
||||
|
||||
resp, err := http.ReadResponse(bufio.NewReader(cli), &http.Request{Method: http.MethodGet})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read upgrade response: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusSwitchingProtocols {
|
||||
t.Fatalf("status = %d, want 101", resp.StatusCode)
|
||||
}
|
||||
|
||||
var srv *net.TCPConn
|
||||
select {
|
||||
case srv = <-srvc:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for the backend connection")
|
||||
}
|
||||
defer srv.Close()
|
||||
|
||||
_, cli, srv := newUpgradedTunnel(t)
|
||||
test(t, cli, srv)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newUpgradedTunnel proxies a websocket upgrade from a raw client connection
|
||||
// through a Handler to a hijacking backend and returns both ends of the
|
||||
// tunnel. The Handler's tunnel tracker holds the proxied connections.
|
||||
func newUpgradedTunnel(t *testing.T) (h *Handler, cli, srv *net.TCPConn) {
|
||||
t.Helper()
|
||||
srvc := make(chan *net.TCPConn, 1)
|
||||
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, _, err := http.NewResponseController(w).Hijack()
|
||||
if err != nil {
|
||||
t.Errorf("backend hijack failed: %v", err)
|
||||
return
|
||||
}
|
||||
tcp, ok := conn.(*net.TCPConn)
|
||||
if !ok {
|
||||
conn.Close()
|
||||
t.Errorf("backend conn is %T, want *net.TCPConn", conn)
|
||||
return
|
||||
}
|
||||
if _, err := io.WriteString(tcp, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil {
|
||||
tcp.Close()
|
||||
t.Errorf("backend upgrade write failed: %v", err)
|
||||
return
|
||||
}
|
||||
srvc <- tcp
|
||||
}))
|
||||
t.Cleanup(backend.Close)
|
||||
|
||||
h = minimalHandler(0, &Upstream{
|
||||
Host: new(Host),
|
||||
Dial: backend.Listener.Addr().String(),
|
||||
})
|
||||
h.tunnelTracker = newTunnelTracker(caddy.Log(), 0)
|
||||
initReverseProxyMetrics(h, prometheus.NewRegistry())
|
||||
|
||||
frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Isolate the tunnel from request cancellation, which has its own
|
||||
// backend-close path.
|
||||
r = r.WithContext(context.WithoutCancel(r.Context()))
|
||||
_ = h.ServeHTTP(w, prepareTestRequest(r), caddyhttp.HandlerFunc(func(http.ResponseWriter, *http.Request) error {
|
||||
return nil
|
||||
}))
|
||||
}))
|
||||
t.Cleanup(frontend.Close)
|
||||
|
||||
frontendURL, err := url.Parse(frontend.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse frontend URL: %v", err)
|
||||
}
|
||||
addr, err := net.ResolveTCPAddr("tcp", frontendURL.Host)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to resolve TCP address: %v", err)
|
||||
}
|
||||
cli, err = net.DialTCP("tcp", nil, addr)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to dial frontend: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { cli.Close() })
|
||||
|
||||
req, _ := http.NewRequest(http.MethodGet, frontend.URL, nil)
|
||||
req.Header.Set("Connection", "Upgrade")
|
||||
req.Header.Set("Upgrade", "websocket")
|
||||
if err := req.Write(cli); err != nil {
|
||||
t.Fatalf("failed to write upgrade request: %v", err)
|
||||
}
|
||||
|
||||
resp, err := http.ReadResponse(bufio.NewReader(cli), &http.Request{Method: http.MethodGet})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read upgrade response: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusSwitchingProtocols {
|
||||
t.Fatalf("status = %d, want 101", resp.StatusCode)
|
||||
}
|
||||
|
||||
select {
|
||||
case srv = <-srvc:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("timed out waiting for the backend connection")
|
||||
}
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
return h, cli, srv
|
||||
}
|
||||
|
||||
// A response carrying the Incremental header field (RFC 10036) must be
|
||||
// forwarded without buffering, whatever its Content-Type and Content-Length.
|
||||
func TestFlushIntervalIncremental(t *testing.T) {
|
||||
|
|
@ -617,7 +626,7 @@ func TestDetachedTrackerRemovalAllowsConcurrentConnectionDeletion(t *testing.T)
|
|||
}
|
||||
deleted := make(chan struct{})
|
||||
conn := newTrackingReadWriteCloser()
|
||||
del := ts.registerConnection(conn, func() error {
|
||||
del := ts.registerConnection(conn, func(time.Time) error {
|
||||
go func() {
|
||||
deleteOther()
|
||||
close(deleted)
|
||||
|
|
@ -644,6 +653,228 @@ func TestDetachedTrackerRemovalAllowsConcurrentConnectionDeletion(t *testing.T)
|
|||
}
|
||||
}
|
||||
|
||||
// newStalledWebsocketPeer returns one end of a pipe whose other end never
|
||||
// reads, with a goroutine stuck writing to it the way a tunnel's copy loop
|
||||
// would be. net.Pipe is unbuffered and lets only one write through at a
|
||||
// time, so a close message written to conn waits behind that stuck write,
|
||||
// just as it does on a socket whose peer stopped reading.
|
||||
func newStalledWebsocketPeer(t *testing.T) net.Conn {
|
||||
t.Helper()
|
||||
conn, peer := net.Pipe()
|
||||
t.Cleanup(func() {
|
||||
conn.Close()
|
||||
peer.Close()
|
||||
})
|
||||
go func() { _, _ = conn.Write([]byte("stalled")) }()
|
||||
return conn
|
||||
}
|
||||
|
||||
// shortenCloseControlWriteTimeout makes a stalled peer time out quickly for
|
||||
// the rest of the test.
|
||||
func shortenCloseControlWriteTimeout(t *testing.T) {
|
||||
t.Helper()
|
||||
saved := closeControlWriteTimeout
|
||||
closeControlWriteTimeout = 10 * time.Millisecond
|
||||
t.Cleanup(func() { closeControlWriteTimeout = saved })
|
||||
}
|
||||
|
||||
func TestHandlerCleanupDoesNotBlockOnStalledWebsocketPeer(t *testing.T) {
|
||||
conn := newStalledWebsocketPeer(t)
|
||||
|
||||
ts := newTunnelTracker(caddy.Log(), 0)
|
||||
shortenCloseControlWriteTimeout(t)
|
||||
ts.registerConnection(conn, func(deadline time.Time) error {
|
||||
return writeCloseControl(conn, false, deadline)
|
||||
}, false, "upstream")
|
||||
|
||||
cleanupDone := make(chan error, 1)
|
||||
go func() { cleanupDone <- ts.cleanupAttachedConnections() }()
|
||||
|
||||
select {
|
||||
case err := <-cleanupDone:
|
||||
if !errors.Is(err, os.ErrDeadlineExceeded) {
|
||||
t.Errorf("cleanup = %v, want %v", err, os.ErrDeadlineExceeded)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("cleanup blocked on a websocket peer that stopped reading")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerCleanupUsesOneDeadlinePerBatch(t *testing.T) {
|
||||
// The close loop is serial and holds the tracker lock, so a deadline per
|
||||
// connection would let every stalled peer add another full timeout.
|
||||
ts := newTunnelTracker(caddy.Log(), 0)
|
||||
shortenCloseControlWriteTimeout(t)
|
||||
var deadlines []time.Time
|
||||
for range 2 {
|
||||
conn := newStalledWebsocketPeer(t)
|
||||
ts.registerConnection(conn, func(deadline time.Time) error {
|
||||
deadlines = append(deadlines, deadline)
|
||||
return writeCloseControl(conn, false, deadline)
|
||||
}, false, "upstream")
|
||||
}
|
||||
|
||||
_ = ts.cleanupAttachedConnections()
|
||||
if len(deadlines) != 2 {
|
||||
t.Fatalf("got %d graceful closes, want 2", len(deadlines))
|
||||
}
|
||||
if !deadlines[0].Equal(deadlines[1]) {
|
||||
t.Errorf("graceful closes got deadlines %v and %v, want one shared deadline", deadlines[0], deadlines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamRemovalUsesOneDeadlineAcrossTrackers(t *testing.T) {
|
||||
// Removing an upstream notifies every stopped tracker in turn, and each
|
||||
// close loop holds its own lock, so a deadline per tracker would let
|
||||
// every tracker with a stalled peer add another full timeout.
|
||||
const upstream = "upstream-removal-one-deadline"
|
||||
shortenCloseControlWriteTimeout(t)
|
||||
var deadlines []time.Time
|
||||
var trackers []*tunnelTracker
|
||||
for range 2 {
|
||||
ts := newTunnelTracker(caddy.Log(), 0)
|
||||
registerDetachedTunnelTrackers(ts)
|
||||
t.Cleanup(func() { unregisterDetachedTunnelTrackers(ts) })
|
||||
conn := newStalledWebsocketPeer(t)
|
||||
ts.registerConnection(conn, func(deadline time.Time) error {
|
||||
deadlines = append(deadlines, deadline)
|
||||
return writeCloseControl(conn, false, deadline)
|
||||
}, true, upstream)
|
||||
if err := ts.cleanupAttachedConnections(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trackers = append(trackers, ts)
|
||||
}
|
||||
|
||||
_ = notifyDetachedTunnelTrackersOfUpstreamRemoval(upstream, trackers[0])
|
||||
if len(deadlines) != 2 {
|
||||
t.Fatalf("got %d graceful closes, want 2", len(deadlines))
|
||||
}
|
||||
if !deadlines[0].Equal(deadlines[1]) {
|
||||
t.Errorf("trackers got deadlines %v and %v, want one shared deadline", deadlines[0], deadlines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteCloseControlSkipsConnectionsWithoutWriteDeadline(t *testing.T) {
|
||||
// A write that cannot be bounded could block forever behind a stalled
|
||||
// peer, so the best-effort close message is skipped instead.
|
||||
var buf bytes.Buffer
|
||||
if err := writeCloseControl(&buf, false, time.Now().Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if buf.Len() != 0 {
|
||||
t.Errorf("close message %q was written to a connection without write deadlines", buf.Bytes())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtendedConnectReadWriteCloserSetWriteDeadline(t *testing.T) {
|
||||
rw := &writeDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
|
||||
rwc := extendedConnectReadWriteCloser{io.NopCloser(strings.NewReader("")), rw}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
if err := rwc.SetWriteDeadline(deadline); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !rw.writeDeadline.Equal(deadline) {
|
||||
t.Errorf("write deadline = %v, want %v", rw.writeDeadline, deadline)
|
||||
}
|
||||
|
||||
// Writers without deadline support report ErrNotSupported, which
|
||||
// writeCloseControl treats as a reason to skip the close message.
|
||||
plain := extendedConnectReadWriteCloser{io.NopCloser(strings.NewReader("")), httptest.NewRecorder()}
|
||||
if err := plain.SetWriteDeadline(deadline); !errors.Is(err, http.ErrNotSupported) {
|
||||
t.Errorf("SetWriteDeadline on a plain recorder = %v, want ErrNotSupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
type writeDeadlineRecorder struct {
|
||||
*httptest.ResponseRecorder
|
||||
writeDeadline time.Time
|
||||
}
|
||||
|
||||
func (r *writeDeadlineRecorder) SetWriteDeadline(deadline time.Time) error {
|
||||
r.writeDeadline = deadline
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestHandlerUpgradeBoundsUpstreamCloseMessage(t *testing.T) {
|
||||
switch runtime.GOOS {
|
||||
case "plan9", "js", "wasip1":
|
||||
t.Skipf("not supported on %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
// net/http hands upgraded HTTP/1 bodies back inside a type that hides the
|
||||
// socket's deadline methods, so without the connection recorded during
|
||||
// the round trip the close message to the upstream would be skipped.
|
||||
h, cli, srv := newUpgradedTunnel(t)
|
||||
if err := cli.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := srv.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Data reaching the backend proves both connections are registered.
|
||||
if _, err := io.WriteString(cli, "hello"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hello := make([]byte, len("hello"))
|
||||
if _, err := io.ReadFull(srv, hello); err != nil || string(hello) != "hello" {
|
||||
t.Fatalf("backend read %q, %v; want %q", hello, err, "hello")
|
||||
}
|
||||
|
||||
// Send the close messages without closing anything. Cleanup closes the
|
||||
// connections in map order, and closing the client side first can tear
|
||||
// down the tunnel before the upstream gets its close message.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
h.tunnelTracker.mu.Lock()
|
||||
for _, oc := range h.tunnelTracker.connections {
|
||||
if err := oc.gracefulClose(deadline); err != nil {
|
||||
t.Errorf("graceful close = %v", err)
|
||||
}
|
||||
}
|
||||
h.tunnelTracker.mu.Unlock()
|
||||
frame := make([]byte, 2)
|
||||
if _, err := io.ReadFull(srv, frame); err != nil {
|
||||
t.Fatalf("backend did not receive a close message: %v", err)
|
||||
}
|
||||
if frame[0] != 0x88 || frame[1]&0x80 == 0 {
|
||||
t.Fatalf("backend received %x, want a masked close frame", frame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamWriteCloser(t *testing.T) {
|
||||
// The HTTP/1 path, including the recorded socket, is covered end to end
|
||||
// by TestHandlerUpgradeBoundsUpstreamCloseMessage. HTTP/2 and HTTP/3 share
|
||||
// their connection between streams, so the close message must never go
|
||||
// to a recorded socket for them.
|
||||
body, _ := net.Pipe()
|
||||
socket, _ := net.Pipe()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
protoMajor int
|
||||
recorded net.Conn
|
||||
want io.ReadWriteCloser
|
||||
}{
|
||||
{name: "http/1 with recorded socket", protoMajor: 1, recorded: socket, want: socket},
|
||||
{name: "http/1 without recorded socket", protoMajor: 1, want: body},
|
||||
{name: "http/2", protoMajor: 2, recorded: socket, want: body},
|
||||
{name: "http/3", protoMajor: 3, recorded: socket, want: body},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
recorded := new(atomic.Pointer[net.Conn])
|
||||
if tc.recorded != nil {
|
||||
recorded.Store(&tc.recorded)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(context.WithValue(req.Context(), upstreamConnCtxKey, recorded))
|
||||
res := &http.Response{ProtoMajor: tc.protoMajor}
|
||||
if got := upstreamWriteCloser(req, res, body); got != tc.want {
|
||||
t.Errorf("upstreamWriteCloser() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// opaqueUpgradeWriter supports hijacking but hides the recorder from the
|
||||
// detachment helper, as a middleware wrapper without Unwrap would do.
|
||||
type opaqueUpgradeWriter struct {
|
||||
|
|
|
|||
Loading…
Reference in a new issue