reverseproxy: fix buffered body ownership and unsafe retries
Fix report de1bcf6d-457c-4348-9d4e-5eeb41cc120b. Release pooled buffers exactly once, serialize reads with release, and retain safe replay cursors for fully buffered requests. Retire each stream attempt so consumed bodies cannot be retried while unread dial failures remain retryable. Add deterministic concurrent ownership checks, boundary coverage, local transport failure cases, and synthetic canary tests. Code written by GitHub Copilot (GPT-6.1 Sol, Claude Opus 5.5); reviewed by mholt. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
parent
7cc0460738
commit
d52d097109
2 changed files with 497 additions and 59 deletions
|
|
@ -1,11 +1,22 @@
|
|||
package reverseproxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"time"
|
||||
|
||||
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
|
||||
)
|
||||
|
||||
type zeroReader struct{}
|
||||
|
|
@ -48,7 +59,7 @@ func TestBuffering(t *testing.T) {
|
|||
limit: -1,
|
||||
},
|
||||
resultCheck: func(res io.ReadCloser, read int64, args args) bool {
|
||||
brc, ok := res.(bodyReadCloser)
|
||||
brc, ok := res.(*bodyReadCloser)
|
||||
return ok && brc.body == nil && brc.buf.Len() == 100 && read == 100
|
||||
},
|
||||
},
|
||||
|
|
@ -59,7 +70,7 @@ func TestBuffering(t *testing.T) {
|
|||
limit: 50,
|
||||
},
|
||||
resultCheck: func(res io.ReadCloser, read int64, args args) bool {
|
||||
brc, ok := res.(bodyReadCloser)
|
||||
brc, ok := res.(*bodyReadCloser)
|
||||
return ok && brc.body != nil && brc.buf.Len() == 50 && read == 50
|
||||
},
|
||||
},
|
||||
|
|
@ -70,7 +81,7 @@ func TestBuffering(t *testing.T) {
|
|||
limit: 101,
|
||||
},
|
||||
resultCheck: func(res io.ReadCloser, read int64, args args) bool {
|
||||
brc, ok := res.(bodyReadCloser)
|
||||
brc, ok := res.(*bodyReadCloser)
|
||||
return ok && brc.body == nil && brc.buf.Len() == 100 && read == 100
|
||||
},
|
||||
},
|
||||
|
|
@ -78,6 +89,7 @@ func TestBuffering(t *testing.T) {
|
|||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
res, read := h.bufferedBody(tt.args.body, tt.args.limit)
|
||||
defer res.Close()
|
||||
if !tt.resultCheck(res, read, tt.args) {
|
||||
t.Error("Handler.bufferedBody() test failed")
|
||||
return
|
||||
|
|
@ -110,7 +122,7 @@ func TestBufferingBoundary(t *testing.T) {
|
|||
t.Errorf("limit %d: closing buffered body: %v", limit, err)
|
||||
}
|
||||
|
||||
brc, ok := res.(bodyReadCloser)
|
||||
brc, ok := res.(*bodyReadCloser)
|
||||
if !ok {
|
||||
t.Fatalf("limit %d: expected a bodyReadCloser", limit)
|
||||
}
|
||||
|
|
@ -143,13 +155,359 @@ func TestBufferingKeepsReadError(t *testing.T) {
|
|||
t.Errorf("limit %d: got %q, want %q", limit, got, "0123456789")
|
||||
}
|
||||
|
||||
brc, ok := res.(bodyReadCloser)
|
||||
brc, ok := res.(*bodyReadCloser)
|
||||
if !ok {
|
||||
t.Fatalf("limit %d: expected a bodyReadCloser", limit)
|
||||
}
|
||||
|
||||
if brc.body == nil {
|
||||
t.Errorf("limit %d: body reported as fully buffered", limit)
|
||||
}
|
||||
res.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type bufferingTestBody struct {
|
||||
io.Reader
|
||||
closes atomic.Int32
|
||||
err error
|
||||
}
|
||||
|
||||
func (b *bufferingTestBody) Close() error {
|
||||
b.closes.Add(1)
|
||||
return b.err
|
||||
}
|
||||
|
||||
func TestBufferedBodyCloseOwnership(t *testing.T) {
|
||||
const payload = "canary"
|
||||
const size = int64(len(payload))
|
||||
for _, limit := range []int64{-1, size - 1, size, size + 1} {
|
||||
t.Run(fmt.Sprint(limit), func(t *testing.T) {
|
||||
closeErr := errors.New("synthetic close error")
|
||||
original := &bufferingTestBody{Reader: strings.NewReader(payload), err: closeErr}
|
||||
body, _ := (Handler{}).bufferedBody(original, limit)
|
||||
b := body.(*bodyReadCloser)
|
||||
var wg sync.WaitGroup
|
||||
for range 32 {
|
||||
wg.Go(func() {
|
||||
err := body.Close()
|
||||
if b.body != nil && !errors.Is(err, closeErr) {
|
||||
t.Errorf("Close error = %v, want %v", err, closeErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if got := original.closes.Load(); got != 1 {
|
||||
t.Fatalf("underlying Close called %d times, want 1", got)
|
||||
}
|
||||
if b.buf != nil || !b.closed {
|
||||
t.Fatal("closed body retained buffer ownership")
|
||||
}
|
||||
if n, err := body.Read(make([]byte, 32)); n != 0 || !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("read after release = %d, %v", n, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedBodyReplayOwnership(t *testing.T) {
|
||||
const payload = "synthetic replay canary"
|
||||
body, _ := (Handler{}).bufferedBody(io.NopCloser(strings.NewReader(payload)), -1)
|
||||
owner := body.(*bodyReadCloser)
|
||||
defer owner.Close()
|
||||
first := &bufferedBodyReader{owner: owner}
|
||||
second := &bufferedBodyReader{owner: owner}
|
||||
if n, err := first.Read(make([]byte, 3)); n != 3 || err != nil {
|
||||
t.Fatalf("first read = %d, %v", n, err)
|
||||
}
|
||||
first.Close()
|
||||
got, err := io.ReadAll(second)
|
||||
if err != nil || string(got) != payload {
|
||||
t.Fatalf("replayed body = %q, %v", got, err)
|
||||
}
|
||||
owner.Close()
|
||||
for _, reader := range []io.ReadCloser{first, second, owner, &bufferedBodyReader{owner: owner}} {
|
||||
if n, err := reader.Read(make([]byte, 32)); n != 0 || !errors.Is(err, net.ErrClosed) {
|
||||
t.Errorf("read after release = %d, %v", n, err)
|
||||
}
|
||||
reader.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedBodyConcurrentReadClose(t *testing.T) {
|
||||
for _, limit := range []int64{-1, 4, 8} {
|
||||
t.Run(fmt.Sprint(limit), func(t *testing.T) {
|
||||
original := &bufferingTestBody{Reader: strings.NewReader("synthetic canary")}
|
||||
body, _ := (Handler{}).bufferedBody(original, limit)
|
||||
defer body.Close()
|
||||
var reader io.ReadCloser = body
|
||||
if limit == -1 {
|
||||
reader = &bufferedBodyReader{owner: body.(*bodyReadCloser)}
|
||||
}
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
for range 64 {
|
||||
_, err := reader.Read(make([]byte, 1))
|
||||
if err != nil && err != io.EOF && !errors.Is(err, net.ErrClosed) {
|
||||
t.Errorf("concurrent read: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
for range 16 {
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
body.Close()
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if n, err := reader.Read(make([]byte, 1)); n != 0 || !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("read after concurrent release = %d, %v", n, err)
|
||||
}
|
||||
if got := original.closes.Load(); got != 1 {
|
||||
t.Fatalf("underlying Close called %d times, want 1", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type blockingBufferingBody struct {
|
||||
started chan struct{}
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockingBufferingBody) Read([]byte) (int, error) {
|
||||
close(b.started)
|
||||
<-b.closed
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
|
||||
func (b *blockingBufferingBody) Close() error {
|
||||
close(b.closed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestBufferedBodyCloseUnblocksRead(t *testing.T) {
|
||||
original := &blockingBufferingBody{started: make(chan struct{}), closed: make(chan struct{})}
|
||||
body, _ := (Handler{}).bufferedBody(&struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{io.MultiReader(strings.NewReader("abcd"), original), original}, 4)
|
||||
defer body.Close()
|
||||
if n, err := body.Read(make([]byte, 4)); n != 4 || err != nil {
|
||||
t.Fatalf("buffer read = %d, %v", n, err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := body.Read(make([]byte, 4))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-original.started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("stream read did not start")
|
||||
}
|
||||
go body.Close()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("stream read error = %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close did not unblock stream read")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedBodyRetiredAttempt(t *testing.T) {
|
||||
original := &bufferingTestBody{Reader: strings.NewReader("untouched canary")}
|
||||
attempt := &bodyNopCloserIfNotRead{ReadCloser: original}
|
||||
attempt.Close()
|
||||
if n, err := attempt.Read(make([]byte, 1)); n != 0 || !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("retired attempt read = %d, %v", n, err)
|
||||
}
|
||||
if attempt.didRead() || original.closes.Load() != 0 {
|
||||
t.Fatal("retiring an unread attempt consumed or closed the stream")
|
||||
}
|
||||
next := &bodyNopCloserIfNotRead{ReadCloser: original}
|
||||
defer next.Close()
|
||||
got, err := io.ReadAll(next)
|
||||
if err != nil || string(got) != "untouched canary" {
|
||||
t.Fatalf("next attempt body = %q, %v", got, err)
|
||||
}
|
||||
attempt.Close()
|
||||
if original.closes.Load() != 0 {
|
||||
t.Fatal("late Close from a retired attempt closed the next attempt")
|
||||
}
|
||||
}
|
||||
|
||||
type bufferingTestTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f bufferingTestTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
func TestBufferedBodyPartialReadNotRetried(t *testing.T) {
|
||||
for _, limit := range []int64{0, 4, -1} {
|
||||
t.Run(fmt.Sprint(limit), func(t *testing.T) {
|
||||
original := &bufferingTestBody{Reader: strings.NewReader("synthetic canary")}
|
||||
req := prepareTestRequest(httptest.NewRequest(http.MethodGet, "http://127.0.0.1/", original))
|
||||
defer req.Body.Close()
|
||||
h := minimalHandler(1, &Upstream{Host: new(Host), Dial: "127.0.0.1:1"})
|
||||
h.RequestBuffers = limit
|
||||
var attempts int
|
||||
var retired io.ReadCloser
|
||||
transportErr := errors.New("forced failure after one byte")
|
||||
h.Transport = bufferingTestTransport(func(r *http.Request) (*http.Response, error) {
|
||||
attempts++
|
||||
retired = r.Body
|
||||
if n, err := r.Body.Read(make([]byte, 1)); n != 1 || err != nil {
|
||||
t.Errorf("prefix read = %d, %v", n, err)
|
||||
}
|
||||
r.Body.Close()
|
||||
return nil, transportErr
|
||||
})
|
||||
if err := h.ServeHTTP(httptest.NewRecorder(), req, nil); !errors.Is(err, transportErr) {
|
||||
t.Fatalf("proxy error = %v, want %v", err, transportErr)
|
||||
}
|
||||
want := 1
|
||||
if limit == -1 {
|
||||
want = 2
|
||||
}
|
||||
if attempts != want {
|
||||
t.Fatalf("attempts = %d, want %d", attempts, want)
|
||||
}
|
||||
if n, err := retired.Read(make([]byte, 1)); n != 0 || !errors.Is(err, net.ErrClosed) {
|
||||
t.Fatalf("retired transport read = %d, %v", n, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedBodyTransportRetries(t *testing.T) {
|
||||
const payload = "synthetic transport canary"
|
||||
const size = int64(len(payload))
|
||||
for _, failure := range []string{"dial", "consumed", "response", "truncated"} {
|
||||
for _, limit := range []int64{0, -1, size - 1, size, size + 1} {
|
||||
t.Run(fmt.Sprintf("%s/limit=%d", failure, limit), func(t *testing.T) {
|
||||
var goodCalls, badCalls atomic.Int32
|
||||
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
goodCalls.Add(1)
|
||||
got, err := io.ReadAll(r.Body)
|
||||
if err != nil || string(got) != payload {
|
||||
t.Errorf("good upstream body = %q, %v", got, err)
|
||||
}
|
||||
_, _ = w.Write(got)
|
||||
}))
|
||||
t.Cleanup(good.Close)
|
||||
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
badCalls.Add(1)
|
||||
got, err := io.ReadAll(r.Body)
|
||||
if failure == "truncated" {
|
||||
if err == nil || !strings.HasPrefix(payload, string(got)) {
|
||||
t.Errorf("truncated request read error = %v", err)
|
||||
}
|
||||
} else if err != nil || string(got) != payload {
|
||||
t.Errorf("failing upstream body = %q, %v", got, err)
|
||||
}
|
||||
if failure == "response" {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte("synthetic response canary"))
|
||||
return
|
||||
}
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
conn.Close()
|
||||
}))
|
||||
t.Cleanup(bad.Close)
|
||||
h := minimalHandler(1,
|
||||
&Upstream{Host: new(Host), Dial: good.Listener.Addr().String()},
|
||||
&Upstream{Host: new(Host), Dial: bad.Listener.Addr().String()})
|
||||
h.RequestBuffers = limit
|
||||
h.ResponseBuffers = 4
|
||||
tr := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
if failure == "dial" && addr == bad.Listener.Addr().String() {
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("forced local dial failure")}
|
||||
}
|
||||
return (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, network, addr)
|
||||
},
|
||||
ResponseHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
h.Transport = testTransport{tr}
|
||||
t.Cleanup(tr.CloseIdleConnections)
|
||||
if failure == "response" {
|
||||
h.LoadBalancing.RetryMatch = []caddyhttp.MatcherSet{{
|
||||
newExpressionMatcher(t, "{http.reverse_proxy.status_code} == 503"),
|
||||
}}
|
||||
}
|
||||
req := prepareTestRequest(httptest.NewRequest(http.MethodGet, "http://127.0.0.1/", strings.NewReader(payload)))
|
||||
if failure == "truncated" {
|
||||
req.Body = io.NopCloser(io.MultiReader(strings.NewReader(payload), iotest.ErrReader(io.ErrUnexpectedEOF)))
|
||||
req.ContentLength++
|
||||
}
|
||||
defer req.Body.Close()
|
||||
rec := httptest.NewRecorder()
|
||||
err := h.ServeHTTP(rec, req, nil)
|
||||
wantRetry := failure != "truncated" && (failure == "dial" || limit < 0 || limit > size)
|
||||
if wantRetry {
|
||||
if err != nil || rec.Body.String() != payload || goodCalls.Load() != 1 {
|
||||
t.Fatalf("retry: body=%q good calls=%d err=%v", rec.Body.String(), goodCalls.Load(), err)
|
||||
}
|
||||
} else if err == nil || goodCalls.Load() != 0 {
|
||||
t.Fatalf("consumed stream retried: good calls=%d err=%v", goodCalls.Load(), err)
|
||||
}
|
||||
wantBad := int32(1)
|
||||
if failure == "dial" {
|
||||
wantBad = 0
|
||||
}
|
||||
// A body read error may abort the write before even the request
|
||||
// headers reach the local server.
|
||||
if got := badCalls.Load(); failure != "truncated" && got != wantBad {
|
||||
t.Fatalf("failing upstream calls = %d, want %d", got, wantBad)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedBodiesConcurrentCanaries(t *testing.T) {
|
||||
const workers = 16
|
||||
for round := range 8 {
|
||||
ready := make(chan *bodyReadCloser, workers)
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for worker := range workers {
|
||||
wg.Go(func() {
|
||||
payload := strings.Repeat(fmt.Sprintf("synthetic-%d-%d|", round, worker), 32)
|
||||
// Exercise the shared request/response pool after repeated closes
|
||||
// of a partially buffered body, while other leases remain active.
|
||||
old, _ := (Handler{}).bufferedBody(io.NopCloser(strings.NewReader(payload)), 4)
|
||||
old.Close()
|
||||
body, _ := (Handler{}).bufferedBody(io.NopCloser(strings.NewReader(payload)), int64(len(payload)+worker%3-1))
|
||||
defer body.Close()
|
||||
ready <- body.(*bodyReadCloser)
|
||||
<-start
|
||||
old.Close()
|
||||
got, err := io.ReadAll(body)
|
||||
if err != nil || string(got) != payload {
|
||||
t.Errorf("body changed: got %q, want %q, err=%v", got, payload, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
leased := make(map[*bytes.Buffer]bool)
|
||||
for range workers {
|
||||
b := <-ready
|
||||
if leased[b.buf] {
|
||||
t.Error("concurrent bodies leased the same buffer")
|
||||
}
|
||||
leased[b.buf] = true
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -168,6 +168,9 @@ type Handler struct {
|
|||
// chunked encodings. An incremental request with an explicitly configured
|
||||
// buffer receives a 501 response. A transport-supplied default is bypassed
|
||||
// only when the request already meets the transport's framing requirements.
|
||||
// Only fully buffered bodies can be replayed after an upstream reads them.
|
||||
// Bodies reaching a positive limit remain streaming and cannot be retried
|
||||
// after reading starts; -1 buffers without a size limit.
|
||||
RequestBuffers int64 `json:"request_buffers,omitempty"`
|
||||
|
||||
// If nonzero, the entire response body up to this size will be read
|
||||
|
|
@ -598,32 +601,41 @@ func (h *Handler) Cleanup() error {
|
|||
// It will close the body as normal if the body is read.
|
||||
type bodyNopCloserIfNotRead struct {
|
||||
io.ReadCloser
|
||||
read int // tracks the number of bytes read, -1 when first Read returns 0, io.EOF
|
||||
mu sync.Mutex
|
||||
read bool
|
||||
closed bool
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (b *bodyNopCloserIfNotRead) Read(p []byte) (int, error) {
|
||||
if b.read == -1 {
|
||||
return 0, io.EOF
|
||||
b.mu.Lock()
|
||||
if b.closed {
|
||||
b.mu.Unlock()
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
n, err := b.ReadCloser.Read(p)
|
||||
// first Read returns 0, io.EOF
|
||||
if b.read == 0 && n == 0 && err == io.EOF {
|
||||
b.read = -1
|
||||
} else {
|
||||
b.read += n
|
||||
}
|
||||
return n, err
|
||||
b.read = true
|
||||
b.mu.Unlock()
|
||||
return b.ReadCloser.Read(p)
|
||||
}
|
||||
|
||||
func (b *bodyNopCloserIfNotRead) Close() error {
|
||||
// don't close the body
|
||||
if b.read == 0 {
|
||||
return nil
|
||||
}
|
||||
// close as usual, when -1, any read will return EOF as the original read will do
|
||||
// in other cases, the read will fail as body is closed because we do not want partial bodies to be sent to the upstream
|
||||
// users can buffer the entire request body to allow the request to be resent
|
||||
return b.ReadCloser.Close()
|
||||
b.closeOnce.Do(func() {
|
||||
b.mu.Lock()
|
||||
b.closed = true
|
||||
read := b.read
|
||||
b.mu.Unlock()
|
||||
if read {
|
||||
b.closeErr = b.ReadCloser.Close()
|
||||
}
|
||||
})
|
||||
return b.closeErr
|
||||
}
|
||||
|
||||
func (b *bodyNopCloserIfNotRead) didRead() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.read
|
||||
}
|
||||
|
||||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
|
||||
|
|
@ -669,8 +681,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
|
|||
reqHeader := clonedReq.Header
|
||||
requestWasIncremental := caddyhttp.IsIncremental(reqHeader)
|
||||
|
||||
// If the request contained a body, wrap it in io.NopCloser
|
||||
// to prevent Go's transport from closing it on dial errors.
|
||||
// If the request contained a body, keep it open on dial errors.
|
||||
// cloneRequest does a shallow copy, so clonedReq.Body and
|
||||
// r.Body share the same io.ReadCloser — a dial-failure Close()
|
||||
// would kill the original body for all subsequent retry
|
||||
|
|
@ -678,20 +689,19 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
|
|||
// the HTTP server when the handler returns.
|
||||
//
|
||||
// If the body was already fully buffered (via request_buffers),
|
||||
// we also extract the buffer so the retry loop can replay it
|
||||
// we retain ownership so the retry loop can replay it
|
||||
// from the beginning on each attempt. (see #6259, #7546, #7713)
|
||||
var bufferedReqBody *bytes.Buffer
|
||||
var bufferedReqBody *bodyReadCloser
|
||||
var streamingReqBody io.ReadCloser
|
||||
if clonedReq.Body != nil {
|
||||
if reqBodyBuf, ok := clonedReq.Body.(bodyReadCloser); ok && reqBodyBuf.body == nil && reqBodyBuf.buf != nil {
|
||||
bufferedReqBody = reqBodyBuf.buf
|
||||
reqBodyBuf.buf = nil
|
||||
clonedReq.Body = io.NopCloser(bytes.NewReader(bufferedReqBody.Bytes()))
|
||||
defer func() {
|
||||
bufferedReqBody.Reset()
|
||||
bufPool.Put(bufferedReqBody)
|
||||
}()
|
||||
} else {
|
||||
clonedReq.Body = &bodyNopCloserIfNotRead{ReadCloser: clonedReq.Body}
|
||||
if reqBodyBuf, ok := clonedReq.Body.(*bodyReadCloser); ok {
|
||||
defer reqBodyBuf.Close()
|
||||
if reqBodyBuf.body == nil {
|
||||
bufferedReqBody = reqBodyBuf
|
||||
}
|
||||
}
|
||||
if bufferedReqBody == nil {
|
||||
streamingReqBody = clonedReq.Body
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -713,12 +723,24 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
|
|||
// and reusable, so if a backend partially or fully reads the body but then
|
||||
// produces an error, the request can be repeated to the next backend with
|
||||
// the full body (retries should only happen for idempotent requests) (see #6259)
|
||||
var attemptBody *bodyNopCloserIfNotRead
|
||||
if bufferedReqBody != nil {
|
||||
clonedReq.Body = io.NopCloser(bytes.NewReader(bufferedReqBody.Bytes()))
|
||||
clonedReq.Body = &bufferedBodyReader{owner: bufferedReqBody}
|
||||
} else if streamingReqBody != nil {
|
||||
attemptBody = &bodyNopCloserIfNotRead{ReadCloser: streamingReqBody}
|
||||
clonedReq.Body = attemptBody
|
||||
}
|
||||
|
||||
var done bool
|
||||
done, proxyErr = h.proxyLoopIteration(clonedReq, r, w, proxyErr, start, retries, repl, reqHeader, reqHost, requestWasIncremental, next)
|
||||
if attemptBody != nil {
|
||||
// Retire this attempt before deciding whether to reuse the stream.
|
||||
// RoundTrippers may close or read the body asynchronously.
|
||||
attemptBody.Close()
|
||||
if attemptBody.didRead() {
|
||||
done = true
|
||||
}
|
||||
}
|
||||
if done {
|
||||
break
|
||||
}
|
||||
|
|
@ -965,7 +987,7 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http.
|
|||
var readBytes int64
|
||||
req.Body, readBytes = h.bufferedBody(req.Body, h.RequestBuffers)
|
||||
// set Content-Length when body is fully buffered
|
||||
if b, ok := req.Body.(bodyReadCloser); ok && b.body == nil {
|
||||
if b, ok := req.Body.(*bodyReadCloser); ok && b.body == nil {
|
||||
req.ContentLength = readBytes
|
||||
req.Header["Content-Length"] = []string{strconv.FormatInt(req.ContentLength, 10)}
|
||||
}
|
||||
|
|
@ -981,6 +1003,9 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http.
|
|||
}
|
||||
|
||||
if req.ContentLength == 0 {
|
||||
if b, ok := req.Body.(*bodyReadCloser); ok {
|
||||
b.Close()
|
||||
}
|
||||
req.Body = nil // Issue golang/go#16036: nil Body for http.Transport retries
|
||||
}
|
||||
|
||||
|
|
@ -1728,7 +1753,7 @@ func (h Handler) provisionUpstream(upstream *Upstream, dynamic bool) {
|
|||
// then returns a reader for the buffer along with how many bytes were buffered. Always close
|
||||
// the return value when done with it, just like if it was the original body! If limit is 0
|
||||
// (which it shouldn't be), this function returns its input; i.e. is a no-op, for safety.
|
||||
// Otherwise, it returns bodyReadCloser, the original body will be closed and body will be nil
|
||||
// Otherwise, it returns *bodyReadCloser, the original body will be closed and body will be nil
|
||||
// if it's explicitly configured to buffer all or EOF is reached when reading.
|
||||
// If reading fails, the body is returned as partially buffered, so the read error
|
||||
// is not hidden from the caller.
|
||||
|
|
@ -1746,16 +1771,14 @@ func (h Handler) bufferedBody(originalBody io.ReadCloser, limit int64) (io.ReadC
|
|||
written, err = io.Copy(buf, originalBody)
|
||||
}
|
||||
if (err != nil && err != io.EOF) || written == limit {
|
||||
return bodyReadCloser{
|
||||
Reader: io.MultiReader(buf, originalBody),
|
||||
buf: buf,
|
||||
body: originalBody,
|
||||
return &bodyReadCloser{
|
||||
buf: buf,
|
||||
body: originalBody,
|
||||
}, written
|
||||
}
|
||||
originalBody.Close() // no point in keeping it open
|
||||
return bodyReadCloser{
|
||||
Reader: buf,
|
||||
buf: buf,
|
||||
return &bodyReadCloser{
|
||||
buf: buf,
|
||||
}, written
|
||||
}
|
||||
|
||||
|
|
@ -2314,23 +2337,80 @@ type retryableResponseError struct {
|
|||
}
|
||||
|
||||
// bodyReadCloser is a reader that, upon closing, will return
|
||||
// its buffer to the pool and close the underlying body reader.
|
||||
// its buffer to the pool and close the underlying body reader exactly once.
|
||||
// Buffer reads and release are serialized, but stream reads must remain
|
||||
// interruptible by Close.
|
||||
type bodyReadCloser struct {
|
||||
io.Reader
|
||||
buf *bytes.Buffer
|
||||
body io.ReadCloser
|
||||
mu sync.Mutex
|
||||
buf *bytes.Buffer
|
||||
body io.ReadCloser
|
||||
closed bool
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (brc bodyReadCloser) Close() error {
|
||||
// Inside this package this will be set to nil for fully-buffered
|
||||
// requests due to the possibility of retrial.
|
||||
if brc.buf != nil {
|
||||
func (brc *bodyReadCloser) Read(p []byte) (int, error) {
|
||||
brc.mu.Lock()
|
||||
if brc.closed {
|
||||
brc.mu.Unlock()
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
if brc.buf.Len() > 0 {
|
||||
n, err := brc.buf.Read(p)
|
||||
brc.mu.Unlock()
|
||||
return n, err
|
||||
}
|
||||
body := brc.body
|
||||
brc.mu.Unlock()
|
||||
if body != nil {
|
||||
// Do not hold the buffer lock across a blocking stream read:
|
||||
// Close must be able to close the stream to unblock it.
|
||||
return body.Read(p)
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
func (brc *bodyReadCloser) Close() error {
|
||||
brc.closeOnce.Do(func() {
|
||||
brc.mu.Lock()
|
||||
brc.closed = true
|
||||
brc.buf.Reset()
|
||||
bufPool.Put(brc.buf)
|
||||
brc.buf = nil
|
||||
brc.mu.Unlock()
|
||||
if brc.body != nil {
|
||||
brc.closeErr = brc.body.Close()
|
||||
}
|
||||
})
|
||||
return brc.closeErr
|
||||
}
|
||||
|
||||
// bufferedBodyReader is an independent replay cursor. The owner serializes
|
||||
// buffer access with release, including reads after a RoundTripper returns.
|
||||
type bufferedBodyReader struct {
|
||||
owner *bodyReadCloser
|
||||
offset int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (r *bufferedBodyReader) Read(p []byte) (int, error) {
|
||||
r.owner.mu.Lock()
|
||||
defer r.owner.mu.Unlock()
|
||||
if r.closed || r.owner.closed {
|
||||
return 0, net.ErrClosed
|
||||
}
|
||||
// For fully-buffered bodies, body is nil, so Close is a no-op.
|
||||
if brc.body != nil {
|
||||
return brc.body.Close()
|
||||
if r.offset == r.owner.buf.Len() {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, r.owner.buf.Bytes()[r.offset:])
|
||||
r.offset += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *bufferedBodyReader) Close() error {
|
||||
r.owner.mu.Lock()
|
||||
r.closed = true
|
||||
r.owner.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue