caddy/modules/caddyhttp/tracing/tracer.go
wangdongyong 69d6ace32e
tracing: fix BatchSpanProcessor goroutine leak on config reload (#7826)
* tracing: fix BatchSpanProcessor goroutine leak on config reload

When the `tracing` directive is enabled, each config reload leaks one
`go.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue`
goroutine. On a server that reloads frequently (e.g. polling a remote
config source every few seconds) this accumulates into tens of thousands
of leaked goroutines over time.

A goroutine dump shows many identical stacks:

```
goroutine ... [select]:
go.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue(...)
	.../sdk/trace/batch_span_processor.go:327
go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2()
	.../sdk/trace/batch_span_processor.go:129
created by go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor
	.../sdk/trace/batch_span_processor.go:127
```

`tracing` keeps a global, reference-counted `TracerProvider` so it can be
reused across reloads (`tracerProvider.getTracerProvider`). Caddy reloads
provision the new config *before* cleaning up the old one, so the counter
never drops to 0 and the provider is correctly reused — `Shutdown` is
never called, by design.

The problem is on the caller side in `newOpenTelemetryWrapper`:

```go
traceExporter, err := autoexport.NewSpanExporter(ctx)
...
tracerProvider := globalTracerProvider.getTracerProvider(
    sdktrace.WithBatcher(traceExporter),   // evaluated on every reload
    sdktrace.WithResource(res),
)
```

`sdktrace.WithBatcher(e)` is `WithSpanProcessor(NewBatchSpanProcessor(e))`,
and `NewBatchSpanProcessor` **starts its `processQueue` goroutine eagerly at
construction time** — not when the option is applied. The option is built
on every `Provision` (every reload), but `getTracerProvider` only applies
it when it actually creates a new provider (`t.tracerProvider == nil`). On
the reuse path the option is silently discarded, so the just-started
BatchSpanProcessor goroutine is orphaned: it is never registered with any
provider and therefore never shut down. Result: one leaked goroutine (plus
a leaked exporter) per reload.

Defer construction of the exporter/batcher until a new provider is actually
needed. `getTracerProvider` now takes a `buildOpts` factory that is invoked
only on the create path, so nothing with a side effect is built on the
reuse path.

The reference counter is now incremented only after the provider is
successfully obtained, preserving the previous semantics where a failed
exporter creation did not affect the counter.

- `Test_tracersProvider_buildOptsOnlyOnCreate` — asserts `buildOpts` runs
  exactly once across one create + five reuses (the regression guard).
- `Test_tracersProvider_buildOptsError` — asserts that on a build error the
  provider stays nil and the counter is not incremented.
- Existing tracing tests updated for the new signature and still pass.

Verified manually with a reload loop: before the fix, 50 reloads leaked 50
`processQueue` goroutines; after the fix, 0 are leaked while the provider is
still reused (counter stays at 1).

* add test for buldOpts error path

---------

Co-authored-by: Zen Dodd <mail@steadytao.com>
2026-06-18 11:42:58 -06:00

163 lines
5.3 KiB
Go

package tracing
import (
"context"
"fmt"
"net/http"
"go.opentelemetry.io/contrib/exporters/autoexport"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/contrib/propagators/autoprop"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
)
const (
defaultSpanName = "handler"
nextCallCtxKey caddy.CtxKey = "nextCall"
)
// nextCall store the next handler, and the error value return on calling it (if any)
type nextCall struct {
next caddyhttp.Handler
err error
}
// openTelemetryWrapper is responsible for the tracing injection, extraction and propagation.
type openTelemetryWrapper struct {
propagators propagation.TextMapPropagator
handler http.Handler
spanName string
spanAttributes map[string]string
}
// newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration.
func newOpenTelemetryWrapper(
ctx context.Context,
spanName string,
spanAttributes map[string]string,
) (openTelemetryWrapper, error) {
if spanName == "" {
spanName = defaultSpanName
}
ot := openTelemetryWrapper{
spanName: spanName,
spanAttributes: spanAttributes,
}
version, _ := caddy.Version()
res, err := ot.newResource(caddyhttp.ServerHeader, version)
if err != nil {
return ot, fmt.Errorf("creating resource error: %w", err)
}
ot.propagators = autoprop.NewTextMapPropagator()
// Defer creation of the exporter (and its batch span processor goroutine)
// until we know a new provider is actually needed. When the global provider
// already exists it is reused and these options are discarded; building them
// here unconditionally would leak the exporter and a BatchSpanProcessor
// goroutine on every config reload.
tracerProvider, err := globalTracerProvider.getTracerProvider(func() ([]sdktrace.TracerProviderOption, error) {
traceExporter, err := autoexport.NewSpanExporter(ctx)
if err != nil {
return nil, fmt.Errorf("creating trace exporter error: %w", err)
}
return []sdktrace.TracerProviderOption{
sdktrace.WithBatcher(traceExporter),
sdktrace.WithResource(res),
}, nil
})
if err != nil {
return ot, err
}
ot.handler = otelhttp.NewHandler(http.HandlerFunc(ot.serveHTTP),
ot.spanName,
otelhttp.WithTracerProvider(tracerProvider),
otelhttp.WithPropagators(ot.propagators),
otelhttp.WithSpanNameFormatter(ot.spanNameFormatter),
)
return ot, nil
}
// serveHTTP injects a tracing context and call the next handler.
func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ot.propagators.Inject(ctx, propagation.HeaderCarrier(r.Header))
spanCtx := trace.SpanContextFromContext(ctx)
if spanCtx.IsValid() {
traceID := spanCtx.TraceID().String()
spanID := spanCtx.SpanID().String()
// Add a trace_id placeholder, accessible via `{http.vars.trace_id}`.
caddyhttp.SetVar(ctx, "trace_id", traceID)
// Add a span_id placeholder, accessible via `{http.vars.span_id}`.
caddyhttp.SetVar(ctx, "span_id", spanID)
// Add the traceID and spanID to the log fields for the request.
if extra, ok := ctx.Value(caddyhttp.ExtraLogFieldsCtxKey).(*caddyhttp.ExtraLogFields); ok {
extra.Add(zap.String("traceID", traceID))
extra.Add(zap.String("spanID", spanID))
}
}
next := ctx.Value(nextCallCtxKey).(*nextCall)
next.err = next.next.ServeHTTP(w, r)
// Add custom span attributes to the current span
span := trace.SpanFromContext(ctx)
if span.IsRecording() && len(ot.spanAttributes) > 0 {
replacer := ctx.Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
attributes := make([]attribute.KeyValue, 0, len(ot.spanAttributes))
for key, value := range ot.spanAttributes {
// Allow placeholder replacement in attribute values
replacedValue := replacer.ReplaceAll(value, "")
attributes = append(attributes, attribute.String(key, replacedValue))
}
span.SetAttributes(attributes...)
}
}
// ServeHTTP propagates call to the by wrapped by `otelhttp` next handler.
func (ot *openTelemetryWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
n := &nextCall{
next: next,
err: nil,
}
ot.handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), nextCallCtxKey, n)))
return n.err
}
// cleanup flush all remaining data and shutdown a tracerProvider
func (ot *openTelemetryWrapper) cleanup(logger *zap.Logger) error {
return globalTracerProvider.cleanupTracerProvider(logger)
}
// newResource creates a resource that describe current handler instance and merge it with a default attributes value.
func (ot *openTelemetryWrapper) newResource(
webEngineName,
webEngineVersion string,
) (*resource.Resource, error) {
return resource.Merge(resource.Default(), resource.NewSchemaless(
semconv.WebEngineName(webEngineName),
semconv.WebEngineVersion(webEngineVersion),
))
}
// spanNameFormatter performs the replacement of placeholders in the span name
func (ot *openTelemetryWrapper) spanNameFormatter(operation string, r *http.Request) string {
return r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer).ReplaceAll(operation, "")
}