* tracing: fix BatchSpanProcessor goroutine leak on config reload
When the `tracing` directive is enabled, each config reload leaks one
`go.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue`
goroutine. On a server that reloads frequently (e.g. polling a remote
config source every few seconds) this accumulates into tens of thousands
of leaked goroutines over time.
A goroutine dump shows many identical stacks:
```
goroutine ... [select]:
go.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue(...)
.../sdk/trace/batch_span_processor.go:327
go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2()
.../sdk/trace/batch_span_processor.go:129
created by go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor
.../sdk/trace/batch_span_processor.go:127
```
`tracing` keeps a global, reference-counted `TracerProvider` so it can be
reused across reloads (`tracerProvider.getTracerProvider`). Caddy reloads
provision the new config *before* cleaning up the old one, so the counter
never drops to 0 and the provider is correctly reused — `Shutdown` is
never called, by design.
The problem is on the caller side in `newOpenTelemetryWrapper`:
```go
traceExporter, err := autoexport.NewSpanExporter(ctx)
...
tracerProvider := globalTracerProvider.getTracerProvider(
sdktrace.WithBatcher(traceExporter), // evaluated on every reload
sdktrace.WithResource(res),
)
```
`sdktrace.WithBatcher(e)` is `WithSpanProcessor(NewBatchSpanProcessor(e))`,
and `NewBatchSpanProcessor` **starts its `processQueue` goroutine eagerly at
construction time** — not when the option is applied. The option is built
on every `Provision` (every reload), but `getTracerProvider` only applies
it when it actually creates a new provider (`t.tracerProvider == nil`). On
the reuse path the option is silently discarded, so the just-started
BatchSpanProcessor goroutine is orphaned: it is never registered with any
provider and therefore never shut down. Result: one leaked goroutine (plus
a leaked exporter) per reload.
Defer construction of the exporter/batcher until a new provider is actually
needed. `getTracerProvider` now takes a `buildOpts` factory that is invoked
only on the create path, so nothing with a side effect is built on the
reuse path.
The reference counter is now incremented only after the provider is
successfully obtained, preserving the previous semantics where a failed
exporter creation did not affect the counter.
- `Test_tracersProvider_buildOptsOnlyOnCreate` — asserts `buildOpts` runs
exactly once across one create + five reuses (the regression guard).
- `Test_tracersProvider_buildOptsError` — asserts that on a build error the
provider stays nil and the counter is not incremented.
- Existing tracing tests updated for the new signature and still pass.
Verified manually with a reload loop: before the fix, 50 reloads leaked 50
`processQueue` goroutines; after the fix, 0 are leaked while the provider is
still reused (counter stays at 1).
* add test for buldOpts error path
---------
Co-authored-by: Zen Dodd <mail@steadytao.com>
163 lines
5.3 KiB
Go
163 lines
5.3 KiB
Go
package tracing
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
|
|
"go.opentelemetry.io/contrib/exporters/autoexport"
|
|
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
|
"go.opentelemetry.io/contrib/propagators/autoprop"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/caddyserver/caddy/v2"
|
|
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
|
|
)
|
|
|
|
const (
|
|
defaultSpanName = "handler"
|
|
nextCallCtxKey caddy.CtxKey = "nextCall"
|
|
)
|
|
|
|
// nextCall store the next handler, and the error value return on calling it (if any)
|
|
type nextCall struct {
|
|
next caddyhttp.Handler
|
|
err error
|
|
}
|
|
|
|
// openTelemetryWrapper is responsible for the tracing injection, extraction and propagation.
|
|
type openTelemetryWrapper struct {
|
|
propagators propagation.TextMapPropagator
|
|
|
|
handler http.Handler
|
|
|
|
spanName string
|
|
spanAttributes map[string]string
|
|
}
|
|
|
|
// newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration.
|
|
func newOpenTelemetryWrapper(
|
|
ctx context.Context,
|
|
spanName string,
|
|
spanAttributes map[string]string,
|
|
) (openTelemetryWrapper, error) {
|
|
if spanName == "" {
|
|
spanName = defaultSpanName
|
|
}
|
|
|
|
ot := openTelemetryWrapper{
|
|
spanName: spanName,
|
|
spanAttributes: spanAttributes,
|
|
}
|
|
|
|
version, _ := caddy.Version()
|
|
res, err := ot.newResource(caddyhttp.ServerHeader, version)
|
|
if err != nil {
|
|
return ot, fmt.Errorf("creating resource error: %w", err)
|
|
}
|
|
|
|
ot.propagators = autoprop.NewTextMapPropagator()
|
|
|
|
// Defer creation of the exporter (and its batch span processor goroutine)
|
|
// until we know a new provider is actually needed. When the global provider
|
|
// already exists it is reused and these options are discarded; building them
|
|
// here unconditionally would leak the exporter and a BatchSpanProcessor
|
|
// goroutine on every config reload.
|
|
tracerProvider, err := globalTracerProvider.getTracerProvider(func() ([]sdktrace.TracerProviderOption, error) {
|
|
traceExporter, err := autoexport.NewSpanExporter(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating trace exporter error: %w", err)
|
|
}
|
|
|
|
return []sdktrace.TracerProviderOption{
|
|
sdktrace.WithBatcher(traceExporter),
|
|
sdktrace.WithResource(res),
|
|
}, nil
|
|
})
|
|
if err != nil {
|
|
return ot, err
|
|
}
|
|
|
|
ot.handler = otelhttp.NewHandler(http.HandlerFunc(ot.serveHTTP),
|
|
ot.spanName,
|
|
otelhttp.WithTracerProvider(tracerProvider),
|
|
otelhttp.WithPropagators(ot.propagators),
|
|
otelhttp.WithSpanNameFormatter(ot.spanNameFormatter),
|
|
)
|
|
|
|
return ot, nil
|
|
}
|
|
|
|
// serveHTTP injects a tracing context and call the next handler.
|
|
func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
ot.propagators.Inject(ctx, propagation.HeaderCarrier(r.Header))
|
|
spanCtx := trace.SpanContextFromContext(ctx)
|
|
if spanCtx.IsValid() {
|
|
traceID := spanCtx.TraceID().String()
|
|
spanID := spanCtx.SpanID().String()
|
|
// Add a trace_id placeholder, accessible via `{http.vars.trace_id}`.
|
|
caddyhttp.SetVar(ctx, "trace_id", traceID)
|
|
// Add a span_id placeholder, accessible via `{http.vars.span_id}`.
|
|
caddyhttp.SetVar(ctx, "span_id", spanID)
|
|
// Add the traceID and spanID to the log fields for the request.
|
|
if extra, ok := ctx.Value(caddyhttp.ExtraLogFieldsCtxKey).(*caddyhttp.ExtraLogFields); ok {
|
|
extra.Add(zap.String("traceID", traceID))
|
|
extra.Add(zap.String("spanID", spanID))
|
|
}
|
|
}
|
|
|
|
next := ctx.Value(nextCallCtxKey).(*nextCall)
|
|
next.err = next.next.ServeHTTP(w, r)
|
|
|
|
// Add custom span attributes to the current span
|
|
span := trace.SpanFromContext(ctx)
|
|
if span.IsRecording() && len(ot.spanAttributes) > 0 {
|
|
replacer := ctx.Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
|
|
attributes := make([]attribute.KeyValue, 0, len(ot.spanAttributes))
|
|
for key, value := range ot.spanAttributes {
|
|
// Allow placeholder replacement in attribute values
|
|
replacedValue := replacer.ReplaceAll(value, "")
|
|
attributes = append(attributes, attribute.String(key, replacedValue))
|
|
}
|
|
span.SetAttributes(attributes...)
|
|
}
|
|
}
|
|
|
|
// ServeHTTP propagates call to the by wrapped by `otelhttp` next handler.
|
|
func (ot *openTelemetryWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
|
|
n := &nextCall{
|
|
next: next,
|
|
err: nil,
|
|
}
|
|
ot.handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), nextCallCtxKey, n)))
|
|
|
|
return n.err
|
|
}
|
|
|
|
// cleanup flush all remaining data and shutdown a tracerProvider
|
|
func (ot *openTelemetryWrapper) cleanup(logger *zap.Logger) error {
|
|
return globalTracerProvider.cleanupTracerProvider(logger)
|
|
}
|
|
|
|
// newResource creates a resource that describe current handler instance and merge it with a default attributes value.
|
|
func (ot *openTelemetryWrapper) newResource(
|
|
webEngineName,
|
|
webEngineVersion string,
|
|
) (*resource.Resource, error) {
|
|
return resource.Merge(resource.Default(), resource.NewSchemaless(
|
|
semconv.WebEngineName(webEngineName),
|
|
semconv.WebEngineVersion(webEngineVersion),
|
|
))
|
|
}
|
|
|
|
// spanNameFormatter performs the replacement of placeholders in the span name
|
|
func (ot *openTelemetryWrapper) spanNameFormatter(operation string, r *http.Request) string {
|
|
return r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer).ReplaceAll(operation, "")
|
|
}
|