Compare commits

...

4 commits

Author SHA1 Message Date
Mohammed Al Sahaf
4cff67dcec
fix collection of WAL files on Windows
Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com>
2026-07-15 02:25:35 +03:00
Mohammed Al Sahaf
81e7c6d7cb
fix test probing
Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com>
2026-07-15 01:56:00 +03:00
Mohammed Al Sahaf
5b297d59ec
logging: address review findings in net writer WAL
- stop re-dumping the entire backlog to stderr every 100ms while the
  destination is down; entries now stay in the WAL until delivered
- truncate the WAL after delivery to prevent unbounded disk growth
- drop the per-entry fsync and per-entry metadata file write; the
  delivery position is now persisted once per flush batch
- replace the lossy bit-packed offset with an explicit chunk position
  tuple comparison
- make the shutdown flush a bounded, best-effort single pass instead
  of retrying with sleeps per entry
- remove the unused buffer_size config field
- support reconnect_interval in the Caddyfile directive
2026-07-15 01:17:05 +03:00
Mohammed Al Sahaf
d095d43920
logging: implement WAL for net writer 2026-07-15 00:38:04 +03:00
4 changed files with 2119 additions and 92 deletions

3
go.mod
View file

@ -22,6 +22,7 @@ require (
github.com/mholt/acmez/v3 v3.1.6 github.com/mholt/acmez/v3 v3.1.6
github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_golang v1.23.2
github.com/quic-go/quic-go v0.59.1 github.com/quic-go/quic-go v0.59.1
github.com/rosedblabs/wal v1.3.6
github.com/smallstep/certificates v0.30.2 github.com/smallstep/certificates v0.30.2
github.com/smallstep/nosql v0.8.0 github.com/smallstep/nosql v0.8.0
github.com/smallstep/truststore v0.13.0 github.com/smallstep/truststore v0.13.0
@ -74,6 +75,7 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect
@ -89,6 +91,7 @@ require (
github.com/smallstep/pkcs7 v0.2.1 // indirect github.com/smallstep/pkcs7 v0.2.1 // indirect
github.com/smallstep/scep v0.0.0-20250318231241-a25cabb69492 // indirect github.com/smallstep/scep v0.0.0-20250318231241-a25cabb69492 // indirect
github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect github.com/zeebo/blake3 v0.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect

6
go.sum
View file

@ -198,6 +198,8 @@ github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
@ -291,6 +293,8 @@ github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBi
github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rosedblabs/wal v1.3.6 h1:oxZYTPX/u4JuGDW98wQ1YamWqerlrlSUFKhgP6Gd/Ao=
github.com/rosedblabs/wal v1.3.6/go.mod h1:wdq54KJUyVTOv1uddMc6Cdh2d/YCIo8yjcwJAb1RCEM=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@ -361,6 +365,8 @@ github.com/tailscale/tscert v0.0.0-20251216020129-aea342f6d747/go.mod h1:ejPAJui
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ=
github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

View file

@ -15,24 +15,46 @@
package logging package logging
import ( import (
"cmp"
"context"
"errors"
"fmt" "fmt"
"io" "io"
"io/fs"
"log/slog"
"net" "net"
"os" "os"
"path/filepath"
"strings"
"sync" "sync"
"time" "time"
"github.com/rosedblabs/wal"
"github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
) )
func init() { func init() {
caddy.RegisterModule(NetWriter{}) caddy.RegisterModule(&NetWriter{})
} }
// NetWriter implements a log writer that outputs to a network socket. If const (
// the socket goes down, it will dump logs to stderr while it attempts to // walSegmentSize is the maximum size of a WAL segment file.
// reconnect. walSegmentSize = 64 * 1024 * 1024
// walTruncateThreshold is the number of delivered payload bytes
// after which the WAL is truncated (once the backlog is fully
// delivered) to reclaim disk space and keep flush scans cheap.
walTruncateThreshold = 4 * 1024 * 1024
)
// NetWriter implements a log writer that outputs to a network socket.
// Log entries are first appended to a write-ahead log (WAL) on disk and
// then delivered to the socket by a background goroutine, so logging
// never blocks on a slow or unavailable destination. If the socket goes
// down, entries accumulate in the WAL and are delivered once the
// connection is re-established, including across process restarts.
type NetWriter struct { type NetWriter struct {
// The address of the network socket to which to connect. // The address of the network socket to which to connect.
Address string `json:"address,omitempty"` Address string `json:"address,omitempty"`
@ -40,16 +62,41 @@ type NetWriter struct {
// The timeout to wait while connecting to the socket. // The timeout to wait while connecting to the socket.
DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`
// If enabled, allow connections errors when first opening the // If enabled, a failure to connect to the socket when first
// writer. The error and subsequent log entries will be reported // opening the writer is not fatal; log entries are buffered
// to stderr instead until a connection can be re-established. // in the WAL until a connection can be established.
SoftStart bool `json:"soft_start,omitempty"` SoftStart bool `json:"soft_start,omitempty"`
addr caddy.NetworkAddress // How often to attempt reconnection when the network connection fails.
ReconnectInterval caddy.Duration `json:"reconnect_interval,omitempty"`
logger *slog.Logger
addr caddy.NetworkAddress
wal *wal.WAL
walDir string
flushCtx context.Context
flushCtxCancel context.CancelFunc
flushWg sync.WaitGroup
// walMu synchronizes WAL open/read/write/truncate operations
walMu sync.Mutex
// mu protects the fields below; when held together with walMu,
// it is always acquired after walMu
mu sync.RWMutex
// position of the last WAL entry delivered to the destination,
// or nil if nothing has been delivered from the current WAL
lastProcessed *wal.ChunkPosition
// whether the WAL may contain undelivered entries
pending bool
// number of payload bytes delivered since the WAL was last
// truncated; only accessed by the background flusher goroutine
deliveredSinceTruncate int64
} }
// CaddyModule returns the Caddy module information. // CaddyModule returns the Caddy module information.
func (NetWriter) CaddyModule() caddy.ModuleInfo { func (*NetWriter) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{ return caddy.ModuleInfo{
ID: "caddy.logging.writers.net", ID: "caddy.logging.writers.net",
New: func() caddy.Module { return new(NetWriter) }, New: func() caddy.Module { return new(NetWriter) },
@ -58,6 +105,7 @@ func (NetWriter) CaddyModule() caddy.ModuleInfo {
// Provision sets up the module. // Provision sets up the module.
func (nw *NetWriter) Provision(ctx caddy.Context) error { func (nw *NetWriter) Provision(ctx caddy.Context) error {
nw.logger = slog.Default()
repl := caddy.NewReplacer() repl := caddy.NewReplacer()
address, err := repl.ReplaceOrErr(nw.Address, true, true) address, err := repl.ReplaceOrErr(nw.Address, true, true)
if err != nil { if err != nil {
@ -77,43 +125,428 @@ func (nw *NetWriter) Provision(ctx caddy.Context) error {
return fmt.Errorf("timeout cannot be less than 0") return fmt.Errorf("timeout cannot be less than 0")
} }
if nw.DialTimeout == 0 {
nw.DialTimeout = caddy.Duration(10 * time.Second)
}
if nw.ReconnectInterval == 0 {
nw.ReconnectInterval = caddy.Duration(10 * time.Second)
}
return nil return nil
} }
func (nw NetWriter) String() string { func (nw *NetWriter) String() string {
return nw.addr.String() return nw.addr.String()
} }
// WriterKey returns a unique key representing this nw. // WriterKey returns a unique key representing this nw.
func (nw NetWriter) WriterKey() string { func (nw *NetWriter) WriterKey() string {
return nw.addr.String() return nw.addr.String()
} }
// OpenWriter opens a new network connection. // OpenWriter opens a new network connection and sets up the WAL.
func (nw NetWriter) OpenWriter() (io.WriteCloser, error) { func (nw *NetWriter) OpenWriter() (io.WriteCloser, error) {
reconn := &redialerConn{ // set up WAL directory
nw: nw, baseDir := caddy.AppDataDir()
timeout: time.Duration(nw.DialTimeout), nw.walDir = filepath.Join(baseDir, "wal", "netwriter", strings.ReplaceAll(nw.addr.String(), ":", "-"))
if err := os.MkdirAll(nw.walDir, 0o755); err != nil {
return nil, fmt.Errorf("failed to create WAL directory: %v", err)
} }
conn, err := reconn.dial()
if err != nil { if err := nw.openWAL(); err != nil {
if !nw.SoftStart { return nil, err
return nil, err }
// resume where a previous run left off, if applicable
nw.loadLastProcessed()
nw.pending = !nw.wal.IsEmpty()
// if SoftStart is disabled, test the connection immediately
if !nw.SoftStart {
testConn, err := net.DialTimeout(nw.addr.Network, nw.addr.JoinHostPort(0), time.Duration(nw.DialTimeout))
if err != nil {
return nil, fmt.Errorf("failed to connect to log destination (SoftStart disabled): %v", err)
} }
// don't block config load if remote is down or some other external problem; testConn.Close()
// we can dump logs to stderr for now (see issue #5520)
fmt.Fprintf(os.Stderr, "[ERROR] net log writer failed to connect: %v (will retry connection and print errors here in the meantime)\n", err)
} }
reconn.connMu.Lock()
reconn.Conn = conn // create the writer wrapper
reconn.connMu.Unlock() writer := &netWriterConn{
return reconn, nil nw: nw,
}
// start the background flusher; the cancel function is called by Close
nw.flushCtx, nw.flushCtxCancel = context.WithCancel(context.Background()) //nolint:gosec
nw.flushWg.Add(1)
go nw.backgroundFlusher()
return writer, nil
}
// openWAL opens the write-ahead log in nw.walDir.
func (nw *NetWriter) openWAL() error {
opts := wal.DefaultOptions
opts.DirPath = nw.walDir
opts.SegmentSize = walSegmentSize
w, err := wal.Open(opts)
if err != nil {
return fmt.Errorf("failed to open WAL: %v", err)
}
nw.wal = w
return nil
}
// walMetaFile returns the path of the file that records the position
// of the last delivered WAL entry.
func (nw *NetWriter) walMetaFile() string {
return filepath.Join(nw.walDir, "last_processed")
}
// comparePositions orders two WAL chunk positions; it returns a negative
// number if a comes before b, 0 if they are equal, and a positive number
// otherwise.
func comparePositions(a, b *wal.ChunkPosition) int {
if c := cmp.Compare(a.SegmentId, b.SegmentId); c != 0 {
return c
}
if c := cmp.Compare(a.BlockNumber, b.BlockNumber); c != 0 {
return c
}
return cmp.Compare(a.ChunkOffset, b.ChunkOffset)
}
// loadLastProcessed restores the position of the last delivered entry
// from the metadata file, if present.
func (nw *NetWriter) loadLastProcessed() {
data, err := os.ReadFile(nw.walMetaFile())
if err != nil {
nw.lastProcessed = nil
return
}
var segmentID, blockNumber uint32
var chunkOffset int64
if _, err := fmt.Sscanf(string(data), "%d %d %d", &segmentID, &blockNumber, &chunkOffset); err != nil {
nw.lastProcessed = nil
return
}
nw.lastProcessed = &wal.ChunkPosition{
SegmentId: segmentID,
BlockNumber: blockNumber,
ChunkOffset: chunkOffset,
}
nw.logger.Debug("loaded last processed WAL position",
"segment", nw.lastProcessed.SegmentId,
"block", nw.lastProcessed.BlockNumber,
"offset", nw.lastProcessed.ChunkOffset)
}
// persistLastProcessed writes the position of the last delivered entry
// to the metadata file so that delivery can resume where it left off
// after a restart.
func (nw *NetWriter) persistLastProcessed() {
nw.mu.RLock()
lp := nw.lastProcessed
nw.mu.RUnlock()
if lp == nil {
return
}
data := fmt.Sprintf("%d %d %d", lp.SegmentId, lp.BlockNumber, lp.ChunkOffset)
if err := os.WriteFile(nw.walMetaFile(), []byte(data), 0o600); err != nil {
nw.logger.Error("failed to save WAL position", "error", err)
}
}
// backgroundFlusher runs in the background and flushes WAL entries to the network
func (nw *NetWriter) backgroundFlusher() {
defer nw.flushWg.Done()
nw.logger.Debug("background flusher started")
var conn net.Conn
var connMu sync.RWMutex
// establish a connection
dial := func() error {
newConn, err := net.DialTimeout(nw.addr.Network, nw.addr.JoinHostPort(0), time.Duration(nw.DialTimeout))
if err != nil {
return err
}
connMu.Lock()
if conn != nil {
conn.Close()
}
conn = newConn
connMu.Unlock()
nw.logger.Info("connected to log destination", "address", nw.addr.String())
return nil
}
hasConn := func() bool {
connMu.RLock()
defer connMu.RUnlock()
return conn != nil
}
// write data to the connection
writeToConn := func(data []byte) error {
connMu.RLock()
currentConn := conn
connMu.RUnlock()
if currentConn == nil {
return errors.New("no connection")
}
_, err := currentConn.Write(data)
if err != nil {
// connection failed, clear it so reconnection logic kicks in
connMu.Lock()
if conn == currentConn {
conn.Close()
conn = nil
}
connMu.Unlock()
}
return err
}
// try initial connection
if err := dial(); err != nil {
nw.logger.Warn("failed to connect to log destination, will retry", "error", err)
}
// deliver any backlog left over from a previous run
nw.processWALEntries(writeToConn)
ticker := time.NewTicker(100 * time.Millisecond) // check for new entries every 100ms
defer ticker.Stop()
reconnectTicker := time.NewTicker(time.Duration(nw.ReconnectInterval))
defer reconnectTicker.Stop()
for {
select {
case <-nw.flushCtx.Done():
// best-effort final flush; anything undelivered stays in
// the WAL and will be delivered after the next start
if !hasConn() {
if err := dial(); err != nil {
nw.logger.Warn("cannot connect for final flush", "error", err)
}
}
if hasConn() {
nw.processWALEntries(writeToConn)
}
nw.mu.RLock()
pending := nw.pending
nw.mu.RUnlock()
if pending {
nw.logger.Warn("undelivered log entries remain in the WAL and will be delivered after the next start", "wal_dir", nw.walDir)
}
connMu.Lock()
if conn != nil {
conn.Close()
}
connMu.Unlock()
return
case <-ticker.C:
// deliver new WAL entries, but only if there may be any
// and the destination is reachable; otherwise entries
// keep accumulating in the WAL until reconnection
if !hasConn() {
continue
}
nw.mu.RLock()
pending := nw.pending
nw.mu.RUnlock()
if pending {
nw.processWALEntries(writeToConn)
}
case <-reconnectTicker.C:
// try to reconnect if we don't have a connection
if hasConn() {
continue
}
if err := dial(); err != nil {
nw.logger.Debug("reconnection attempt failed", "error", err)
continue
}
// successfully reconnected, deliver any buffered WAL entries
nw.processWALEntries(writeToConn)
}
}
}
// processWALEntries delivers all undelivered WAL entries to the network
// connection. If a write fails, delivery stops and the remaining backlog
// is kept in the WAL to be retried once the connection is restored.
func (nw *NetWriter) processWALEntries(writeToConn func([]byte) error) {
// optimistically mark the backlog as flushed; this is set again if
// delivery fails partway or if an entry is written concurrently
nw.mu.Lock()
nw.pending = false
lastProcessed := nw.lastProcessed
nw.mu.Unlock()
nw.walMu.Lock()
reader := nw.wal.NewReader()
nw.walMu.Unlock()
delivered := 0
caughtUp := true
for {
nw.walMu.Lock()
data, cp, err := reader.Next()
nw.walMu.Unlock()
if err == io.EOF {
break
}
if err != nil {
nw.logger.Error("error reading from WAL", "error", err)
caughtUp = false
break
}
// skip entries that were already delivered
if lastProcessed != nil && comparePositions(cp, lastProcessed) <= 0 {
continue
}
if err := writeToConn(data); err != nil {
// connection is down; keep the backlog in the WAL and let
// the reconnect logic trigger another delivery attempt
caughtUp = false
break
}
nw.mu.Lock()
nw.lastProcessed = cp
nw.mu.Unlock()
lastProcessed = cp
delivered++
nw.deliveredSinceTruncate += int64(len(data))
}
if !caughtUp {
nw.mu.Lock()
nw.pending = true
nw.mu.Unlock()
}
if delivered > 0 {
nw.persistLastProcessed()
nw.logger.Debug("delivered WAL entries", "count", delivered)
}
if caughtUp {
nw.maybeTruncateWAL()
}
}
// maybeTruncateWAL discards delivered WAL entries by deleting and
// re-opening the WAL once enough data has been delivered, reclaiming
// disk space; without this, the WAL would grow without bound. It must
// only be called by the background flusher, and only when the backlog
// has been fully delivered.
func (nw *NetWriter) maybeTruncateWAL() {
if nw.deliveredSinceTruncate < walTruncateThreshold {
return
}
nw.walMu.Lock()
defer nw.walMu.Unlock()
nw.mu.Lock()
defer nw.mu.Unlock()
// an entry may have been written after the backlog was drained;
// truncating now would discard it, so wait for the next chance
if nw.pending {
return
}
if err := nw.wal.Delete(); err != nil {
nw.logger.Error("failed to truncate WAL", "error", err)
return
}
if err := nw.openWAL(); err != nil {
nw.logger.Error("failed to re-open WAL after truncation", "error", err)
return
}
// positions restart in the new WAL, so the old ones no longer apply
nw.lastProcessed = nil
nw.deliveredSinceTruncate = 0
if err := os.Remove(nw.walMetaFile()); err != nil && !errors.Is(err, fs.ErrNotExist) {
nw.logger.Error("failed to remove WAL position file", "error", err)
}
nw.logger.Debug("truncated WAL")
}
// netWriterConn implements io.WriteCloser and writes to the WAL
type netWriterConn struct {
nw *NetWriter
}
// Write appends the entry to the WAL; the background flusher delivers
// it to the network destination asynchronously.
func (w *netWriterConn) Write(p []byte) (n int, err error) {
nw := w.nw
nw.walMu.Lock()
defer nw.walMu.Unlock()
if nw.wal == nil {
return 0, errors.New("WAL not initialized")
}
if _, err = nw.wal.Write(p); err != nil {
// the entry cannot be persisted, so dump it to stderr
// rather than losing it silently
os.Stderr.Write(p)
return 0, fmt.Errorf("failed to write to WAL: %v", err)
}
nw.mu.Lock()
nw.pending = true
nw.mu.Unlock()
return len(p), nil
}
// Close closes the writer and flushes all remaining data
func (w *netWriterConn) Close() error {
if w.nw.flushCtxCancel != nil {
w.nw.flushCtxCancel()
}
// wait for the background flusher to finish its final flush
w.nw.flushWg.Wait()
var errs []error
w.nw.walMu.Lock()
if w.nw.wal != nil {
if err := w.nw.wal.Sync(); err != nil {
errs = append(errs, fmt.Errorf("WAL sync error: %v", err))
}
if err := w.nw.wal.Close(); err != nil {
errs = append(errs, fmt.Errorf("WAL close error: %v", err))
}
}
w.nw.walMu.Unlock()
return errors.Join(errs...)
} }
// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax: // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
// //
// net <address> { // net <address> {
// dial_timeout <duration> // dial_timeout <duration>
// reconnect_interval <duration>
// soft_start // soft_start
// } // }
func (nw *NetWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { func (nw *NetWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
@ -140,6 +573,19 @@ func (nw *NetWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
} }
nw.DialTimeout = caddy.Duration(timeout) nw.DialTimeout = caddy.Duration(timeout)
case "reconnect_interval":
if !d.NextArg() {
return d.ArgErr()
}
interval, err := caddy.ParseDuration(d.Val())
if err != nil {
return d.Errf("invalid duration: %s", d.Val())
}
if d.NextArg() {
return d.ArgErr()
}
nw.ReconnectInterval = caddy.Duration(interval)
case "soft_start": case "soft_start":
if d.NextArg() { if d.NextArg() {
return d.ArgErr() return d.ArgErr()
@ -153,71 +599,6 @@ func (nw *NetWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
return nil return nil
} }
// redialerConn wraps an underlying Conn so that if any
// writes fail, the connection is redialed and the write
// is retried.
type redialerConn struct {
net.Conn
connMu sync.RWMutex
nw NetWriter
timeout time.Duration
lastRedial time.Time
}
// Write wraps the underlying Conn.Write method, but if that fails,
// it will re-dial the connection anew and try writing again.
func (reconn *redialerConn) Write(b []byte) (n int, err error) {
reconn.connMu.RLock()
conn := reconn.Conn
reconn.connMu.RUnlock()
if conn != nil {
if n, err = conn.Write(b); err == nil {
return n, err
}
}
// problem with the connection - lock it and try to fix it
reconn.connMu.Lock()
defer reconn.connMu.Unlock()
// if multiple concurrent writes failed on the same broken conn, then
// one of them might have already re-dialed by now; try writing again
if reconn.Conn != nil {
if n, err = reconn.Conn.Write(b); err == nil {
return n, err
}
}
// there's still a problem, so try to re-attempt dialing the socket
// if some time has passed in which the issue could have potentially
// been resolved - we don't want to block at every single log
// emission (!) - see discussion in #4111
if time.Since(reconn.lastRedial) > 10*time.Second {
reconn.lastRedial = time.Now()
conn2, err2 := reconn.dial()
if err2 != nil {
// logger socket still offline; instead of discarding the log, dump it to stderr
os.Stderr.Write(b)
return n, err
}
if n, err = conn2.Write(b); err == nil {
if reconn.Conn != nil {
reconn.Conn.Close()
}
reconn.Conn = conn2
}
} else {
// last redial attempt was too recent; just dump to stderr for now
os.Stderr.Write(b)
}
return n, err
}
func (reconn *redialerConn) dial() (net.Conn, error) {
return net.DialTimeout(reconn.nw.addr.Network, reconn.nw.addr.JoinHostPort(0), reconn.timeout)
}
// Interface guards // Interface guards
var ( var (
_ caddy.Provisioner = (*NetWriter)(nil) _ caddy.Provisioner = (*NetWriter)(nil)

File diff suppressed because it is too large Load diff