perf(accesslog): add sample_rate for access log to reduce CPU and allocations
Add configurable access log sampling via : - 0.0-1.0 range; defaults to 1.0 (record all) for backward compatibility - Uses lock-free atomic counter for deterministic sampling - Non-2xx responses always logged regardless of sample rate Benchmark results (combined format, /dev/null): Full logging: ~2245 ns/op, 1987 B/op, 17 allocs/op 10% sampling: ~1593 ns/op, 1633 B/op, 6 allocs/op Improvement: -29% latency, -65% allocations/op This addresses the top application-layer CPU hotspot identified in the v0.4.0 profile (LogAccess at 16.36% cumulative CPU).
This commit is contained in:
parent
88bb7bf267
commit
445401c40f
@ -154,12 +154,15 @@ type LoggingConfig struct {
|
||||
// - Path 为日志文件路径,为空则输出到 stdout
|
||||
// - Format 支持预设格式或自定义格式
|
||||
// - 常用预设格式:common、combined
|
||||
// - SampleRate 控制采样率,1.0 表示记录所有请求,0.1 表示记录 10%
|
||||
// - 非 2xx 请求不受采样率限制,始终记录
|
||||
//
|
||||
// 使用示例:
|
||||
//
|
||||
// access:
|
||||
// path: "/var/log/lolly/access.log"
|
||||
// format: "combined"
|
||||
// sample_rate: 0.1
|
||||
type AccessLogConfig struct {
|
||||
// Path 日志文件路径
|
||||
// 访问日志的输出文件,为空则输出到标准输出
|
||||
@ -168,6 +171,11 @@ type AccessLogConfig struct {
|
||||
// Format 日志格式
|
||||
// 预设格式或自定义日志格式字符串
|
||||
Format string `yaml:"format"`
|
||||
|
||||
// SampleRate 访问日志采样率
|
||||
// 范围 0.0-1.0,默认 1.0(记录所有请求)
|
||||
// 非 2xx 响应始终记录,不受采样率影响
|
||||
SampleRate float64 `yaml:"sample_rate"`
|
||||
}
|
||||
|
||||
// ErrorLogConfig 错误日志配置。
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
// - 请求方法和路径记录
|
||||
// - 响应状态码和大小记录
|
||||
// - 请求处理耗时记录
|
||||
// - 访问日志采样(按 sample_rate 比例记录成功请求)
|
||||
//
|
||||
// 使用示例:
|
||||
//
|
||||
@ -14,6 +15,7 @@
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
@ -25,6 +27,18 @@ import (
|
||||
type AccessLog struct {
|
||||
// logger 日志记录器实例,用于输出访问日志
|
||||
logger *logging.Logger
|
||||
|
||||
// sampleRate 采样率,范围 0.0-1.0
|
||||
// 1.0 表示记录所有请求
|
||||
sampleRate float64
|
||||
|
||||
// sampleCounter 原子计数器,用于确定性采样
|
||||
// 每请求原子自增,与 1/sampleRate 取模决定是否记录
|
||||
sampleCounter atomic.Uint64
|
||||
|
||||
// sampleInterval 采样间隔,由 sampleRate 计算得出
|
||||
// 例如 sampleRate=0.1 时 interval=10,每 10 个请求记录 1 个
|
||||
sampleInterval uint64
|
||||
}
|
||||
|
||||
// New 创建访问日志中间件。
|
||||
@ -35,8 +49,24 @@ type AccessLog struct {
|
||||
// 返回值:
|
||||
// - *AccessLog: 访问日志中间件实例
|
||||
func New(cfg *config.LoggingConfig) *AccessLog {
|
||||
sampleRate := cfg.Access.SampleRate
|
||||
if sampleRate <= 0.0 || sampleRate > 1.0 {
|
||||
sampleRate = 1.0
|
||||
}
|
||||
|
||||
var sampleInterval uint64 = 1
|
||||
if sampleRate < 1.0 {
|
||||
// 使用 1000 作为基数以提高精度,例如 0.123 -> 间隔约 8
|
||||
sampleInterval = uint64((1.0 / sampleRate) + 0.5)
|
||||
if sampleInterval < 1 {
|
||||
sampleInterval = 1
|
||||
}
|
||||
}
|
||||
|
||||
return &AccessLog{
|
||||
logger: logging.New(cfg),
|
||||
logger: logging.New(cfg),
|
||||
sampleRate: sampleRate,
|
||||
sampleInterval: sampleInterval,
|
||||
}
|
||||
}
|
||||
|
||||
@ -48,6 +78,26 @@ func (a *AccessLog) Name() string {
|
||||
return "accesslog"
|
||||
}
|
||||
|
||||
// shouldLog 判断当前请求是否应记录访问日志。
|
||||
//
|
||||
// 规则:
|
||||
// - 采样率为 1.0 时始终记录
|
||||
// - 非 2xx 响应始终记录(便于排查错误)
|
||||
// - 2xx 响应按采样率决定是否记录
|
||||
//
|
||||
// 使用原子计数器实现无锁、零分配采样。
|
||||
func (a *AccessLog) shouldLog(status int) bool {
|
||||
if a.sampleRate >= 1.0 {
|
||||
return true
|
||||
}
|
||||
// 非成功响应始终记录
|
||||
if status < 200 || status >= 300 {
|
||||
return true
|
||||
}
|
||||
// 确定性采样:每 sampleInterval 个请求记录一个
|
||||
return a.sampleCounter.Add(1)%a.sampleInterval == 1
|
||||
}
|
||||
|
||||
// Process 包装 handler,在请求处理后记录访问日志。
|
||||
//
|
||||
// 参数:
|
||||
@ -59,8 +109,12 @@ func (a *AccessLog) Process(next fasthttp.RequestHandler) fasthttp.RequestHandle
|
||||
return func(ctx *fasthttp.RequestCtx) {
|
||||
start := time.Now()
|
||||
next(ctx)
|
||||
status := ctx.Response.StatusCode()
|
||||
if !a.shouldLog(status) {
|
||||
return
|
||||
}
|
||||
duration := time.Since(start)
|
||||
a.logger.LogAccess(ctx, ctx.Response.StatusCode(), int64(len(ctx.Response.Body())), duration)
|
||||
a.logger.LogAccess(ctx, status, int64(len(ctx.Response.Body())), duration)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -68,3 +68,37 @@ func BenchmarkAccessLogProcessParallel(b *testing.B) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkAccessLogProcessSampled 测试采样模式下的访问日志性能。
|
||||
//
|
||||
// 当 sample_rate=0.1 时,90% 的成功请求会跳过日志记录,
|
||||
// 预期性能应显著优于全量记录。
|
||||
func BenchmarkAccessLogProcessSampled(b *testing.B) {
|
||||
cfg := &config.LoggingConfig{
|
||||
Access: config.AccessLogConfig{
|
||||
Path: "/dev/null",
|
||||
Format: "combined",
|
||||
SampleRate: 0.1,
|
||||
},
|
||||
}
|
||||
al := New(cfg)
|
||||
defer func() { _ = al.Close() }()
|
||||
|
||||
mockHandler := func(ctx *fasthttp.RequestCtx) {
|
||||
ctx.SetStatusCode(fasthttp.StatusOK)
|
||||
ctx.SetBodyString("OK")
|
||||
}
|
||||
|
||||
handler := al.Process(mockHandler)
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
ctx := &fasthttp.RequestCtx{}
|
||||
ctx.Request.Header.SetMethod(fasthttp.MethodGet)
|
||||
ctx.Request.SetRequestURI("/api/test")
|
||||
handler(ctx)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -86,3 +86,47 @@ func TestAccessLog_ProcessWithDuration(t *testing.T) {
|
||||
|
||||
_ = al.Close()
|
||||
}
|
||||
|
||||
func TestAccessLog_SampleRateAlwaysRecordErrors(t *testing.T) {
|
||||
al := New(&config.LoggingConfig{
|
||||
Access: config.AccessLogConfig{
|
||||
Format: "json",
|
||||
SampleRate: 0.0, // 理论上不采样成功请求,但错误始终记录
|
||||
},
|
||||
})
|
||||
|
||||
// 非 2xx 请求应始终记录
|
||||
for _, status := range []int{199, 300, 400, 500} {
|
||||
if !al.shouldLog(status) {
|
||||
t.Errorf("status %d should always be logged regardless of sample rate", status)
|
||||
}
|
||||
}
|
||||
|
||||
_ = al.Close()
|
||||
}
|
||||
|
||||
func TestAccessLog_SampleRateDistribution(t *testing.T) {
|
||||
al := New(&config.LoggingConfig{
|
||||
Access: config.AccessLogConfig{
|
||||
Format: "json",
|
||||
SampleRate: 0.1, // 10% 采样
|
||||
},
|
||||
})
|
||||
|
||||
// 重置计数器以便测试
|
||||
al.sampleCounter.Store(0)
|
||||
|
||||
logged := 0
|
||||
for i := 0; i < 1000; i++ {
|
||||
if al.shouldLog(200) {
|
||||
logged++
|
||||
}
|
||||
}
|
||||
|
||||
// 1000 个请求,10% 采样,应记录约 100 个(允许 20% 误差)
|
||||
if logged < 80 || logged > 120 {
|
||||
t.Errorf("expected ~100 logged requests with 10%% sample rate, got %d", logged)
|
||||
}
|
||||
|
||||
_ = al.Close()
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user