diff --git a/pkg/app/carbonapi/app.go b/pkg/app/carbonapi/app.go index a7f76d95..495c3df3 100644 --- a/pkg/app/carbonapi/app.go +++ b/pkg/app/carbonapi/app.go @@ -51,6 +51,11 @@ type App struct { defaultTimeZone *time.Location + // RequestPrefixes is the validated list of metric-name prefixes used to + // emit the requests_per_prefix_total counter. Populated from + // config.RequestsPerPrefixList at construction time. + RequestPrefixes []string + // During processing we use two independent queues that share a semaphore to prevent stampeding. // fastQ includes regular requests fastQ chan *RenderReq @@ -83,6 +88,8 @@ func New(config cfg.API, lg *zap.Logger, buildVersion string) (*App, error) { } app.requestBlocker.ReloadRules() + app.RequestPrefixes = initRequestPrefixes(config.RequestsPerPrefixList, lg) + setUpConfig(app, lg) app.ZipperConfig, app.Backends, app.TopLevelDomainCache, app.TopLevelDomainPrefixes, app.ZipperMetrics = SetupZipper(config.ZipperConfig, BuildVersion, &ms, lg) @@ -258,6 +265,34 @@ func setUpConfig(app *App, logger *zap.Logger) { } +// initRequestPrefixes validates the configured prefix list for the +// requests_per_prefix_total counter: drops empty entries, drops entries +// that contain glob characters (those cannot be matched literally), and +// dedupes the result while preserving input order. +func initRequestPrefixes(cfgPrefixes []string, lg *zap.Logger) []string { + if len(cfgPrefixes) == 0 { + return nil + } + seen := make(map[string]struct{}, len(cfgPrefixes)) + out := make([]string, 0, len(cfgPrefixes)) + for _, p := range cfgPrefixes { + if p == "" { + lg.Warn("requestsPerPrefixList: ignoring empty prefix") + continue + } + if strings.ContainsAny(p, "*?[{") { + lg.Warn("requestsPerPrefixList: ignoring prefix with glob characters", zap.String("prefix", p)) + continue + } + if _, dup := seen[p]; dup { + continue + } + seen[p] = struct{}{} + out = append(out, p) + } + return out +} + func (app *App) deferredAccessLogging(accessLogger *zap.Logger, r *http.Request, accessLogDetails *carbonapipb.AccessLogDetails, t time.Time, level zapcore.Level) { accessLogDetails.Runtime = time.Since(t).Seconds() accessLogDetails.RequestMethod = r.Method diff --git a/pkg/app/carbonapi/http_handlers.go b/pkg/app/carbonapi/http_handlers.go index 5810d760..9c1ad6d2 100644 --- a/pkg/app/carbonapi/http_handlers.go +++ b/pkg/app/carbonapi/http_handlers.go @@ -192,6 +192,16 @@ func (app *App) renderHandler(w http.ResponseWriter, r *http.Request, lg *zap.Lo return } + matchedPrefixes := collectMatchedPrefixes(form.targets, app.RequestPrefixes) + defer func() { + if toLog.HttpCode/100 != 2 { + return + } + for p := range matchedPrefixes { + app.ms.RequestsPerPrefix.WithLabelValues(p).Inc() + } + }() + if form.useCache { Trace(lg, "query request cache") @@ -1601,3 +1611,45 @@ func addCacheErrorToLogDetails(d *carbonapipb.AccessLogDetails, isRead bool, err } d.CacheErrs += prefix + err.Error() + "," } + +// metricRefsPrefix reports whether `metric` directly references `prefix`: +// metric == prefix, or metric begins with `prefix + "."`. Glob characters +// in metric past the prefix boundary are irrelevant, but `prefix` itself +// is matched literally and is expected to be glob-free. +func metricRefsPrefix(metric, prefix string) bool { + if !strings.HasPrefix(metric, prefix) { + return false + } + if len(metric) == len(prefix) { + return true + } + return metric[len(prefix)] == '.' +} + +// collectMatchedPrefixes parses each target string and returns the set of +// configuredPrefixes that are directly referenced by at least one metric in +// any of the parsed expressions (see metricRefsPrefix). Targets that fail to +// parse are silently skipped. Returns nil when configuredPrefixes is empty. +func collectMatchedPrefixes(targets []string, configuredPrefixes []string) map[string]struct{} { + if len(configuredPrefixes) == 0 { + return nil + } + matched := make(map[string]struct{}) + for _, target := range targets { + exp, e, parseErr := parser.ParseExpr(target) + if parseErr != nil || e != "" { + continue + } + for _, m := range exp.Metrics() { + for _, p := range configuredPrefixes { + if _, already := matched[p]; already { + continue + } + if metricRefsPrefix(m.Metric, p) { + matched[p] = struct{}{} + } + } + } + } + return matched +} diff --git a/pkg/app/carbonapi/http_handlers_test.go b/pkg/app/carbonapi/http_handlers_test.go index 83f1bf35..4da3c95c 100644 --- a/pkg/app/carbonapi/http_handlers_test.go +++ b/pkg/app/carbonapi/http_handlers_test.go @@ -5,6 +5,7 @@ import ( "testing" typ "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" ) func TestGetCompleterQuery(t *testing.T) { @@ -215,3 +216,145 @@ func TestOptimistErrsFanIn(t *testing.T) { }) } } + +func TestMetricRefsPrefix(t *testing.T) { + tests := []struct { + metric string + prefix string + want bool + }{ + {"a.b.c.*", "a.b", true}, + {"a.*", "a.b", false}, + {"a.b", "a.b", true}, + {"a.bc", "a.b", false}, + {"a.b.c", "a", true}, + {"apple.x", "a", false}, + {"{a,x}.b.c", "a", false}, + {"a.b.c.d", "a.b", true}, + {"", "a", false}, + {"a", "a", true}, + {"a", "a.b.c", false}, + } + + for _, tt := range tests { + got := metricRefsPrefix(tt.metric, tt.prefix) + if got != tt.want { + t.Errorf("metricRefsPrefix(%q, %q) = %v, want %v", tt.metric, tt.prefix, got, tt.want) + } + } +} + +func TestCollectMatchedPrefixes(t *testing.T) { + tests := []struct { + name string + targets []string + prefixes []string + want []string // sorted expected matches; nil means no matches (empty or nil map) + }{ + { + name: "no configured prefixes", + targets: []string{"a.b.c.*"}, + prefixes: nil, + want: nil, + }, + { + name: "single target matches single prefix", + targets: []string{"a.b.c.*"}, + prefixes: []string{"a.b"}, + want: []string{"a.b"}, + }, + { + name: "single target does not match prefix", + targets: []string{"x.y.z"}, + prefixes: []string{"a.b"}, + want: nil, + }, + { + name: "multiple targets, one matches", + targets: []string{"x.y.z", "a.b.c"}, + prefixes: []string{"a.b"}, + want: []string{"a.b"}, + }, + { + name: "multiple targets match different prefixes", + targets: []string{"a.b.c", "x.y.z"}, + prefixes: []string{"a.b", "x.y"}, + want: []string{"a.b", "x.y"}, + }, + { + name: "target with function expression", + targets: []string{"sum(a.b.c.*)"}, + prefixes: []string{"a.b"}, + want: []string{"a.b"}, + }, + { + name: "unparseable target is skipped, valid target still counted", + targets: []string{"(((bad target", "a.b.c"}, + prefixes: []string{"a.b"}, + want: []string{"a.b"}, + }, + { + name: "prefix only matched at dot boundary", + targets: []string{"a.bc.d"}, + prefixes: []string{"a.b"}, + want: nil, + }, + { + name: "exact match counts", + targets: []string{"a.b"}, + prefixes: []string{"a.b"}, + want: []string{"a.b"}, + }, + { + name: "each prefix counted at most once across all targets", + targets: []string{"a.b.c", "a.b.d"}, + prefixes: []string{"a.b"}, + want: []string{"a.b"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := collectMatchedPrefixes(tt.targets, tt.prefixes) + if len(got) != len(tt.want) { + t.Fatalf("collectMatchedPrefixes(%v, %v) = %v, want %v", tt.targets, tt.prefixes, got, tt.want) + } + for _, p := range tt.want { + if _, ok := got[p]; !ok { + t.Errorf("collectMatchedPrefixes(%v, %v): missing prefix %q in result %v", tt.targets, tt.prefixes, p, got) + } + } + }) + } +} + +func TestInitRequestPrefixes(t *testing.T) { + lg := zap.NewNop() + + tests := []struct { + name string + in []string + want []string + }{ + {"nil input", nil, nil}, + {"empty input", []string{}, nil}, + {"drops empty entries", []string{"a.b", "", "x"}, []string{"a.b", "x"}}, + {"drops glob entries", []string{"a.b", "a.*", "a.b{c,d}", "a?b", "x[1-2]"}, []string{"a.b"}}, + {"dedupes", []string{"a.b", "x", "a.b", "x"}, []string{"a.b", "x"}}, + {"preserves order", []string{"z", "a", "m"}, []string{"z", "a", "m"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := initRequestPrefixes(tt.in, lg) + if len(got) != len(tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("got %v, want %v", got, tt.want) + } + } + }) + } +} diff --git a/pkg/app/carbonapi/metrics.go b/pkg/app/carbonapi/metrics.go index 3ebc3f6d..66a21925 100644 --- a/pkg/app/carbonapi/metrics.go +++ b/pkg/app/carbonapi/metrics.go @@ -12,6 +12,8 @@ type PrometheusMetrics struct { RenderPartialFail prometheus.Counter RequestCancel *prometheus.CounterVec + RequestsPerPrefix *prometheus.CounterVec + DurationTotal *prometheus.HistogramVec UpstreamDuration *prometheus.HistogramVec UpstreamTimeInQSec *prometheus.HistogramVec @@ -97,6 +99,14 @@ func newPrometheusMetrics(config cfg.API) PrometheusMetrics { []string{"handler", "cause"}, ), + RequestsPerPrefix: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "requests_per_prefix_total", + Help: "Count of /render requests where at least one metric pattern directly references the configured prefix (i.e. equals the prefix or starts with `.`).", + }, + []string{"prefix"}, + ), + DurationTotal: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "duration_total_seconds", Help: "The total duration of an HTTP request in seconds.", @@ -280,6 +290,7 @@ func registerPrometheusMetrics(ms *PrometheusMetrics, zms *ZipperPrometheusMetri prometheus.MustRegister(ms.FindNotFound) prometheus.MustRegister(ms.RenderPartialFail) prometheus.MustRegister(ms.RequestCancel) + prometheus.MustRegister(ms.RequestsPerPrefix) prometheus.MustRegister(ms.DurationTotal) prometheus.MustRegister(ms.UpstreamDuration) diff --git a/pkg/cfg/api.go b/pkg/cfg/api.go index 229776c2..80e84bd1 100644 --- a/pkg/cfg/api.go +++ b/pkg/cfg/api.go @@ -168,6 +168,14 @@ type API struct { ZipperConfig string `yaml:"zipperConfig"` SimpleRequestThreshold int `yaml:"simpleRequestThreshold"` + + // RequestsPerPrefixList configures the prefixes used to emit the + // requests_per_prefix_total counter. Each entry is a dot-separated + // metric-name prefix (e.g. "a.b"). A request is counted toward a + // prefix when at least one of its metric patterns equals the prefix + // or begins with `.`. Empty entries and entries containing + // glob characters are ignored. + RequestsPerPrefixList []string `yaml:"requestsPerPrefixList"` } // CacheConfig configs the cache