diff --git a/src/lib/alert.ts b/src/lib/alert.ts index d9a5d486..98fbd8ca 100644 --- a/src/lib/alert.ts +++ b/src/lib/alert.ts @@ -18,33 +18,42 @@ export const ALERT_METRICS: AlertMetricMeta[] = [ { value: 'egress', label: 'Egress (bytes per minute)' } ] +export const ALERT_CUSTOM_METRICS: AlertMetricMeta[] = [ + { value: 'value', label: 'Value (gauge)' }, + { value: 'rate', label: 'Rate (per minute)' } +] + export const ALERT_OPS = [ { value: '>=', label: '>= (at or above)' }, { value: '<=', label: '<= (at or below)' } ] export function alertMetricLabel (metric: string): string { - return ALERT_METRICS.find((m) => m.value === metric)?.label ?? metric + return ALERT_METRICS.concat(ALERT_CUSTOM_METRICS).find((m) => m.value === metric)?.label ?? metric } /** * Format a threshold/value for its metric's unit — percent for cpu/memory, - * binary bytes/min for egress, req/min otherwise. + * binary bytes/min for egress, the raw gauge for kind=custom value, per-minute + * for rate/requests. */ export function alertThresholdString (metric: string, value: number): string { if (metric === 'cpu' || metric === 'memory') return `${value}%` if (metric === 'egress') return `${format.storage(value)}/min` + if (metric === 'value') return String(value) return `${value}/min` } /** - * Human-readable one-liner for a condition, e.g. "cpu >= 90% for 10m". + * Human-readable one-liner for a condition, e.g. "cpu >= 90% for 10m" or + * "value >= 10 for 5m". */ export function alertConditionString (c: Api.AlertCondition): string { return `${c.metric} ${c.op} ${alertThresholdString(c.metric, c.threshold)} for ${c.forMinutes}m` } export function alertTargetString (t: Api.AlertTarget): string { + if (t.kind === 'custom') return `${t.source} / ${t.series}` return `${t.location} / ${t.deployment}` } diff --git a/src/lib/components/CustomMetricsChart.svelte b/src/lib/components/CustomMetricsChart.svelte new file mode 100644 index 00000000..1094ad23 --- /dev/null +++ b/src/lib/components/CustomMetricsChart.svelte @@ -0,0 +1,185 @@ + + +{#if source.truncated} + +{/if} +{#if source.lastError} + +{/if} +{#if queryError} + +{/if} + +
+
+ + +
+ +
+ +{#if isEmpty} + +{:else} + +{/if} + + diff --git a/src/lib/metricsource.ts b/src/lib/metricsource.ts new file mode 100644 index 00000000..5e533b8b --- /dev/null +++ b/src/lib/metricsource.ts @@ -0,0 +1,20 @@ +/** + * Status + chart helpers for metric sources. Mirrors api/metricsource.go's + * metricSourceStatus (disabled → error → truncated → ok). + */ +import type { MetricSeries } from '$lib/charts/util' + +export type MetricSourceStatus = 'ok' | 'disabled' | 'truncated' | 'error' + +export function metricSourceStatus (s: Api.MetricSourceItem): MetricSourceStatus { + if (s.disabled) return 'disabled' + if (s.lastError) return 'error' + if (s.truncated) return 'truncated' + return 'ok' +} + +export const TRUNCATED_BANNER = 'series cap hit — extra series were dropped' + +export function metricSourceChartSeries (items: Api.UsageMetricsLine[]): MetricSeries[] { + return items.map((l) => ({ prefix: l.name, lines: [l] })) +} diff --git a/src/lib/nav.ts b/src/lib/nav.ts index 0da3688e..a51d2d32 100644 --- a/src/lib/nav.ts +++ b/src/lib/nav.ts @@ -38,5 +38,6 @@ export const projectMenu: ProjectMenuItem[] = [ { id: 'scheduler', title: 'Scheduler', icon: 'fa-clock', link: '/scheduler', preview: true }, { id: 'notification', title: 'Notifications', icon: 'fa-bell', link: '/notification', preview: true }, { id: 'alert', title: 'Alerts', icon: 'fa-bell-exclamation', link: '/alert', preview: true }, + { id: 'metric-source', title: 'Metric sources', icon: 'fa-satellite-dish', link: '/metrics-sources', preview: true }, { id: 'audit-log', title: 'Audit Logs', icon: 'fa-clipboard-list', link: '/audit-log' } ] diff --git a/src/lib/server/mock.ts b/src/lib/server/mock.ts index bcf1525c..295dce6f 100644 --- a/src/lib/server/mock.ts +++ b/src/lib/server/mock.ts @@ -1149,7 +1149,7 @@ const notificationDeliveries = [ interface AlertRuleFixture { project: string name: string - target: { location: string, deployment: string } + target: { kind?: string, location?: string, deployment?: string, source?: string, series?: string } condition: { metric: string, op: string, threshold: number, forMinutes: number } renotifyMinutes: number disabled: boolean @@ -1214,6 +1214,22 @@ const alertRules: AlertRuleFixture[] = [ updatedAt: CREATED_AT, updatedBy: USER_EMAIL }, + { + project: 'acme', + name: 'web-queue-depth', + target: { kind: 'custom', source: 'web', series: 'queue_depth{queue="email"}' }, + condition: { metric: 'value', op: '>=', threshold: 10, forMinutes: 5 }, + renotifyMinutes: 0, + disabled: false, + status: 'ok', + lastValue: 4, + firingSince: null, + lastEvaluatedAt: CREATED_AT, + createdAt: CREATED_AT, + createdBy: USER_EMAIL, + updatedAt: CREATED_AT, + updatedBy: USER_EMAIL + }, { project: 'acme', name: 'website-egress-spike', @@ -1245,6 +1261,92 @@ const alertEvents: AlertEventFixture[] = [ { at: CREATED_AT, transition: 'trigger', value: 88.4 } ] +interface MetricSourceFixture { + project: string + name: string + location: string + deployment: string + port: number + path: string + disabled: boolean + truncated: boolean + lastScrapedAt: string | null + lastError: string + createdAt: string + createdBy: string + updatedAt: string + updatedBy: string +} + +interface MetricSourceSeriesFixture { + series: string + type: 'gauge' | 'counter' | 'untyped' + lastSeenAt: string +} + +const defaultMetricSourceSeries: MetricSourceSeriesFixture[] = [ + { series: 'queue_depth{queue="email"}', type: 'gauge', lastSeenAt: CREATED_AT }, + { series: 'http_requests_total{code="200"}', type: 'counter', lastSeenAt: CREATED_AT }, + { series: 'up', type: 'untyped', lastSeenAt: CREATED_AT } +] + +const metricSources: MetricSourceFixture[] = [ + { + project: 'acme', + name: 'web', + location: LOCATION_ID, + deployment: 'web', + port: 9090, + path: '/metrics', + disabled: false, + truncated: false, + lastScrapedAt: CREATED_AT, + lastError: '', + createdAt: CREATED_AT, + createdBy: USER_EMAIL, + updatedAt: CREATED_AT, + updatedBy: USER_EMAIL + }, + { + project: 'acme', + name: 'api-metrics', + location: LOCATION_ID, + deployment: 'api', + port: 9090, + path: '/metrics', + disabled: false, + truncated: true, + lastScrapedAt: CREATED_AT, + lastError: '', + createdAt: CREATED_AT, + createdBy: USER_EMAIL, + updatedAt: CREATED_AT, + updatedBy: USER_EMAIL + }, + { + project: 'acme', + name: 'worker-metrics', + location: LOCATION_ID, + deployment: 'worker', + port: 9090, + path: '/metrics', + disabled: false, + truncated: false, + lastScrapedAt: CREATED_AT, + lastError: 'scrape failed: connection refused', + createdAt: CREATED_AT, + createdBy: USER_EMAIL, + updatedAt: CREATED_AT, + updatedBy: USER_EMAIL + } +] + +const metricSourceSeriesByName: Record = { + web: defaultMetricSourceSeries, + 'api-metrics': defaultMetricSourceSeries, + 'worker-metrics': defaultMetricSourceSeries +} + const roles = [ { role: 'viewer', @@ -2613,6 +2715,64 @@ const handlers: Record object> = { return list(alertEvents) }, + 'metricSource.list': () => list(metricSources), + 'metricSource.get': (args) => { + const item = metricSources.find((s) => s.name === args?.name) + if (!item) return err('api: metric source not found') + return ok(item) + }, + 'metricSource.set': (args) => { + const name = String(args?.name ?? '') + const existing = metricSources.find((s) => s.name === name) + const next: MetricSourceFixture = { + project: args?.project ?? 'acme', + name, + location: args?.location ?? '', + deployment: args?.deployment ?? '', + port: args?.port ?? 9090, + path: args?.path || '/metrics', + disabled: args?.disabled ?? false, + truncated: existing?.truncated ?? false, + lastScrapedAt: existing?.lastScrapedAt ?? null, + lastError: existing?.lastError ?? '', + createdAt: existing?.createdAt ?? CREATED_AT, + createdBy: existing?.createdBy ?? USER_EMAIL, + updatedAt: CREATED_AT, + updatedBy: USER_EMAIL + } + if (existing) { + Object.assign(existing, next) + } else { + metricSources.push(next) + metricSourceSeriesByName[name] = defaultMetricSourceSeries + } + return ok({}) + }, + 'metricSource.delete': (args) => { + const i = metricSources.findIndex((s) => s.name === args?.name) + if (i < 0) return err('api: metric source not found') + metricSources.splice(i, 1) + delete metricSourceSeriesByName[String(args?.name ?? '')] + return ok({}) + }, + 'metricSource.series': (args) => { + const item = metricSources.find((s) => s.name === args?.name) + if (!item) return err('api: metric source not found') + const items = metricSourceSeriesByName[item.name] ?? defaultMetricSourceSeries + return ok({ project: item.project, name: item.name, items }) + }, + 'metricSource.query': (args) => { + const item = metricSources.find((s) => s.name === args?.name) + if (!item) return err('api: metric source not found') + const requested: string[] = (args?.series ?? []).filter(Boolean) + const names = requested.length + ? requested + : (metricSourceSeriesByName[item.name] ?? defaultMetricSourceSeries).slice(0, 2).map((s) => s.series) + return ok({ + items: names.map((name, i) => metricLine(name, i === 0 ? 12 : 80)[0]) + }) + }, + 'email.list': () => list([{ domain: 'mail.acme.example.com', createdAt: CREATED_AT }]), 'role.list': () => list(roles), diff --git a/src/routes/(auth)/(project)/alert/create/+page.svelte b/src/routes/(auth)/(project)/alert/create/+page.svelte index 682d5054..b0c6045e 100644 --- a/src/routes/(auth)/(project)/alert/create/+page.svelte +++ b/src/routes/(auth)/(project)/alert/create/+page.svelte @@ -6,7 +6,7 @@ import api from '$lib/api' import Select from '$lib/components/Select.svelte' import GuardedButton from '$lib/components/GuardedButton.svelte' - import { ALERT_METRICS, ALERT_OPS, alertMetricLabel } from '$lib/alert' + import { ALERT_CUSTOM_METRICS, ALERT_METRICS, ALERT_OPS, alertMetricLabel } from '$lib/alert' const { data }: { data: PageData } = $props() @@ -15,14 +15,17 @@ const alert = $derived(data.alert) const isEdit = $derived(!!data.alert) const hasChannels = $derived(data.hasChannels) + const metricSources = $derived(data.metricSources) - const metricOptions = ALERT_METRICS.map((m) => ({ value: m.value, label: m.label })) const opOptions = ALERT_OPS const form = $state(untrack(() => ({ name: alert?.name ?? '', + kind: (alert?.target.kind === 'custom' ? 'custom' : 'deployment') as 'deployment' | 'custom', location: alert?.target.location ?? '', deployment: alert?.target.deployment ?? '', + source: alert?.target.source ?? '', + series: alert?.target.series ?? '', metric: alert?.condition.metric ?? 'cpu', op: alert?.condition.op ?? '>=', threshold: alert?.condition.threshold ?? 90, @@ -32,12 +35,19 @@ disabled: alert?.disabled ?? false }))) - const metricUnit = $derived( - form.metric === 'cpu' || form.metric === 'memory' - ? '%' - : form.metric === 'egress' ? 'bytes/min' : 'req/min' + const metricOptions = $derived( + (form.kind === 'custom' ? ALERT_CUSTOM_METRICS : ALERT_METRICS) + .map((m) => ({ value: m.value, label: m.label })) ) + const metricUnit = $derived((() => { + if (form.metric === 'cpu' || form.metric === 'memory') return '%' + if (form.metric === 'egress') return 'bytes/min' + if (form.metric === 'value') return 'value' + if (form.metric === 'rate') return 'per min' + return 'req/min' + })()) + let deployments = $state<{ name: string, paused: boolean }[]>([]) // Stay false until deployment.list settles so an edit of a still-valid // target doesn't flash "Not found" while the picker is empty. @@ -65,6 +75,29 @@ })) ]) + let seriesItems = $state([]) + let seriesLoaded = $state(false) + + const sourceOptions = $derived([ + ...(form.source && !metricSources.some((s) => s.name === form.source) + ? [{ value: form.source, label: form.source, dot: 'negative' as const, badge: 'Not found', badgeTone: 'negative' as const }] + : []), + ...metricSources.map((s) => ({ + value: s.name, + label: s.name + })) + ]) + + const seriesOptions = $derived([ + ...(form.series && seriesLoaded && !seriesItems.some((s) => s.series === form.series) + ? [{ value: form.series, label: form.series, dot: 'negative' as const, badge: 'Not found', badgeTone: 'negative' as const }] + : []), + ...seriesItems.map((s) => ({ + value: s.series, + label: s.series + })) + ]) + async function fetchDeployments () { deployments = [] deploymentsLoaded = false @@ -88,6 +121,51 @@ fetchDeployments() } + async function fetchSeries () { + seriesItems = [] + seriesLoaded = false + if (!form.source) { + seriesLoaded = true + return + } + const resp = await api.invoke('metricSource.series', { + project, + name: form.source + }, fetch) + if (!resp.ok) { + modal.error({ error: resp.error }) + seriesLoaded = true + return + } + seriesItems = resp.result?.items ?? [] + seriesLoaded = true + } + + function onSourceChange () { + form.series = '' + fetchSeries() + } + + function setKind (kind: 'deployment' | 'custom') { + if (form.kind === kind) return + form.kind = kind + if (kind === 'custom') { + if (form.metric !== 'value' && form.metric !== 'rate') { + form.metric = 'value' + form.threshold = 10 + prevMetric = 'value' + } + if (form.source) fetchSeries() + } else { + if (form.metric === 'value' || form.metric === 'rate') { + form.metric = 'cpu' + form.threshold = 90 + prevMetric = 'cpu' + } + if (form.location) fetchDeployments() + } + } + // Threshold units change with the metric (percent vs req/min vs bytes/min). // Keep the number when the unit stays the same (cpu ↔ memory); otherwise // replace it with a unit-appropriate default so switching cpu→egress @@ -95,12 +173,15 @@ function defaultThreshold (metric: string): number { if (metric === 'cpu' || metric === 'memory') return 90 if (metric === 'egress') return 10 * 1024 * 1024 + if (metric === 'value' || metric === 'rate') return 10 return 100 } - function metricUnitKind (metric: string): 'percent' | 'bytes' | 'count' { + function metricUnitKind (metric: string): 'percent' | 'bytes' | 'count' | 'value' | 'rate' { if (metric === 'cpu' || metric === 'memory') return 'percent' if (metric === 'egress') return 'bytes' + if (metric === 'value') return 'value' + if (metric === 'rate') return 'rate' return 'count' } @@ -114,7 +195,8 @@ } onMount(() => { - if (form.location) fetchDeployments() + if (form.kind === 'deployment' && form.location) fetchDeployments() + if (form.kind === 'custom' && form.source) fetchSeries() }) let saving = $state(false) @@ -126,13 +208,13 @@ saving = true try { const fn = isEdit ? 'alert.update' : 'alert.create' + const target = form.kind === 'custom' + ? { kind: 'custom' as const, source: form.source, series: form.series } + : { location: form.location, deployment: form.deployment } const args = { project, name: form.name, - target: { - location: form.location, - deployment: form.deployment - }, + target, condition: { metric: form.metric, op: form.op, @@ -178,7 +260,7 @@

{isEdit ? 'Edit alert rule' : 'Create alert rule'}

-

Notify when a deployment metric crosses a threshold for a sustained period.

+

Notify when a deployment or custom metric crosses a threshold for a sustained period.

@@ -197,29 +279,68 @@
Target
-
-
- - +
+ {#if form.source} +
+ + + placeholder="Select Location" + options={locations.map((it) => ({ value: it.id, label: it.id }))} />
- {/if} -
+ {#if form.location} +
+ + ({ value: s.name, label: s.name }))} /> +
+ {/if} + {#key customSource.name} + + {/key} +{:else}
Metrics
@@ -370,3 +409,4 @@ {/if} +{/if} diff --git a/src/routes/(auth)/(project)/deployment/(detail)/metrics/+page.ts b/src/routes/(auth)/(project)/deployment/(detail)/metrics/+page.ts new file mode 100644 index 00000000..dbb07d29 --- /dev/null +++ b/src/routes/(auth)/(project)/deployment/(detail)/metrics/+page.ts @@ -0,0 +1,14 @@ +import api from '$lib/api' +import type { PageLoad } from './$types' + +export const load: PageLoad = async ({ parent, fetch }) => { + const { project, deployment } = await parent() + if (!deployment) return { metricSources: [] as Api.MetricSourceItem[] } + const res = await api.invoke>('metricSource.list', { project }, fetch) + const items = res.ok ? (res.result?.items ?? []) : [] + return { + metricSources: items.filter((s) => + s.location === deployment.location && s.deployment === deployment.name + ) + } +} diff --git a/src/routes/(auth)/(project)/metrics-sources/+layout.ts b/src/routes/(auth)/(project)/metrics-sources/+layout.ts new file mode 100644 index 00000000..e02c7d28 --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/+layout.ts @@ -0,0 +1,8 @@ +import type { LayoutLoad } from './$types' + +export const load: LayoutLoad = () => { + return { + menu: 'metric-source', + overrideRedirect: '/metrics-sources' + } +} diff --git a/src/routes/(auth)/(project)/metrics-sources/+page.svelte b/src/routes/(auth)/(project)/metrics-sources/+page.svelte new file mode 100644 index 00000000..2ce59740 --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/+page.svelte @@ -0,0 +1,85 @@ + + + it.name}> + {#snippet row(it)} + {@const status = metricSourceStatus(it)} + + + {it.name} + + + {it.location} / {it.deployment} + {it.port} + {it.path} + + {#if status === 'disabled'} + Disabled + {:else} + {status} + {/if} + + + + +
+ +
+ + + + {/snippet} + + + diff --git a/src/routes/(auth)/(project)/metrics-sources/+page.ts b/src/routes/(auth)/(project)/metrics-sources/+page.ts new file mode 100644 index 00000000..ab742e4c --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/+page.ts @@ -0,0 +1,3 @@ +import { listLoad } from '$lib/loaders' + +export const load = listLoad('metricSource.list', 'metricSources') diff --git a/src/routes/(auth)/(project)/metrics-sources/create/+page.svelte b/src/routes/(auth)/(project)/metrics-sources/create/+page.svelte new file mode 100644 index 00000000..b713ed98 --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/create/+page.svelte @@ -0,0 +1,196 @@ + + +
+
+
Metric sources
 +
+ {#if isEdit && source} +
+
{source.name}
 +
+
Edit
+ {:else} +
Create
+ {/if} +
+ +
+ +
+
+

{isEdit ? 'Edit metric source' : 'Create metric source'}

+

Scrape a path on one of your deployments once a minute. The platform reaches the pod in-cluster — there is no URL to set.

+
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
Scrape target
+ +
+
+ + +
+ {/if} +
+ +
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+
+
+ + + +
+ +
+ + {isEdit ? 'Save' : 'Create'} + + +
+
+
diff --git a/src/routes/(auth)/(project)/metrics-sources/create/+page.ts b/src/routes/(auth)/(project)/metrics-sources/create/+page.ts new file mode 100644 index 00000000..bd239144 --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/create/+page.ts @@ -0,0 +1,24 @@ +import { redirect, error } from '@sveltejs/kit' +import api from '$lib/api' +import type { PageLoad } from './$types' + +export const load: PageLoad = async ({ url, parent, fetch }) => { + const { project } = await parent() + const name = url.searchParams.get('name') + + let source: Api.MetricSourceItem | null = null + if (name) { + const res = await api.invoke('metricSource.get', { project, name }, fetch) + if (!res.ok) { + if (res.error?.notFound) redirect(302, `/metrics-sources?project=${project}`) + error(500, res.error?.message) + } + if (!res.result) redirect(302, `/metrics-sources?project=${project}`) + source = res.result + } + + return { + menu: 'metric-source', + source + } +} diff --git a/src/routes/(auth)/(project)/metrics-sources/detail/+page.svelte b/src/routes/(auth)/(project)/metrics-sources/detail/+page.svelte new file mode 100644 index 00000000..11e55835 --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/detail/+page.svelte @@ -0,0 +1,206 @@ + + +
+
+
Metric sources
 +
+
+
{source.name}
+
+
+ +
+ +
+
+

{source.name}

+

+ {#if status === 'disabled'} + Disabled + {:else} + {status} + {/if} + {source.location} / {source.deployment}:{source.port}{source.path} +

+
+
+ + + Create alert + + + + Edit + +
+
+ +
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
Chart
+

Scraped series for this source. Leave the series filter empty to chart the most recently seen.

+
+ + + +
+ +
+
Discovered series
+

Gauges, counters, and untyped metrics kept from the last scrape. Histograms and summaries are dropped.

+
+ +
+ + + + + + + + + + {#each series as s (s.series)} + + + + + + {:else} + + {/each} + +
SeriesTypeLast seen
{s.series}{s.type} + {format.fromNow(s.lastSeenAt) || '—'} +
No series discovered yet.
+
+ + + Delete + +
+
+ + diff --git a/src/routes/(auth)/(project)/metrics-sources/detail/+page.ts b/src/routes/(auth)/(project)/metrics-sources/detail/+page.ts new file mode 100644 index 00000000..e67889fe --- /dev/null +++ b/src/routes/(auth)/(project)/metrics-sources/detail/+page.ts @@ -0,0 +1,24 @@ +import { redirect, error } from '@sveltejs/kit' +import api from '$lib/api' +import type { PageLoad } from './$types' + +export const load: PageLoad = async ({ url, parent, fetch }) => { + const { project } = await parent() + const name = url.searchParams.get('name') + if (!name) redirect(302, `/metrics-sources?project=${project}`) + + const res = await api.invoke('metricSource.get', { project, name }, fetch) + if (!res.ok) { + if (res.error?.notFound) redirect(302, `/metrics-sources?project=${project}`) + error(500, res.error?.message) + } + if (!res.result) redirect(302, `/metrics-sources?project=${project}`) + + const series = await api.invoke('metricSource.series', { project, name }, fetch) + + return { + menu: 'metric-source', + source: res.result, + series: series.result?.items ?? [] + } +} diff --git a/src/types/api.d.ts b/src/types/api.d.ts index fa0ee72f..d65677f0 100644 --- a/src/types/api.d.ts +++ b/src/types/api.d.ts @@ -1140,16 +1140,22 @@ declare namespace Api { // Notification carries its delivery config). Evaluated by an apiserver cron // tick against the existing per-minute deployment_usages table; delivery // reuses the notification-channels feature entirely. + // Empty Kind is treated as deployment. kind=custom targets a metricSource + // series; Location and Deployment must be empty (omit them or send ""). export type AlertTarget = { - location: string - deployment: string + kind?: '' | 'deployment' | 'custom' + location?: string + deployment?: string + source?: string + series?: string } // Threshold's unit depends on Metric: percent 0-100 for cpu/memory, req/min - // for requests, bytes/min for egress. Op defaults to ">=" server-side when - // left empty. + // for requests, bytes/min for egress, the gauge value for kind=custom + // Metric=value, or per-minute increase for kind=custom Metric=rate. Op + // defaults to ">=" server-side when left empty. export type AlertCondition = { - metric: 'cpu' | 'memory' | 'requests' | 'egress' + metric: 'cpu' | 'memory' | 'requests' | 'egress' | 'value' | 'rate' op: '>=' | '<=' threshold: number forMinutes: number @@ -1191,6 +1197,50 @@ declare namespace Api { items: AlertEvent[] } + // MetricSource — Prometheus scrape targets on the project's own deployments + // (port+path, never a free-form URL). Project-scoped; location lives in the + // config. Set is a full upsert. Query reuses the DeploymentMetricsLine / + // UsageMetricsLine shape so Chart.svelte consumes it unmodified. + export type MetricSourceItem = { + project: string + name: string + location: string + deployment: string + port: number + path: string + disabled: boolean + truncated: boolean + lastScrapedAt: string | null + lastError: string + createdAt: string + createdBy: string + updatedAt: string + updatedBy: string + } + + export type MetricSourceListResult = { + project: string + items: MetricSourceItem[] + } + + export type MetricSourceSeriesType = 'gauge' | 'counter' | 'untyped' + + export type MetricSourceSeriesItem = { + series: string + type: MetricSourceSeriesType + lastSeenAt: string + } + + export type MetricSourceSeriesResult = { + project: string + name: string + items: MetricSourceSeriesItem[] + } + + export type MetricSourceQueryResult = { + items: UsageMetricsLine[] + } + // cache.metrics reuses WafMetricsTimeRange. export type CacheMetricsSeries = { overrideId: string diff --git a/tests/alert.spec.js b/tests/alert.spec.js index 66a4010a..d41d004e 100644 --- a/tests/alert.spec.js +++ b/tests/alert.spec.js @@ -1,5 +1,11 @@ import { test, expect, setMocks, getRequestLog, pickSelect } from './helpers.js' -import { sampleAlertRule, sampleAlertEvent, sampleDeployment } from './fixtures/mocks.js' +import { + sampleAlertRule, + sampleAlertEvent, + sampleDeployment, + sampleMetricSource, + sampleMetricSourceSeries +} from './fixtures/mocks.js' test.describe('alerts', () => { test('lists alert rules with status badges', async ({ page }) => { @@ -179,6 +185,52 @@ test.describe('alert — create', () => { await expect(main.getByText('No notification channels exist yet')).toHaveCount(0) }) + test('submits alert.create with a custom source and series and no deployment', async ({ page }) => { + await setMocks({ + 'metricSource.list': { ok: true, result: { items: [sampleMetricSource] } }, + 'metricSource.series': { ok: true, result: { items: sampleMetricSourceSeries } }, + 'alert.create': { ok: true, result: {} }, + 'alert.get': { + ok: true, + result: { + ...sampleAlertRule, + name: 'web-queue-depth', + target: { kind: 'custom', source: 'web', series: 'queue_depth{queue="email"}' }, + condition: { metric: 'value', op: '>=', threshold: 10, forMinutes: 5 } + } + }, + 'alert.events': { ok: true, result: { items: [] } } + }) + + await page.goto('/alert/create?project=test-project') + + const main = page.locator('.content-wrapper') + await main.locator('#input-name').fill('web-queue-depth') + await main.getByRole('tab', { name: 'Custom' }).click() + await pickSelect(page, 'input-source', 'web') + await pickSelect(page, 'input-series', 'queue_depth{queue="email"}') + await expect(main.locator('#input-metric')).toContainText('Value (gauge)') + await main.locator('#input-threshold').fill('10') + await main.locator('#input-for-minutes').fill('5') + await main.getByRole('button', { name: 'Create', exact: true }).click() + + await expect.poll(async () => { + const log = await getRequestLog() + return log.some((r) => r.path === '/alert.create') + }).toBeTruthy() + + const req = (await getRequestLog()).find((r) => r.path === '/alert.create') + if (!req) throw new Error('expected an alert.create request') + const body = JSON.parse(req.body) + expect(body.name).toBe('web-queue-depth') + expect(body.target).toEqual({ kind: 'custom', source: 'web', series: 'queue_depth{queue="email"}' }) + expect(body.target).not.toHaveProperty('location') + expect(body.target).not.toHaveProperty('deployment') + expect(body.condition).toEqual({ metric: 'value', op: '>=', threshold: 10, forMinutes: 5 }) + + await expect(page).toHaveURL(/\/alert\/detail\?project=test-project&name=web-queue-depth/) + }) + test('resets the threshold when the metric unit changes', async ({ page }) => { await page.goto('/alert/create?project=test-project') diff --git a/tests/fixtures/mocks.js b/tests/fixtures/mocks.js index d5707edb..41fdc937 100644 --- a/tests/fixtures/mocks.js +++ b/tests/fixtures/mocks.js @@ -176,6 +176,30 @@ export function defaultMocks () { '/alert.events': { ok: true, result: { items: [] } + }, + '/metricSource.list': { + ok: true, + result: { items: [] } + }, + '/metricSource.get': { + ok: false, + error: { message: 'api: metric source not found' } + }, + '/metricSource.set': { + ok: true, + result: {} + }, + '/metricSource.delete': { + ok: true, + result: {} + }, + '/metricSource.series': { + ok: true, + result: { items: [] } + }, + '/metricSource.query': { + ok: true, + result: { items: [] } } } } @@ -433,6 +457,64 @@ export const sampleAlertEvent = { value: 93.2 } +export const sampleMetricSource = { + project: 'test-project', + name: 'web', + location: 'gke', + deployment: 'web', + port: 9090, + path: '/metrics', + disabled: false, + truncated: false, + lastScrapedAt: now, + lastError: '', + createdAt: now, + createdBy: '[email protected]', + updatedAt: now, + updatedBy: '[email protected]' +} + +export const sampleTruncatedMetricSource = { + ...sampleMetricSource, + name: 'api-metrics', + deployment: 'api', + truncated: true +} + +export const sampleErrorMetricSource = { + ...sampleMetricSource, + name: 'worker-metrics', + deployment: 'worker', + lastError: 'scrape failed: connection refused' +} + +export const sampleMetricSourceSeries = [ + { series: 'queue_depth{queue="email"}', type: 'gauge', lastSeenAt: now }, + { series: 'http_requests_total{code="200"}', type: 'counter', lastSeenAt: now }, + { series: 'up', type: 'untyped', lastSeenAt: now } +] + +export const sampleMetricSourceQuery = { + items: [ + { + name: 'queue_depth{queue="email"}', + points: [ + [1704067200, 4], + [1704067260, 8], + [1704067320, 12] + ] + }, + { + name: 'http_requests_total{code="200"}', + points: [ + [1704067200, 40], + [1704067260, 55], + [1704067320, 80] + ] + } + ] +} + export const sampleBillingAccount = { id: 'ba-1', name: 'Personal', diff --git a/tests/metrics-sources.spec.js b/tests/metrics-sources.spec.js new file mode 100644 index 00000000..788169f0 --- /dev/null +++ b/tests/metrics-sources.spec.js @@ -0,0 +1,153 @@ +import { test, expect, setMocks, getRequestLog, pickSelect } from './helpers.js' +import { + defaultLocation, + sampleDeployment, + sampleMetricSource, + sampleTruncatedMetricSource, + sampleErrorMetricSource, + sampleMetricSourceSeries, + sampleMetricSourceQuery +} from './fixtures/mocks.js' + +const threeSources = [ + sampleMetricSource, + sampleTruncatedMetricSource, + sampleErrorMetricSource +] + +test.describe('metric sources', () => { + test('lists sources with ok / truncated / error status', async ({ page }) => { + await setMocks({ + 'metricSource.list': { ok: true, result: { items: threeSources } } + }) + + await page.goto('/metrics-sources?project=test-project') + + const main = page.locator('.content-wrapper') + await expect(main.getByRole('heading', { name: 'Metric sources' })).toBeVisible() + await expect(main.getByRole('link', { name: 'web' })).toBeVisible() + await expect(main.getByRole('link', { name: 'api-metrics' })).toBeVisible() + await expect(main.getByRole('link', { name: 'worker-metrics' })).toBeVisible() + + await expect(main.getByRole('cell', { name: 'ok', exact: true })).toBeVisible() + await expect(main.getByRole('cell', { name: 'truncated', exact: true })).toBeVisible() + await expect(main.getByRole('cell', { name: 'error', exact: true })).toBeVisible() + + await expect(main.getByText('gke / web', { exact: true })).toBeVisible() + await expect(main.getByText('/metrics', { exact: true }).first()).toBeVisible() + }) + + test('empty state when no sources', async ({ page }) => { + await page.goto('/metrics-sources?project=test-project') + const main = page.locator('.content-wrapper') + await expect(main.getByText('Nothing here yet')).toBeVisible() + }) + + test('gates the create button when the set permission is missing', async ({ page }) => { + await setMocks({ + 'me.permissions': { ok: true, result: { permissions: ['metricSource.list'], admin: false } } + }) + await page.goto('/metrics-sources?project=test-project') + const main = page.locator('.content-wrapper') + await expect(main.getByRole('button', { name: 'Create source' })).toBeDisabled() + await expect(main.getByRole('link', { name: 'Create source' })).toHaveCount(0) + }) +}) + +test.describe('metric source — create', () => { + test('submits metricSource.set with deployment, port, and path and has no URL field', async ({ page }) => { + await setMocks({ + 'deployment.list': { ok: true, result: { items: [sampleDeployment] } }, + 'metricSource.set': { ok: true, result: {} }, + 'metricSource.get': { ok: true, result: sampleMetricSource }, + 'metricSource.series': { ok: true, result: { items: sampleMetricSourceSeries } }, + 'metricSource.query': { ok: true, result: sampleMetricSourceQuery } + }) + + await page.goto('/metrics-sources/create?project=test-project') + + const main = page.locator('.content-wrapper') + await expect(main.locator('#input-url')).toHaveCount(0) + await expect(main.getByLabel('URL', { exact: true })).toHaveCount(0) + + await main.locator('#input-name').fill('web') + await pickSelect(page, 'input-location', 'gke') + await pickSelect(page, 'input-deployment', 'web') + await expect(main.locator('#input-port')).toHaveValue('9090') + await expect(main.locator('#input-path')).toHaveValue('/metrics') + + await main.getByRole('button', { name: 'Create', exact: true }).click() + + await expect.poll(async () => { + const log = await getRequestLog() + return log.some((r) => r.path === '/metricSource.set') + }).toBeTruthy() + + const req = (await getRequestLog()).find((r) => r.path === '/metricSource.set') + if (!req) throw new Error('expected a metricSource.set request') + const body = JSON.parse(req.body) + expect(body.name).toBe('web') + expect(body.location).toBe('gke') + expect(body.deployment).toBe('web') + expect(body.port).toBe(9090) + expect(body.path).toBe('/metrics') + expect(body).not.toHaveProperty('url') + + await expect(page).toHaveURL(/\/metrics-sources\/detail\?project=test-project&name=web/) + }) +}) + +test.describe('metric source — chart banners', () => { + test('shows truncated, error, and empty banners on the source detail chart', async ({ page }) => { + await setMocks({ + 'metricSource.get': { + ok: true, + result: { + ...sampleTruncatedMetricSource, + name: 'web', + deployment: 'web', + lastError: 'scrape failed: connection refused' + } + }, + 'metricSource.series': { ok: true, result: { items: sampleMetricSourceSeries } }, + 'metricSource.query': { ok: true, result: { items: [] } } + }) + + await page.goto('/metrics-sources/detail?project=test-project&name=web') + + const main = page.locator('.content-wrapper') + await expect(main.getByText('series cap hit — extra series were dropped')).toBeVisible() + await expect(main.getByText('scrape failed: connection refused')).toBeVisible() + await expect(main.getByText('No custom metric samples in this window')).toBeVisible() + }) + + test('shows the Custom tab on a matching deployment with truncated and error banners', async ({ page }) => { + await setMocks({ + 'deployment.get': { ok: true, result: sampleDeployment }, + 'location.get': { ok: true, result: defaultLocation }, + 'deployment.metrics': { ok: true, result: {} }, + 'metricSource.list': { + ok: true, + result: { + items: [{ + ...sampleTruncatedMetricSource, + name: 'web', + location: 'gke', + deployment: 'web', + lastError: 'scrape failed: connection refused' + }] + } + }, + 'metricSource.series': { ok: true, result: { items: sampleMetricSourceSeries } }, + 'metricSource.query': { ok: true, result: { items: [] } } + }) + + await page.goto('/deployment/metrics?project=test-project&location=gke&name=web') + + const main = page.locator('.content-wrapper') + await main.getByRole('tab', { name: 'Custom' }).click() + await expect(main.getByText('series cap hit — extra series were dropped')).toBeVisible() + await expect(main.getByText('scrape failed: connection refused')).toBeVisible() + await expect(main.getByText('No custom metric samples in this window')).toBeVisible() + }) +})