diff --git a/.server-changes/db-pool-metrics-per-client.md b/.server-changes/db-pool-metrics-per-client.md new file mode 100644 index 0000000000..f657849a87 --- /dev/null +++ b/.server-changes/db-pool-metrics-per-client.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Database connection-pool and query metrics are now reported for every configured database connection rather than only the primary, and keep working regardless of which database driver a connection uses. diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index 249b851456..a06321e167 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -24,6 +24,7 @@ import { logTransactionInfrastructureError, } from "./utils/prismaErrors"; import { singleton } from "./utils/singleton"; +import { registerDatabaseMetricsSource } from "./utils/databaseMetrics.server"; import { isSplitEnabled, assertSplitRealtimeInterlock, @@ -247,16 +248,16 @@ export function selectRunOpsTopology( if (config.legacySharesControlPlane) { legacyRunOps = controlPlane; } else { - const legacyWriter = builders.buildLegacyWriter(config.legacyUrl, "run-ops-legacy-writer"); + const legacyWriter = builders.buildLegacyWriter(config.legacyUrl, "legacy-run-ops-writer"); const legacyReplica: PrismaReplicaClient = config.legacyReplicaUrl - ? builders.buildLegacyReplica(config.legacyReplicaUrl, "run-ops-legacy-reader") + ? builders.buildLegacyReplica(config.legacyReplicaUrl, "legacy-run-ops-replica") : legacyWriter; legacyRunOps = { writer: legacyWriter, replica: legacyReplica }; } - const newWriter = builders.buildNewWriter(config.newUrl, "run-ops-new-writer"); + const newWriter = builders.buildNewWriter(config.newUrl, "run-ops-writer"); const newReplica: RunOpsPrismaClient = config.newReplicaUrl - ? builders.buildNewReplica(config.newReplicaUrl, "run-ops-new-reader") + ? builders.buildNewReplica(config.newReplicaUrl, "run-ops-replica") : newWriter; return { @@ -430,19 +431,25 @@ function getClient() { return buildWriterClient({ url, - clientType: "writer", + clientType: "control-plane-writer", poolTimeout: env.DATABASE_WRITER_POOL_TIMEOUT, connectTimeout: env.DATABASE_WRITER_CONNECTION_TIMEOUT, useDriverAdapter: env.CONTROL_PLANE_DATABASE_WRITER_DRIVER_ADAPTER === "1", }); } +type DriverAdapterPool = { + adapter: PrismaPg; + pool: Pool; + poolCounters: { opened: () => number; closed: () => number }; +}; + function buildDriverAdapterPool( connectionString: string, clientType: string, poolTimeoutSeconds: number, connectionLimit: number -): PrismaPg { +): DriverAdapterPool { const pool = new Pool({ connectionString, max: connectionLimit, @@ -457,6 +464,15 @@ function buildDriverAdapterPool( }); }); + let opened = 0; + let closed = 0; + pool.on("connect", () => { + opened += 1; + }); + pool.on("remove", () => { + closed += 1; + }); + let schema: string | undefined; try { schema = new URL(connectionString).searchParams.get("schema") ?? undefined; @@ -464,7 +480,11 @@ function buildDriverAdapterPool( schema = undefined; } - return new PrismaPg(pool, { schema, disposeExternalPool: true }); + return { + adapter: new PrismaPg(pool, { schema, disposeExternalPool: true }), + pool, + poolCounters: { opened: () => opened, closed: () => closed }, + }; } // Generalized writer builder shared by the control-plane client and the run-ops @@ -548,21 +568,34 @@ export function buildWriterClient({ : []) satisfies Prisma.LogDefinition[]), ] satisfies Prisma.LogDefinition[]; - const client = useDriverAdapter - ? new PrismaClient({ - adapter: buildDriverAdapterPool( - url, - clientType, - poolTimeout ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ), - log: logConfig, - }) + const driverPool = useDriverAdapter + ? buildDriverAdapterPool( + url, + clientType, + poolTimeout ?? env.DATABASE_POOL_TIMEOUT, + env.DATABASE_CONNECTION_LIMIT + ) + : undefined; + + const client = driverPool + ? new PrismaClient({ adapter: driverPool.adapter, log: logConfig }) : new PrismaClient({ datasources: { db: { url: databaseUrl.href } }, log: logConfig, }); + registerDatabaseMetricsSource( + driverPool + ? { + clientType, + usesDriverAdapter: true, + client, + pool: driverPool.pool, + poolCounters: driverPool.poolCounters, + } + : { clientType, usesDriverAdapter: false, client } + ); + // Only use structured logging if we're not already logging to stdout if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { client.$on("info", (log) => { @@ -631,7 +664,7 @@ function getReplicaClient() { return buildReplicaClient({ url, - clientType: "reader", + clientType: "control-plane-replica", poolTimeout: env.DATABASE_READ_REPLICA_POOL_TIMEOUT, connectTimeout: env.DATABASE_READ_REPLICA_CONNECTION_TIMEOUT, useDriverAdapter: env.CONTROL_PLANE_DATABASE_REPLICA_DRIVER_ADAPTER === "1", @@ -719,21 +752,34 @@ export function buildReplicaClient({ : []) satisfies Prisma.LogDefinition[]), ] satisfies Prisma.LogDefinition[]; - const replicaClient = useDriverAdapter - ? new PrismaClient({ - adapter: buildDriverAdapterPool( - url, - clientType, - poolTimeout ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ), - log: logConfig, - }) + const driverPool = useDriverAdapter + ? buildDriverAdapterPool( + url, + clientType, + poolTimeout ?? env.DATABASE_POOL_TIMEOUT, + env.DATABASE_CONNECTION_LIMIT + ) + : undefined; + + const replicaClient = driverPool + ? new PrismaClient({ adapter: driverPool.adapter, log: logConfig }) : new PrismaClient({ datasources: { db: { url: replicaUrl.href } }, log: logConfig, }); + registerDatabaseMetricsSource( + driverPool + ? { + clientType, + usesDriverAdapter: true, + client: replicaClient, + pool: driverPool.pool, + poolCounters: driverPool.poolCounters, + } + : { clientType, usesDriverAdapter: false, client: replicaClient } + ); + // Only use structured logging if we're not already logging to stdout if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { replicaClient.$on("info", (log) => { @@ -813,14 +859,18 @@ function buildRunOpsWriterClient({ }` ); - const client = useDriverAdapter + const driverPool = useDriverAdapter + ? buildDriverAdapterPool( + url, + clientType, + env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + env.DATABASE_CONNECTION_LIMIT + ) + : undefined; + + const client = driverPool ? new RunOpsPrismaClient({ - adapter: buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ), + adapter: driverPool.adapter, log: [ { emit: "event", level: "error" }, { emit: "event", level: "info" }, @@ -844,6 +894,18 @@ function buildRunOpsWriterClient({ ], }); + registerDatabaseMetricsSource( + driverPool + ? { + clientType, + usesDriverAdapter: true, + client, + pool: driverPool.pool, + poolCounters: driverPool.poolCounters, + } + : { clientType, usesDriverAdapter: false, client } + ); + if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); @@ -894,14 +956,18 @@ function buildRunOpsReplicaClient({ }` ); - const client = useDriverAdapter + const driverPool = useDriverAdapter + ? buildDriverAdapterPool( + url, + clientType, + env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT + ) + : undefined; + + const client = driverPool ? new RunOpsPrismaClient({ - adapter: buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ), + adapter: driverPool.adapter, log: [ { emit: "event", level: "error" }, { emit: "event", level: "info" }, @@ -925,6 +991,18 @@ function buildRunOpsReplicaClient({ ], }); + registerDatabaseMetricsSource( + driverPool + ? { + clientType, + usesDriverAdapter: true, + client, + pool: driverPool.pool, + poolCounters: driverPool.poolCounters, + } + : { clientType, usesDriverAdapter: false, client } + ); + if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); diff --git a/apps/webapp/app/routes/metrics.ts b/apps/webapp/app/routes/metrics.ts index 038d9e75fc..042b18d07b 100644 --- a/apps/webapp/app/routes/metrics.ts +++ b/apps/webapp/app/routes/metrics.ts @@ -1,5 +1,4 @@ import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { prisma } from "~/db.server"; import { metricsRegister } from "~/metrics.server"; export async function loader({ request }: LoaderFunctionArgs) { @@ -13,17 +12,7 @@ export async function loader({ request }: LoaderFunctionArgs) { } } - // We need to remove empty lines from the prisma metrics, grafana doesn't like them - let prismaMetrics = ""; - try { - prismaMetrics = (await prisma.$metrics.prometheus()).replace(/^\s*[\r\n]/gm, ""); - } catch { - prismaMetrics = ""; - } - const coreMetrics = await metricsRegister.metrics(); - - // Order matters, core metrics end with `# EOF`, prisma metrics don't - const metrics = prismaMetrics + coreMetrics; + const metrics = await metricsRegister.metrics(); return new Response(metrics, { headers: { diff --git a/apps/webapp/app/utils/databaseMetrics.server.test.ts b/apps/webapp/app/utils/databaseMetrics.server.test.ts new file mode 100644 index 0000000000..9289d4b7c8 --- /dev/null +++ b/apps/webapp/app/utils/databaseMetrics.server.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; +import { + normalizeDatabaseMetrics, + type DatabaseMetricsSource, + type MetricHistogramValue, +} from "./databaseMetrics.server"; + +const durationHistogram: MetricHistogramValue = { + buckets: [ + [1, 10], + [10, 5], + ], + sum: 1234, + count: 15, +}; + +function quaintJson() { + return { + counters: [ + { key: "prisma_client_queries_total", value: 100 }, + { key: "prisma_datasource_queries_total", value: 250 }, + { key: "prisma_pool_connections_opened_total", value: 12 }, + { key: "prisma_pool_connections_closed_total", value: 3 }, + ], + gauges: [ + { key: "prisma_client_queries_active", value: 4 }, + { key: "prisma_client_queries_wait", value: 2 }, + { key: "prisma_pool_connections_open", value: 9 }, + { key: "prisma_pool_connections_busy", value: 4 }, + { key: "prisma_pool_connections_idle", value: 5 }, + ], + histograms: [{ key: "prisma_client_queries_duration_histogram_ms", value: durationHistogram }], + }; +} + +const stubClient = { $metrics: { json: async () => quaintJson() } }; + +describe("normalizeDatabaseMetrics", () => { + it("reads pool figures from $metrics for a quaint (Rust) client", () => { + const source: DatabaseMetricsSource = { + clientType: "writer", + usesDriverAdapter: false, + client: stubClient, + }; + + const result = normalizeDatabaseMetrics(source, quaintJson()); + + expect(result.driver).toBe("quaint"); + expect(result.clientType).toBe("writer"); + expect(result.pool).toEqual({ + open: 9, + busy: 4, + idle: 5, + waiting: 0, + openedTotal: 12, + closedTotal: 3, + }); + expect(result.counters).toEqual({ queriesTotal: 100, datasourceQueriesTotal: 250 }); + expect(result.gauges).toEqual({ queriesActive: 4, queriesWait: 2 }); + expect(result.histograms.queriesDuration).toEqual(durationHistogram); + }); + + it("reads pool figures from pg.Pool for a driver-adapter client and keeps engine query metrics", () => { + const source: DatabaseMetricsSource = { + clientType: "control-plane-writer", + usesDriverAdapter: true, + client: stubClient, + pool: { totalCount: 8, idleCount: 3, waitingCount: 6 }, + poolCounters: { opened: () => 20, closed: () => 12 }, + }; + + const result = normalizeDatabaseMetrics(source, quaintJson()); + + expect(result.driver).toBe("pg-adapter"); + expect(result.pool).toEqual({ + open: 8, + busy: 5, + idle: 3, + waiting: 6, + openedTotal: 20, + closedTotal: 12, + }); + expect(result.counters).toEqual({ queriesTotal: 100, datasourceQueriesTotal: 250 }); + }); + + it("does not report negative busy when idle exceeds total for an adapter pool", () => { + const source: DatabaseMetricsSource = { + clientType: "reader", + usesDriverAdapter: true, + client: stubClient, + pool: { totalCount: 2, idleCount: 5, waitingCount: 0 }, + poolCounters: { opened: () => 0, closed: () => 0 }, + }; + + const result = normalizeDatabaseMetrics(source, quaintJson()); + + expect(result.pool.busy).toBe(0); + }); + + it("falls back to zeroed query/pool metrics when $metrics is unavailable for a quaint client", () => { + const source: DatabaseMetricsSource = { + clientType: "writer", + usesDriverAdapter: false, + client: stubClient, + }; + + const result = normalizeDatabaseMetrics(source, undefined); + + expect(result.pool).toEqual({ + open: 0, + busy: 0, + idle: 0, + waiting: 0, + openedTotal: 0, + closedTotal: 0, + }); + expect(result.counters).toEqual({ queriesTotal: 0, datasourceQueriesTotal: 0 }); + expect(result.histograms.queriesDuration).toBeUndefined(); + }); + + it("keeps adapter pool figures even when $metrics is unavailable", () => { + const source: DatabaseMetricsSource = { + clientType: "control-plane-writer", + usesDriverAdapter: true, + client: stubClient, + pool: { totalCount: 7, idleCount: 2, waitingCount: 1 }, + poolCounters: { opened: () => 9, closed: () => 2 }, + }; + + const result = normalizeDatabaseMetrics(source, undefined); + + expect(result.pool).toEqual({ + open: 7, + busy: 5, + idle: 2, + waiting: 1, + openedTotal: 9, + closedTotal: 2, + }); + expect(result.counters).toEqual({ queriesTotal: 0, datasourceQueriesTotal: 0 }); + }); +}); diff --git a/apps/webapp/app/utils/databaseMetrics.server.ts b/apps/webapp/app/utils/databaseMetrics.server.ts new file mode 100644 index 0000000000..a27cc049b8 --- /dev/null +++ b/apps/webapp/app/utils/databaseMetrics.server.ts @@ -0,0 +1,143 @@ +export type MetricHistogramValue = { + buckets: [number, number][]; + sum: number; + count: number; +}; + +type PrismaMetricsJson = { + counters: Array<{ key: string; value: number }>; + gauges: Array<{ key: string; value: number }>; + histograms: Array<{ key: string; value: MetricHistogramValue }>; +}; + +type MetricsCapableClient = { + $metrics: { json: () => Promise }; +}; + +type PoolLike = { + totalCount: number; + idleCount: number; + waitingCount: number; +}; + +export type DatabaseMetricsSource = { + clientType: string; + usesDriverAdapter: boolean; + client: MetricsCapableClient; + pool?: PoolLike; + poolCounters?: { opened: () => number; closed: () => number }; +}; + +export type NormalizedPoolMetrics = { + open: number; + busy: number; + idle: number; + waiting: number; + openedTotal: number; + closedTotal: number; +}; + +export type NormalizedDatabaseMetrics = { + clientType: string; + driver: "pg-adapter" | "quaint"; + pool: NormalizedPoolMetrics; + counters: { queriesTotal: number; datasourceQueriesTotal: number }; + gauges: { queriesActive: number; queriesWait: number }; + histograms: { + queriesWait?: MetricHistogramValue; + queriesDuration?: MetricHistogramValue; + datasourceQueriesDuration?: MetricHistogramValue; + }; +}; + +const sources: DatabaseMetricsSource[] = []; + +export function registerDatabaseMetricsSource(source: DatabaseMetricsSource): void { + sources.push(source); +} + +export function listDatabaseMetricsSources(): ReadonlyArray { + return sources; +} + +export function resetDatabaseMetricsSources(): void { + sources.length = 0; +} + +function indexByKey(entries: Array<{ key: string; value: number }>): Record { + const out: Record = {}; + for (const entry of entries) { + out[entry.key] = entry.value; + } + return out; +} + +export function normalizeDatabaseMetrics( + source: DatabaseMetricsSource, + json: PrismaMetricsJson | undefined +): NormalizedDatabaseMetrics { + const counters = json ? indexByKey(json.counters) : {}; + const gauges = json ? indexByKey(json.gauges) : {}; + const histograms: Record = {}; + if (json) { + for (const histogram of json.histograms) { + histograms[histogram.key] = histogram.value; + } + } + + let pool: NormalizedPoolMetrics; + if (source.usesDriverAdapter && source.pool) { + const total = source.pool.totalCount; + const idle = source.pool.idleCount; + pool = { + open: total, + idle, + busy: Math.max(0, total - idle), + waiting: source.pool.waitingCount, + openedTotal: source.poolCounters?.opened() ?? 0, + closedTotal: source.poolCounters?.closed() ?? 0, + }; + } else { + pool = { + open: gauges["prisma_pool_connections_open"] ?? 0, + busy: gauges["prisma_pool_connections_busy"] ?? 0, + idle: gauges["prisma_pool_connections_idle"] ?? 0, + waiting: 0, + openedTotal: counters["prisma_pool_connections_opened_total"] ?? 0, + closedTotal: counters["prisma_pool_connections_closed_total"] ?? 0, + }; + } + + return { + clientType: source.clientType, + driver: source.usesDriverAdapter ? "pg-adapter" : "quaint", + pool, + counters: { + queriesTotal: counters["prisma_client_queries_total"] ?? 0, + datasourceQueriesTotal: counters["prisma_datasource_queries_total"] ?? 0, + }, + gauges: { + queriesActive: gauges["prisma_client_queries_active"] ?? 0, + queriesWait: gauges["prisma_client_queries_wait"] ?? 0, + }, + histograms: { + queriesWait: histograms["prisma_client_queries_wait_histogram_ms"], + queriesDuration: histograms["prisma_client_queries_duration_histogram_ms"], + datasourceQueriesDuration: histograms["prisma_datasource_queries_duration_histogram_ms"], + }, + }; +} + +export async function collectDatabaseClientMetrics(): Promise { + return Promise.all( + sources.map(async (source) => { + let json: PrismaMetricsJson | undefined; + try { + json = await source.client.$metrics.json(); + } catch { + json = undefined; + } + return normalizeDatabaseMetrics(source, json); + }) + ); +} diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index d91c9ee2ff..8685612fea 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -66,9 +66,8 @@ import { LoggerSpanExporter } from "./telemetry/loggerExporter.server"; import { CompactMetricExporter } from "./telemetry/compactMetricExporter.server"; import { logger } from "~/services/logger.server"; import { flattenAttributes } from "@trigger.dev/core/v3"; -import { prisma } from "~/db.server"; import { metricsRegister } from "~/metrics.server"; -import type { Prisma } from "@trigger.dev/database"; +import { collectDatabaseClientMetrics } from "~/utils/databaseMetrics.server"; import { performance } from "node:perf_hooks"; export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; @@ -455,6 +454,10 @@ function configurePrismaMetrics({ meter }: { meter: Meter }) { description: "Idle (free) connections in the pool", unit: "connections", }); + const waitingGauge = meter.createObservableGauge("db.pool.connections.waiting", { + description: "Requests waiting to acquire a pool connection", + unit: "requests", + }); // Histogram statistics as gauges const queriesWaitTimeCount = meter.createObservableGauge("db.client.queries.wait_time.count", { @@ -505,105 +508,75 @@ function configurePrismaMetrics({ meter }: { meter: Meter }) { } ); - // Single helper so we hit Prisma only once per scrape --------------------- - async function readPrismaMetrics() { - const metrics = await prisma.$metrics.json(); - - // Extract counter values - const counters: Record = {}; - for (const counter of metrics.counters) { - counters[counter.key] = counter.value; - } - - // Extract gauge values - const gauges: Record = {}; - for (const gauge of metrics.gauges) { - gauges[gauge.key] = gauge.value; - } - - // Extract histogram values - const histograms: Record = {}; - for (const histogram of metrics.histograms) { - histograms[histogram.key] = histogram.value; - } - - return { - counters: { - queriesTotal: counters["prisma_client_queries_total"] ?? 0, - datasourceQueriesTotal: counters["prisma_datasource_queries_total"] ?? 0, - connectionsOpenedTotal: counters["prisma_pool_connections_opened_total"] ?? 0, - connectionsClosedTotal: counters["prisma_pool_connections_closed_total"] ?? 0, - }, - gauges: { - queriesActive: gauges["prisma_client_queries_active"] ?? 0, - queriesWait: gauges["prisma_client_queries_wait"] ?? 0, - connectionsOpen: gauges["prisma_pool_connections_open"] ?? 0, - connectionsBusy: gauges["prisma_pool_connections_busy"] ?? 0, - connectionsIdle: gauges["prisma_pool_connections_idle"] ?? 0, - }, - histograms: { - queriesWait: histograms["prisma_client_queries_wait_histogram_ms"], - queriesDuration: histograms["prisma_client_queries_duration_histogram_ms"], - datasourceQueriesDuration: histograms["prisma_datasource_queries_duration_histogram_ms"], - }, - }; - } - meter.addBatchObservableCallback( async (res) => { - let prismaMetrics: Awaited>; + let clients: Awaited>; try { - prismaMetrics = await readPrismaMetrics(); + clients = await collectDatabaseClientMetrics(); } catch { return; } - const { counters, gauges, histograms } = prismaMetrics; - - // Observe counters - res.observe(queriesTotal, counters.queriesTotal); - res.observe(datasourceQueriesTotal, counters.datasourceQueriesTotal); - res.observe(connectionsOpenedTotal, counters.connectionsOpenedTotal); - res.observe(connectionsClosedTotal, counters.connectionsClosedTotal); - - // Observe gauges - res.observe(queriesActive, gauges.queriesActive); - res.observe(queriesWait, gauges.queriesWait); - res.observe(totalGauge, gauges.connectionsOpen); - res.observe(busyGauge, gauges.connectionsBusy); - res.observe(freeGauge, gauges.connectionsIdle); - - // Observe histogram statistics as gauges - if (histograms.queriesWait) { - res.observe(queriesWaitTimeCount, histograms.queriesWait.count); - res.observe(queriesWaitTimeSum, histograms.queriesWait.sum); - res.observe( - queriesWaitTimeMean, - histograms.queriesWait.count > 0 - ? histograms.queriesWait.sum / histograms.queriesWait.count - : 0 - ); - } - if (histograms.queriesDuration) { - res.observe(queriesDurationCount, histograms.queriesDuration.count); - res.observe(queriesDurationSum, histograms.queriesDuration.sum); - res.observe( - queriesDurationMean, - histograms.queriesDuration.count > 0 - ? histograms.queriesDuration.sum / histograms.queriesDuration.count - : 0 - ); - } + for (const client of clients) { + const attributes = { db_client: client.clientType, db_driver: client.driver }; + const { pool, counters, gauges, histograms } = client; + + res.observe(queriesTotal, counters.queriesTotal, attributes); + res.observe(datasourceQueriesTotal, counters.datasourceQueriesTotal, attributes); + res.observe(connectionsOpenedTotal, pool.openedTotal, attributes); + res.observe(connectionsClosedTotal, pool.closedTotal, attributes); + + res.observe(queriesActive, gauges.queriesActive, attributes); + res.observe(queriesWait, gauges.queriesWait, attributes); + res.observe(totalGauge, pool.open, attributes); + res.observe(busyGauge, pool.busy, attributes); + res.observe(freeGauge, pool.idle, attributes); + res.observe(waitingGauge, pool.waiting, attributes); + + if (histograms.queriesWait) { + res.observe(queriesWaitTimeCount, histograms.queriesWait.count, attributes); + res.observe(queriesWaitTimeSum, histograms.queriesWait.sum, attributes); + res.observe( + queriesWaitTimeMean, + histograms.queriesWait.count > 0 + ? histograms.queriesWait.sum / histograms.queriesWait.count + : 0, + attributes + ); + } - if (histograms.datasourceQueriesDuration) { - res.observe(datasourceQueriesDurationCount, histograms.datasourceQueriesDuration.count); - res.observe(datasourceQueriesDurationSum, histograms.datasourceQueriesDuration.sum); - res.observe( - datasourceQueriesDurationMean, - histograms.datasourceQueriesDuration.count > 0 - ? histograms.datasourceQueriesDuration.sum / histograms.datasourceQueriesDuration.count - : 0 - ); + if (histograms.queriesDuration) { + res.observe(queriesDurationCount, histograms.queriesDuration.count, attributes); + res.observe(queriesDurationSum, histograms.queriesDuration.sum, attributes); + res.observe( + queriesDurationMean, + histograms.queriesDuration.count > 0 + ? histograms.queriesDuration.sum / histograms.queriesDuration.count + : 0, + attributes + ); + } + + if (histograms.datasourceQueriesDuration) { + res.observe( + datasourceQueriesDurationCount, + histograms.datasourceQueriesDuration.count, + attributes + ); + res.observe( + datasourceQueriesDurationSum, + histograms.datasourceQueriesDuration.sum, + attributes + ); + res.observe( + datasourceQueriesDurationMean, + histograms.datasourceQueriesDuration.count > 0 + ? histograms.datasourceQueriesDuration.sum / + histograms.datasourceQueriesDuration.count + : 0, + attributes + ); + } } }, [ @@ -616,6 +589,7 @@ function configurePrismaMetrics({ meter }: { meter: Meter }) { totalGauge, busyGauge, freeGauge, + waitingGauge, queriesWaitTimeCount, queriesWaitTimeSum, queriesWaitTimeMean,