Skip to content

feat(webapp): per-client database pool metrics that survive the driver adapter #4541

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ericallam wants to merge 2 commits into
base: main
Choose a base branch
from
+479 −147
Draft

feat(webapp): per-client database pool metrics that survive the driver adapter #4541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c71c46b
feat(webapp): per-client database pool metrics that survive the drive…
ericallam Aug 8, 2026
5d964b2
feat(webapp): use canonical datasource labels for db_client metric at…
ericallam Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .server-changes/db-pool-metrics-per-client.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
area: webapp
type: improvement
---

Database connection-pool and query metrics are now reported for every configured database connection rather than only the primary, and keep working regardless of which database driver a connection uses.
162 changes: 120 additions & 42 deletions apps/webapp/app/db.server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
logTransactionInfrastructureError,
} from "./utils/prismaErrors";
import { singleton } from "./utils/singleton";
import { registerDatabaseMetricsSource } from "./utils/databaseMetrics.server";
import {
isSplitEnabled,
assertSplitRealtimeInterlock,
Expand Down Expand Up @@ -247,16 +248,16 @@ export function selectRunOpsTopology(
if (config.legacySharesControlPlane) {
legacyRunOps = controlPlane;
} else {
const legacyWriter = builders.buildLegacyWriter(config.legacyUrl, "run-ops-legacy-writer");
const legacyWriter = builders.buildLegacyWriter(config.legacyUrl, "legacy-run-ops-writer");
const legacyReplica: PrismaReplicaClient = config.legacyReplicaUrl
? builders.buildLegacyReplica(config.legacyReplicaUrl, "run-ops-legacy-reader")
? builders.buildLegacyReplica(config.legacyReplicaUrl, "legacy-run-ops-replica")
: legacyWriter;
legacyRunOps = { writer: legacyWriter, replica: legacyReplica };
}

const newWriter = builders.buildNewWriter(config.newUrl, "run-ops-new-writer");
const newWriter = builders.buildNewWriter(config.newUrl, "run-ops-writer");
const newReplica: RunOpsPrismaClient = config.newReplicaUrl
? builders.buildNewReplica(config.newReplicaUrl, "run-ops-new-reader")
? builders.buildNewReplica(config.newReplicaUrl, "run-ops-replica")
: newWriter;

return {
Expand Down Expand Up @@ -430,19 +431,25 @@ function getClient() {

return buildWriterClient({
url,
clientType: "writer",
clientType: "control-plane-writer",
poolTimeout: env.DATABASE_WRITER_POOL_TIMEOUT,
connectTimeout: env.DATABASE_WRITER_CONNECTION_TIMEOUT,
useDriverAdapter: env.CONTROL_PLANE_DATABASE_WRITER_DRIVER_ADAPTER === "1",
});
}

type DriverAdapterPool = {
adapter: PrismaPg;
pool: Pool;
poolCounters: { opened: () => number; closed: () => number };
};

function buildDriverAdapterPool(
connectionString: string,
clientType: string,
poolTimeoutSeconds: number,
connectionLimit: number
): PrismaPg {
): DriverAdapterPool {
const pool = new Pool({
connectionString,
max: connectionLimit,
Expand All @@ -457,14 +464,27 @@ function buildDriverAdapterPool(
});
});

let opened = 0;
let closed = 0;
pool.on("connect", () => {
opened += 1;
});
pool.on("remove", () => {
closed += 1;
});

let schema: string | undefined;
try {
schema = new URL(connectionString).searchParams.get("schema") ?? undefined;
} catch {
schema = undefined;
}

return new PrismaPg(pool, { schema, disposeExternalPool: true });
return {
adapter: new PrismaPg(pool, { schema, disposeExternalPool: true }),
pool,
poolCounters: { opened: () => opened, closed: () => closed },
};
}

// Generalized writer builder shared by the control-plane client and the run-ops
Expand Down Expand Up @@ -548,21 +568,34 @@ export function buildWriterClient({
: []) satisfies Prisma.LogDefinition[]),
] satisfies Prisma.LogDefinition[];

const client = useDriverAdapter
? new PrismaClient({
adapter: buildDriverAdapterPool(
url,
clientType,
poolTimeout ?? env.DATABASE_POOL_TIMEOUT,
env.DATABASE_CONNECTION_LIMIT
),
log: logConfig,
})
const driverPool = useDriverAdapter
? buildDriverAdapterPool(
url,
clientType,
poolTimeout ?? env.DATABASE_POOL_TIMEOUT,
env.DATABASE_CONNECTION_LIMIT
)
: undefined;

const client = driverPool
? new PrismaClient({ adapter: driverPool.adapter, log: logConfig })
: new PrismaClient({
datasources: { db: { url: databaseUrl.href } },
log: logConfig,
});

registerDatabaseMetricsSource(
driverPool
? {
clientType,
usesDriverAdapter: true,
client,
pool: driverPool.pool,
poolCounters: driverPool.poolCounters,
}
: { clientType, usesDriverAdapter: false, client }
);

// Only use structured logging if we're not already logging to stdout
if (process.env.PRISMA_LOG_TO_STDOUT !== "1") {
client.$on("info", (log) => {
Expand Down Expand Up @@ -631,7 +664,7 @@ function getReplicaClient() {

return buildReplicaClient({
url,
clientType: "reader",
clientType: "control-plane-replica",
poolTimeout: env.DATABASE_READ_REPLICA_POOL_TIMEOUT,
connectTimeout: env.DATABASE_READ_REPLICA_CONNECTION_TIMEOUT,
useDriverAdapter: env.CONTROL_PLANE_DATABASE_REPLICA_DRIVER_ADAPTER === "1",
Expand Down Expand Up @@ -719,21 +752,34 @@ export function buildReplicaClient({
: []) satisfies Prisma.LogDefinition[]),
] satisfies Prisma.LogDefinition[];

const replicaClient = useDriverAdapter
? new PrismaClient({
adapter: buildDriverAdapterPool(
url,
clientType,
poolTimeout ?? env.DATABASE_POOL_TIMEOUT,
env.DATABASE_CONNECTION_LIMIT
),
log: logConfig,
})
const driverPool = useDriverAdapter
? buildDriverAdapterPool(
url,
clientType,
poolTimeout ?? env.DATABASE_POOL_TIMEOUT,
env.DATABASE_CONNECTION_LIMIT
)
: undefined;

const replicaClient = driverPool
? new PrismaClient({ adapter: driverPool.adapter, log: logConfig })
: new PrismaClient({
datasources: { db: { url: replicaUrl.href } },
log: logConfig,
});

registerDatabaseMetricsSource(
driverPool
? {
clientType,
usesDriverAdapter: true,
client: replicaClient,
pool: driverPool.pool,
poolCounters: driverPool.poolCounters,
}
: { clientType, usesDriverAdapter: false, client: replicaClient }
);

// Only use structured logging if we're not already logging to stdout
if (process.env.PRISMA_LOG_TO_STDOUT !== "1") {
replicaClient.$on("info", (log) => {
Expand Down Expand Up @@ -813,14 +859,18 @@ function buildRunOpsWriterClient({
}`
);

const client = useDriverAdapter
const driverPool = useDriverAdapter
? buildDriverAdapterPool(
url,
clientType,
env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
env.DATABASE_CONNECTION_LIMIT
)
: undefined;

const client = driverPool
? new RunOpsPrismaClient({
adapter: buildDriverAdapterPool(
url,
clientType,
env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
env.DATABASE_CONNECTION_LIMIT
),
adapter: driverPool.adapter,
log: [
{ emit: "event", level: "error" },
{ emit: "event", level: "info" },
Expand All @@ -844,6 +894,18 @@ function buildRunOpsWriterClient({
],
});

registerDatabaseMetricsSource(
driverPool
? {
clientType,
usesDriverAdapter: true,
client,
pool: driverPool.pool,
poolCounters: driverPool.poolCounters,
}
: { clientType, usesDriverAdapter: false, client }
);

if (process.env.PRISMA_LOG_TO_STDOUT !== "1") {
client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log }));
client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log }));
Expand Down Expand Up @@ -894,14 +956,18 @@ function buildRunOpsReplicaClient({
}`
);

const client = useDriverAdapter
const driverPool = useDriverAdapter
? buildDriverAdapterPool(
url,
clientType,
env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT
)
: undefined;

const client = driverPool
? new RunOpsPrismaClient({
adapter: buildDriverAdapterPool(
url,
clientType,
env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT
),
adapter: driverPool.adapter,
log: [
{ emit: "event", level: "error" },
{ emit: "event", level: "info" },
Expand All @@ -925,6 +991,18 @@ function buildRunOpsReplicaClient({
],
});

registerDatabaseMetricsSource(
driverPool
? {
clientType,
usesDriverAdapter: true,
client,
pool: driverPool.pool,
poolCounters: driverPool.poolCounters,
}
: { clientType, usesDriverAdapter: false, client }
);

if (process.env.PRISMA_LOG_TO_STDOUT !== "1") {
client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log }));
client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log }));
Expand Down
13 changes: 1 addition & 12 deletions apps/webapp/app/routes/metrics.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
import { prisma } from "~/db.server";
import { metricsRegister } from "~/metrics.server";

export async function loader({ request }: LoaderFunctionArgs) {
Expand All @@ -13,17 +12,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
}
}

// We need to remove empty lines from the prisma metrics, grafana doesn't like them
let prismaMetrics = "";
try {
prismaMetrics = (await prisma.$metrics.prometheus()).replace(/^\s*[\r\n]/gm, "");
} catch {
prismaMetrics = "";
}
const coreMetrics = await metricsRegister.metrics();

// Order matters, core metrics end with `# EOF`, prisma metrics don't
const metrics = prismaMetrics + coreMetrics;
const metrics = await metricsRegister.metrics();

@coderabbitai coderabbitai Bot Aug 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Description: Inspect the OTEL metric exporter gate and its default.
rg -n -C5 'INTERNAL_OTEL_METRIC_EXPORTER_ENABLED' apps/webapp --type=ts
rg -n -C3 'initializeMetricsRegister' apps/webapp/app/metrics.server.ts

Repository: triggerdotdev/trigger.dev

Length of output: 2208

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== files =="
git ls-files 'apps/webapp/app/routes/metrics.ts' 'apps/webapp/app/metrics.server.ts' 'apps/webapp/app/v3/tracer.server.ts' 'apps/webapp/app/services/**/*server*.ts' 'apps/webapp/app/db.server/*' 2>/dev/null || true

echo "== metrics route =="
sed -n '1,120p' apps/webapp/app/routes/metrics.ts

echo "== metrics server =="
sed -n '1,220p' apps/webapp/app/metrics.server.ts

echo "== tracer setupMetrics + configurePrismaMetrics =="
sed -n '340,470p' apps/webapp/app/v3/tracer.server.ts

Repository: triggerdotdev/trigger.dev

Length of output: 9489

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== current git diff stat =="
git diff --stat -- packages/core apps/webapp || true

echo "== current git diff metrics route and related files =="
git diff -- apps/webapp/app/routes/metrics.ts apps/webapp/app/metrics.server.ts apps/webapp/app/v3/tracer.server.ts apps/webapp/app/env.server.ts || true

echo "== Prisma pool/config references in route/tracer/metrics =="
rg -n 'PrismaPool|openPool|connectionsOpened|connectionsClosed|db\.pool|db\.client\.queries|configurePrismaMetrics|internalInstrumentPrisma' apps/webapp/app/routes/metrics.ts apps/webapp/app/metrics.server.ts apps/webapp/app/v3/tracer.server.ts apps/webapp/app/env.server.ts || true

Repository: triggerdotdev/trigger.dev

Length of output: 2793

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== tracer Prisma metrics implementation =="
sed -n '470,610p' apps/webapp/app/v3/tracer.server.ts

echo "== setupMetrics call sites =="
rg -n -C3 'setupMetrics|configurePrismaMetrics|configureNodejsMetrics|configureHostMetrics|INTERNAL_OTEL_METRIC_EXPORTER_ENABLED === "0"' apps/webapp/app --type=ts

echo "== route/prisma instrumentation references across webapp =="
rg -n 'INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED|PrismaInstrumentation|PrismaPool|prisma' apps/webapp/app --type=ts | head -n 200

Repository: triggerdotdev/trigger.dev

Length of output: 29710

Keep Prisma DB metrics on the Prometheus registry.

configurePrismaMetrics() only runs when INTERNAL_OTEL_METRIC_EXPORTER_ENABLED !== "0", but that variable defaults to "0", and the /metrics route only reads from metricsRegister. Self-hosted scrapes that do not enable the OTLP exporter now miss db.client.* and db.pool.* series. Register those observations on metricsRegister as another source.


return new Response(metrics, {
headers: {
Expand Down
Loading
Loading