feat(monitoring): add project health and failure alerts

AGENTS.md

@@ -178,7 +178,7 @@ Requires secrets: `VERCEL_TOKEN`, `VERCEL_ORG_ID`, `VERCEL_PROJECT_ID`
 | `DATABASE_URL` | `postgresql://dequel:dequel@localhost:5432/dequel` | PostgreSQL connection string |
 | `WORKSPACE_ROOT` | `./workspace` | Build staging |
 | `CADDY_ROUTES_DIR` | `./infra/caddy/routes` | Caddy route output |
-| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Set to a real domain (e.g. `example.com`) for Let's Encrypt auto-SSL. |
+| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Set to a real domain (e.g. `example.com`) for Let's Encrypt auto-SSL. Public links (e.g. failure email logs) derive their base URL from this. |
 | `CADDY_EMAIL` | _(empty)_ | Email for Let's Encrypt SSL certificate notifications |
 | `DOCKER_NETWORK` | `dequel_net` | Docker network for deployments |
 | `BUILDKIT_HOST` | `tcp://buildkit:1234` | Buildkit daemon |

apps/api/src/agents/job-channel.ts

@@ -9,6 +9,7 @@ import {
 	leaseNextAgentJob,
 	listCancelledJobIds,
 	listDeployments,
+	recordDeploymentFailure,
 	updateAgentHeartbeat,
 	updateDeploymentCommitSha,
 	updateDeploymentStatus,
@@ -119,8 +120,10 @@ export const processAgentJobUpdate = async (
 				}
 			}
 		} else {
-			await updateDeploymentStatus(deploymentId, "failed", {
-				failureReason: update.error || "Remote agent deployment failed",
+			await recordDeploymentFailure({
+				deploymentId,
+				reason: update.error || "Remote agent deployment failed",
+				source: "job-channel",
 			});
 			await appendLog(deploymentId, "system", `Remote deployment failed: ${update.error || "Unknown agent error"}`);
 		}

apps/api/src/api/alerts/index.ts

@@ -1,14 +1,38 @@
 import { Elysia } from "elysia";
 import { createAlert, deleteAlert, listAlerts, updateAlertEnabled } from "../../db/repo";
+import type { AlertChannel, AlertType } from "../../types";
 import { created, fail, ok } from "../response";
 
+const ALERT_TYPES: ReadonlySet<string> = new Set<AlertType>(["cpu", "memory", "downtime", "cert_expiry"]);
+const ALERT_CHANNELS: ReadonlySet<string> = new Set<AlertChannel>(["email", "slack", "webhook"]);
+
 export const alertsRoutes = new Elysia()
 	.get("/projects/:id/alerts", async ({ params }) => ok(await listAlerts(params.id)))
 	.post("/projects/:id/alerts", async ({ params, body, set }: any) => {
 		if (!body?.type || !body?.channel) {
 			set.status = 400;
 			return fail("type and channel are required");
 		}
+		if (!ALERT_TYPES.has(String(body.type))) {
+			set.status = 400;
+			return fail(`type must be one of: ${[...ALERT_TYPES].join(", ")}`);
+		}
+		if (!ALERT_CHANNELS.has(String(body.channel))) {
+			set.status = 400;
+			return fail(`channel must be one of: ${[...ALERT_CHANNELS].join(", ")}`);
+		}
+		if (body.channel !== "email") {
+			let valid = false;
+			try {
+				valid = ["http:", "https:"].includes(new URL(String(body.destination ?? "")).protocol);
+			} catch {
+				valid = false;
+			}
+			if (!valid) {
+				set.status = 400;
+				return fail("destination must be an http(s) URL for this channel");
+			}
+		}
 		return created(
 			await createAlert({
 				projectId: params.id,

apps/api/src/api/backups/index.ts

@@ -1,7 +1,7 @@
 import { Elysia } from "elysia";
 import { BackupOrchestrator } from "../../backup/orchestrator";
-import { S3_BACKUP_PREFIX } from "../../backup/types";
 import type { BackupTarget, StorageConfig } from "../../backup/types";
+import { S3_BACKUP_PREFIX } from "../../backup/types";
 import { deleteBackupRecord, getBackupRecord, listBackupRecords } from "../../db/repo/backups";
 import { getDatabaseById } from "../../db/repo/databases";
 import { getBackupStorageSettings } from "../../db/repo/settings";

apps/api/src/api/deployments/index.ts

@@ -10,9 +10,11 @@ import {
 	getProjectById,
 	getServerById,
 	listDeployments,
+	recordDeploymentFailure,
 } from "../../db/repo";
 import { executorFor } from "../../executors/dispatch";
 import { orchestrator } from "../../orchestrator";
+import { summarizeDeploymentError } from "../../orchestrator/deployment-errors";
 import { logBus } from "../../orchestrator/log-bus";
 import { config } from "../../utils/config";
 import { isPrivateGitUrl } from "../../utils/validate";
@@ -31,8 +33,13 @@ const dispatchDeployment = async (
 	if (deployment.sourceType !== "git") throw new Error("Remote servers currently support Git deployments only");
 	const executor = executorFor(server.mode);
 	if (server.mode === "ssh") {
-		void executor.deploy({ deployment, project, server }).catch((error) => {
+		void executor.deploy({ deployment, project, server }).catch(async (error) => {
 			console.error(`[SSH Executor] Deployment ${deployment.id} failed:`, error);
+			await recordDeploymentFailure({
+				deploymentId: deployment.id,
+				reason: summarizeDeploymentError(error),
+				source: "dispatch",
+			}).catch((e) => console.error(`[SSH Executor] Failed to record failure for ${deployment.id}:`, e));
 		});
 		return;
 	}

apps/api/src/api/index.ts

@@ -1,4 +1,5 @@
 import { Elysia } from "elysia";
+import { fail } from "./response";
 import { agentRoutes } from "./agents";
 import { alertsRoutes } from "./alerts";
 import { apiKeysRoutes } from "./api-keys";
@@ -11,6 +12,7 @@ import { envVarsRoutes } from "./env-vars";
 import { githubRoutes } from "./github";
 import { healthRoutes } from "./health";
 import { projectsRoutes } from "./projects";
+import { projectStatusRoutes } from "./projects/status";
 import { prometheusRoutes } from "./prometheus";
 import { routesRoutes } from "./routes";
 import { scalingRoutes } from "./scaling";
@@ -44,7 +46,7 @@ const authMiddleware = (app: Elysia) =>
 			const payload = await verifyAccessToken(match[1]);
 			if (payload) return;
 			set.status = 401;
-			return { error: "Invalid session" };
+			return fail("Invalid session");
 		}
 
 		const authHeader = request.headers.get("authorization");
@@ -55,22 +57,36 @@ const authMiddleware = (app: Elysia) =>
 				const key = await validateApiKey(token);
 				if (key) return;
 				set.status = 401;
-				return { error: "Invalid API key" };
+				return fail("Invalid API key");
 			}
 		}
 
 		set.status = 401;
-		return { error: "Authentication required" };
+		return fail("Authentication required");
 	});
 
+const INTERNAL_ERROR =
+	/Failed query:|params:|getaddrinfo|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EHOSTUNREACH|timeout exceeded|node:internal|Cannot read propert|is not a function|is not a constructor|Unexpected token/i;
+
 export const apiRoutes = new Elysia({
 	prefix: "/api",
 })
+	.onError(({ error, set }) => {
+		const err = error as { status?: number; message?: string };
+		set.status = typeof err?.status === "number" ? err.status : 500;
+		const message = err?.message ?? "Internal server error";
+		if (INTERNAL_ERROR.test(message)) {
+			console.error("[API] Unhandled error:", error);
+			return fail("Internal server error");
+		}
+		return fail(message);
+	})
 	.use(authRoutes)
 	.use(authMiddleware)
 	.use(agentRoutes)
 	.use(healthRoutes)
 	.use(projectsRoutes)
+	.use(projectStatusRoutes)
 	.use(deploymentsRoutes)
 	.use(envVarsRoutes)
 	.use(sharedEnvVarsRoutes)

apps/api/src/api/projects/index.ts

@@ -6,7 +6,6 @@ import {
 	deleteProjectCascade,
 	getProjectById,
 	getServerById,
-	listDomains,
 	listProjects,
 	updateProject,
 } from "../../db/repo";
@@ -15,6 +14,7 @@ import { reloadCaddy, tryRun } from "../../orchestrator/runtime";
 import { config } from "../../utils/config";
 import { dockerBin } from "../../utils/docker-bin";
 import { removeFromCaddyRoute } from "../../utils/domain-verifier";
+import { buildProjectRequestHostRegex, caddyRequestLogSelector } from "../../utils/loki";
 import { isPort, isPrivateGitUrl, SERVICE_NAME_RE, validateComposeServices } from "../../utils/validate";
 import { created, fail, ok } from "../response";
 
@@ -194,22 +194,8 @@ export const projectsRoutes = new Elysia()
 			set.status = 404;
 			return fail("Project not found");
 		}
-		const slugify = (s: string) =>
-			s
-				.toLowerCase()
-				.replace(/[^a-z0-9-]+/g, "-")
-				.replace(/^-+|-+$/g, "")
-				.slice(0, 63);
-		const slug = slugify(project.name);
-		const domains = [`${slug}.${config.caddyBaseDomain}`];
-		const projectDomains = await listDomains(id);
-		const verified = projectDomains.filter((d) => d.validationStatus === "verified");
-		for (const d of verified) {
-			domains.push(d.domain);
-		}
-
-		const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|");
-		const queryStr = `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$"`;
+		const hostRegex = await buildProjectRequestHostRegex(id);
+		const queryStr = caddyRequestLogSelector(hostRegex);
 
 		const startParam = (queryParams as any)?.start;
 		const endParam = (queryParams as any)?.end;
@@ -274,23 +260,9 @@ export const projectsRoutes = new Elysia()
 			set.status = 404;
 			return fail("Project not found");
 		}
-		const slugify = (s: string) =>
-			s
-				.toLowerCase()
-				.replace(/[^a-z0-9-]+/g, "-")
-				.replace(/^-+|-+$/g, "")
-				.slice(0, 63);
-		const slug = slugify(project.name);
-		const domains = [`${slug}.${config.caddyBaseDomain}`];
-		const projectDomains = await listDomains(id);
-		const verified = projectDomains.filter((d) => d.validationStatus === "verified");
-		for (const d of verified) {
-			domains.push(d.domain);
-		}
+		const hostRegex = await buildProjectRequestHostRegex(id);
 
-		const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|");
-
-		const query = `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [5m]))`;
+		const query = `sum(count_over_time(${caddyRequestLogSelector(hostRegex)} [5m]))`;
 
 		const end = Math.floor(Date.now() / 1000);
 		const start = end - 6 * 60 * 60;
@@ -325,22 +297,8 @@ export const projectsRoutes = new Elysia()
 			return fail("Project not found");
 		}
 		const encoder = new TextEncoder();
-		const slugify = (s: string) =>
-			s
-				.toLowerCase()
-				.replace(/[^a-z0-9-]+/g, "-")
-				.replace(/^-+|-+$/g, "")
-				.slice(0, 63);
-		const slug = slugify(project.name);
-		const domains = [`${slug}.${config.caddyBaseDomain}`];
-		const projectDomains = await listDomains(id);
-		const verified = projectDomains.filter((d) => d.validationStatus === "verified");
-		for (const d of verified) {
-			domains.push(d.domain);
-		}
-
-		const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|");
-		const query = `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$"`;
+		const hostRegex = await buildProjectRequestHostRegex(id);
+		const query = caddyRequestLogSelector(hostRegex);
 
 		let ws: WebSocket | null = null;
 		let closed = false;

apps/api/src/api/projects/status.ts

@@ -0,0 +1,29 @@
+import { Elysia } from "elysia";
+import { getProjectById } from "../../db/repo";
+import { getProjectStatus } from "../../monitoring/project-status";
+import { fail, ok } from "../response";
+
+const DEFAULT_WINDOW_SECONDS = 3600;
+const MIN_WINDOW_SECONDS = 60;
+const MAX_WINDOW_SECONDS = 604800;
+
+const clampWindow = (raw: unknown): number => {
+	const value = Number(raw);
+	if (!Number.isFinite(value) || value <= 0) return DEFAULT_WINDOW_SECONDS;
+	return Math.min(MAX_WINDOW_SECONDS, Math.max(MIN_WINDOW_SECONDS, Math.floor(value)));
+};
+
+export const projectStatusRoutes = new Elysia().get("/projects/:id/status", async ({ params: { id }, query, set }) => {
+	const project = await getProjectById(id);
+	if (!project) {
+		set.status = 404;
+		return fail("Project not found");
+	}
+	try {
+		return ok(await getProjectStatus(id, clampWindow((query as any)?.window)));
+	} catch (err) {
+		console.error(`[Status] Failed to build status for project ${id}:`, err);
+		set.status = 500;
+		return fail("Failed to build project status");
+	}
+});

apps/api/src/api/settings/index.ts

@@ -9,6 +9,7 @@ import {
 	upsertBackupStorageSettings,
 	upsertSmtpSettings,
 } from "../../db/repo";
+import { buildSmtpTestEmail } from "../../monitoring/templates";
 import { failoverState } from "../../orchestrator/failover";
 import { rerenderAllIngressRoutes } from "../../orchestrator/ingress-sync";
 import { fail, ok } from "../response";
@@ -87,10 +88,12 @@ export const settingsRoutes = new Elysia({ prefix: "/settings" })
 				secure: settings.port === 465,
 				auth: settings.user && settings.pass ? { user: settings.user, pass: settings.pass } : undefined,
 			});
+			const { subject, html } = buildSmtpTestEmail();
 			await transporter.sendMail({
 				from: settings.fromAddress,
 				to: settings.fromAddress,
-				subject: "[Dequel] SMTP Test Email",
+				subject,
+				html,
 				text: "This is a test email from Dequel. Your SMTP settings are working correctly.",
 			});
 			return ok(null, "Test email sent");

apps/api/src/api/shared-env-vars/index.ts

@@ -1,7 +1,7 @@
-import { Elysia } from "elysia";
 import { and, eq } from "drizzle-orm";
+import { Elysia } from "elysia";
 import { getDb } from "../../db/db-provider";
-import { projectSharedEnvLinks } from "../../db/schema";
+import { getRowsAffected } from "../../db/repo/helpers";
 import {
 	createSharedEnvVar,
 	deleteSharedEnvVar,
@@ -12,7 +12,7 @@ import {
 	listSharedEnvVars,
 	updateSharedEnvVar,
 } from "../../db/repo/shared-env-vars";
-import { getRowsAffected } from "../../db/repo/helpers";
+import { projectSharedEnvLinks } from "../../db/schema";
 import { fail, ok } from "../response";
 
 export const sharedEnvVarsRoutes = new Elysia()
@@ -89,12 +89,7 @@ export const sharedEnvLinksRoutes = new Elysia()
 			getRowsAffected(
 				await db
 					.delete(projectSharedEnvLinks)
-					.where(
-					and(
-						eq(projectSharedEnvLinks.id, params.linkId),
-						eq(projectSharedEnvLinks.projectId, params.id),
-					),
-				)
+					.where(and(eq(projectSharedEnvLinks.id, params.linkId), eq(projectSharedEnvLinks.projectId, params.id)))
 					.execute(),
 			) > 0;
 		if (!removed) {