Merge pull request #305 from techulus/automation/release/v0.60.0

Release v0.60.0

Author: arjunkomath

web/app/api/inngest/route.ts

@@ -20,6 +20,7 @@ import {
 	previewServiceReconcileWorkflow,
 	previewSyncWorkflow,
 	registryArtifactRetention,
+	resourceUsageCheck,
 	restoreTriggerWorkflow,
 	restoreWorkflow,
 	rolloutWorkflow,
@@ -52,6 +53,7 @@ export const { GET, POST, PUT } = serve({
 		controlPlaneUpdateCheck,
 		agentUpgradeTimeoutCheck,
 		registryArtifactRetention,
+		resourceUsageCheck,
 		migrationWorkflow,
 		sentryFailureWorkflow,
 		backupWorkflow,

web/app/api/v1/agent/builds/[id]/status/route.ts

@@ -283,7 +283,7 @@ export async function POST(
 						update.status === "completed"
 							? "Preview image built; preparing deployment"
 							: update.status === "failed"
-								? update.error || "Preview build failed"
+								? "Preview build failed"
 								: `Preview build ${update.status}...`,
 					logUrl,
 				});
@@ -325,7 +325,7 @@ export async function POST(
 						build.githubDeploymentId,
 						"failure",
 						{
-							description: update.error || "Build failed",
+							description: "Build failed",
 							logUrl,
 							environmentUrl,
 						},

web/components/cluster/cluster-health-summary.tsx

@@ -9,6 +9,11 @@ type ClusterHealthData = {
 		onlineServers: number;
 		networkHealthy: number;
 		containerHealthy: number;
+		resourceWarnings: {
+			cpu: number;
+			memory: number;
+			disk: number;
+		};
 	};
 };
 
@@ -30,6 +35,7 @@ export function ClusterHealthSummary({
 
 	const stats = [
 		{
+			key: "servers",
 			label: "servers",
 			value: `${summary.onlineServers}/${summary.totalServers}`,
 			subtitle: "online",
@@ -38,17 +44,35 @@ export function ClusterHealthSummary({
 				summary.onlineServers === summary.totalServers,
 		},
 		{
+			key: "tunnels",
 			label: "tunnels",
 			value: `${summary.networkHealthy}/${summary.onlineServers}`,
 			subtitle: "connected",
 			healthy: anyOnline && summary.networkHealthy === summary.onlineServers,
 		},
 		{
+			key: "runtimes",
 			label: "runtimes",
 			value: `${summary.containerHealthy}/${summary.onlineServers}`,
 			subtitle: "responsive",
 			healthy: anyOnline && summary.containerHealthy === summary.onlineServers,
 		},
+		...(
+			[
+				["cpu", "CPU"],
+				["memory", "memory"],
+				["disk", "disk"],
+			] as const
+		).map(([key, label]) => {
+			const count = summary.resourceWarnings[key];
+			return {
+				key,
+				label: count === 1 ? "server" : "servers",
+				value: String(count),
+				subtitle: `at high ${label} usage`,
+				healthy: count === 0,
+			};
+		}),
 	];
 
 	const degraded = stats.filter((stat) => !stat.healthy);
@@ -65,7 +89,7 @@ export function ClusterHealthSummary({
 	return (
 		<div className="flex flex-wrap items-center gap-x-4 gap-y-1 text-sm">
 			{degraded.map((stat) => (
-				<div key={stat.label} className="flex items-center gap-1.5">
+				<div key={stat.key} className="flex items-center gap-1.5">
 					<span className="h-2 w-2 shrink-0 rounded-full bg-amber-500" />
 					<span className="font-semibold tabular-nums text-foreground">
 						{stat.value}

web/components/settings/email-settings.tsx

@@ -17,6 +17,7 @@ type Props = {
 
 type AlertField =
 	| "serverOfflineAlert"
+	| "resourceUsageAlert"
 	| "buildFailure"
 	| "deploymentFailure"
 	| "deploymentMovedAlert"
@@ -34,6 +35,12 @@ const ALERT_SETTINGS: AlertSetting[] = [
 		label: "Server Offline Alert",
 		description: "Receive a notification when a server goes offline",
 	},
+	{
+		field: "resourceUsageAlert",
+		label: "Resource Usage Alert",
+		description:
+			"Receive a notification when server CPU, memory, or disk usage is high",
+	},
 	{
 		field: "buildFailure",
 		label: "Build Failure Alert",
@@ -59,6 +66,7 @@ const ALERT_SETTINGS: AlertSetting[] = [
 
 type State = {
 	serverOfflineAlert: boolean;
+	resourceUsageAlert: boolean;
 	buildFailure: boolean;
 	deploymentFailure: boolean;
 	deploymentMovedAlert: boolean;
@@ -74,6 +82,7 @@ function createInitialState(props: Props): State {
 	const { initialAlertsConfig: alertsConfig } = props;
 	return {
 		serverOfflineAlert: alertsConfig?.serverOfflineAlert ?? true,
+		resourceUsageAlert: alertsConfig?.resourceUsageAlert ?? true,
 		buildFailure: alertsConfig?.buildFailure ?? true,
 		deploymentFailure: alertsConfig?.deploymentFailure ?? true,
 		deploymentMovedAlert: alertsConfig?.deploymentMovedAlert ?? true,
@@ -104,6 +113,7 @@ export function EmailSettings({ initialAlertsConfig }: Props) {
 		try {
 			await updateEmailAlertsConfig({
 				serverOfflineAlert: state.serverOfflineAlert,
+				resourceUsageAlert: state.resourceUsageAlert,
 				buildFailure: state.buildFailure,
 				deploymentFailure: state.deploymentFailure,
 				deploymentMovedAlert: state.deploymentMovedAlert,
@@ -125,6 +135,8 @@ export function EmailSettings({ initialAlertsConfig }: Props) {
 	const hasAlertsChanges =
 		state.serverOfflineAlert !==
 			(initialAlertsConfig?.serverOfflineAlert ?? true) ||
+		state.resourceUsageAlert !==
+			(initialAlertsConfig?.resourceUsageAlert ?? true) ||
 		state.buildFailure !== (initialAlertsConfig?.buildFailure ?? true) ||
 		state.deploymentFailure !==
 			(initialAlertsConfig?.deploymentFailure ?? true) ||

web/db/queries.ts

@@ -165,6 +165,7 @@ export async function getClusterHealth() {
 			networkHealth: servers.networkHealth,
 			containerHealth: servers.containerHealth,
 			agentHealth: servers.agentHealth,
+			resourceAlerts: servers.resourceAlerts,
 		})
 		.from(servers);
 
@@ -176,13 +177,20 @@ export async function getClusterHealth() {
 	const containerHealthy = onlineServers.filter(
 		(s) => s.containerHealth?.runtimeResponsive,
 	).length;
+	const resourceWarnings = {
+		cpu: onlineServers.filter((server) => server.resourceAlerts?.cpu).length,
+		memory: onlineServers.filter((server) => server.resourceAlerts?.memory)
+			.length,
+		disk: onlineServers.filter((server) => server.resourceAlerts?.disk).length,
+	};
 
 	return {
 		summary: {
 			totalServers: allServers.length,
 			onlineServers: onlineServers.length,
 			networkHealthy,
 			containerHealthy,
+			resourceWarnings,
 		},
 		servers: allServers.map((server) => ({
 			id: server.id,

web/db/schema.ts

@@ -353,6 +353,17 @@ export type AgentHealth = {
 	capabilities?: string[];
 };
 
+export type ServerResourceAlert = {
+	usagePercent: number;
+	thresholdPercent: number;
+	detectedAt: string;
+	notificationEnqueued: boolean;
+};
+
+export type ServerResourceAlerts = Partial<
+	Record<"cpu" | "memory" | "disk", ServerResourceAlert>
+>;
+
 export type CrowdSecDecision = {
 	scope: string;
 	value: string;
@@ -427,6 +438,7 @@ export const servers = pgTable("servers", {
 	networkHealth: jsonb("network_health").$type<NetworkHealth>(),
 	containerHealth: jsonb("container_health").$type<ContainerHealth>(),
 	agentHealth: jsonb("agent_health").$type<AgentHealth>(),
+	resourceAlerts: jsonb("resource_alerts").$type<ServerResourceAlerts>(),
 	crowdsecHealth: jsonb("crowdsec_health").$type<CrowdSecHealth>(),
 	agentUpgradeTargetVersion: text("agent_upgrade_target_version"),
 	agentUpgradeStatus: text("agent_upgrade_status", {

web/lib/email/index.ts

@@ -208,6 +208,48 @@ async function sendManualRecoveryRequiredAlert(
 	});
 }
 
+type ServerResourceUsageAlertOptions = Extract<
+	NotificationEvent,
+	{ kind: "server.resource_usage" }
+> & { to: string };
+
+async function sendServerResourceUsageAlert(
+	options: ServerResourceUsageAlertOptions,
+): Promise<void> {
+	const baseUrl = getAppBaseUrl();
+	const serverUrl = baseUrl
+		? `${baseUrl}/dashboard/servers/${options.serverId}`
+		: undefined;
+	const resource =
+		options.resource === "cpu"
+			? "CPU"
+			: `${options.resource.charAt(0).toUpperCase()}${options.resource.slice(1)}`;
+	const usage = `${options.usagePercent.toFixed(1)}%`;
+
+	await sendAlert({
+		to: options.to,
+		subject: `Alert: High ${resource} usage on "${options.serverName}"`,
+		template: Alert({
+			bannerText: `HIGH ${resource.toUpperCase()} USAGE`,
+			heading: `${resource} Usage Alert`,
+			description: `${resource} usage on "${options.serverName}" has reached ${usage}, above the ${options.thresholdPercent}% threshold.`,
+			details: [
+				{ label: "Server", value: options.serverName },
+				{ label: "Resource", value: resource },
+				{ label: "Usage", value: usage },
+				{ label: "Threshold", value: `${options.thresholdPercent}%` },
+				{
+					label: "Detected At",
+					value: formatDateTimeUtc(new Date(options.detectedAt)),
+				},
+			],
+			buttonText: serverUrl ? "View Server" : undefined,
+			buttonUrl: serverUrl,
+			baseUrl,
+		}),
+	});
+}
+
 type BuildFailureAlertOptions = {
 	to: string;
 	serviceId: string;
@@ -451,6 +493,9 @@ export async function deliverNotificationEmail(
 		case "manual_recovery.required":
 			await sendManualRecoveryRequiredAlert({ ...event, to });
 			return;
+		case "server.resource_usage":
+			await sendServerResourceUsageAlert({ ...event, to });
+			return;
 		case "build.failed":
 			await sendBuildFailureAlert({ ...event, to });
 			return;

web/lib/inngest/events/notification.ts

@@ -15,6 +15,16 @@ export type NotificationEvent =
 			impactedReplicas: number;
 			serviceNames: string[];
 	  }
+	| {
+			kind: "server.resource_usage";
+			occurrenceId: string;
+			serverId: string;
+			serverName: string;
+			resource: "cpu" | "memory" | "disk";
+			usagePercent: number;
+			thresholdPercent: number;
+			detectedAt: string;
+	  }
 	| {
 			kind: "build.failed";
 			occurrenceId: string;

web/lib/inngest/functions/crons.ts

@@ -10,6 +10,7 @@ import { cleanupOldBackups, runScheduledBackups } from "@/lib/backup-scheduler";
 import { checkAndPersistControlPlaneUpdate } from "@/lib/control-plane-updates";
 import { cleanupReadNotifications } from "@/lib/notifications";
 import { cleanupRegistryArtifactsDaily } from "@/lib/registry-retention";
+import { evaluateServerResourceAlerts } from "@/lib/server-resource-alerts";
 import { cleanupOldServiceCommands } from "@/lib/service-command-retention";
 import {
 	checkAndRecoverStaleServers,
@@ -70,6 +71,19 @@ export const autoscalingCheck = inngest.createFunction(
 	},
 );
 
+export const resourceUsageCheck = inngest.createFunction(
+	{
+		id: "cron-resource-usage-check",
+		triggers: [cron("* * * * *")],
+		singleton: { mode: "skip" },
+	},
+	async ({ step }) => {
+		await step.run("evaluate-server-resource-alerts", () =>
+			evaluateServerResourceAlerts(),
+		);
+	},
+);
+
 export const scheduledDeploymentsCheck = inngest.createFunction(
 	{
 		id: "cron-scheduled-deployments",

web/lib/inngest/functions/index.ts

@@ -9,6 +9,7 @@ export {
 	controlPlaneUpdateCheck,
 	notificationRetention,
 	previewReconciliation,
+	resourceUsageCheck,
 	serviceCommandRetention,
 	serviceCronDispatcher,
 	oldBackupsCleanup,