From d4dbeb2046c582bc36b8633a65bdfe9b317f278e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 02:29:50 +0300 Subject: [PATCH 01/16] fix(traefik): override entrypoint certResolver for non-letsencrypt domains --- apps/dokploy/__test__/traefik/traefik.test.ts | 26 +++++++++++++++++++ packages/server/src/utils/traefik/domain.ts | 7 +++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/apps/dokploy/__test__/traefik/traefik.test.ts b/apps/dokploy/__test__/traefik/traefik.test.ts index b7b0f56455..22c2fd138e 100644 --- a/apps/dokploy/__test__/traefik/traefik.test.ts +++ b/apps/dokploy/__test__/traefik/traefik.test.ts @@ -346,6 +346,32 @@ test("Web entrypoint with empty middlewares array", async () => { /** Certificates */ +test("CertificateType none on websecure sets an empty tls block", async () => { + const router = await createRouterConfig( + baseApp, + { ...baseDomain, certificateType: "none" }, + "websecure", + ); + + // Must be defined, otherwise Traefik applies the websecure entrypoint's + // certResolver default and requests a Let's Encrypt certificate anyway. + expect(router.tls).toEqual({}); +}); + +test("CertificateType custom without a resolver sets an empty tls block", async () => { + const router = await createRouterConfig( + baseApp, + { + ...baseDomain, + certificateType: "custom", + customCertResolver: null, + }, + "websecure", + ); + + expect(router.tls).toEqual({}); +}); + test("CertificateType on websecure entrypoint", async () => { const router = await createRouterConfig( baseApp, diff --git a/packages/server/src/utils/traefik/domain.ts b/packages/server/src/utils/traefik/domain.ts index d35473a543..501180f62c 100644 --- a/packages/server/src/utils/traefik/domain.ts +++ b/packages/server/src/utils/traefik/domain.ts @@ -215,8 +215,11 @@ export const createRouterConfig = async ( routerConfig.tls = { certResolver: "letsencrypt" }; } else if (certificateType === "custom" && domain.customCertResolver) { routerConfig.tls = { certResolver: domain.customCertResolver }; - } else if (certificateType === "none") { - routerConfig.tls = undefined; + } else { + // An empty object still enables TLS, but marks the router's TLS as + // non-nil so Traefik does not fall back to the entrypoint's + // certResolver default. Same handling as forward-auth.ts. + routerConfig.tls = {}; } } From 448900e8efbae454be222eebae7056ee5231d228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 02:34:05 +0300 Subject: [PATCH 02/16] fix(traefik): emit tls=true for custom certificates without a resolver --- .../__test__/compose/domain/labels.test.ts | 16 ++++++++++++++++ packages/server/src/utils/docker/domain.ts | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/dokploy/__test__/compose/domain/labels.test.ts b/apps/dokploy/__test__/compose/domain/labels.test.ts index e48b54452e..8bd2e76ffd 100644 --- a/apps/dokploy/__test__/compose/domain/labels.test.ts +++ b/apps/dokploy/__test__/compose/domain/labels.test.ts @@ -543,4 +543,20 @@ describe("createDomainLabels", () => { // Should not contain redirect-to-https since there's only one router expect(middlewareLabel).toBeUndefined(); }); + + it("should add tls=true for certificateType custom without a resolver", async () => { + const customDomain = { + ...baseDomain, + https: true, + certificateType: "custom" as const, + customCertResolver: null, + }; + const labels = await createDomainLabels(appName, customDomain, "websecure"); + expect(labels).toContain( + "traefik.http.routers.test-app-1-websecure.tls=true", + ); + expect(labels).not.toContain( + "traefik.http.routers.test-app-1-websecure.tls.certresolver=letsencrypt", + ); + }); }); diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index 26acf29869..be22df8fda 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -405,9 +405,10 @@ export const createDomainLabels = ( labels.push( `traefik.http.routers.${routerName}.tls.certresolver=${customCertResolver}`, ); - } else if (certificateType === "none" && https) { + } else if (https) { // No cert resolver, but HTTPS is enabled (default/custom certificate): - // explicitly enable TLS so Traefik serves the router over HTTPS. + // explicitly enable TLS so Traefik serves the router over HTTPS and + // does not inherit the entrypoint's certResolver default. labels.push(`traefik.http.routers.${routerName}.tls=true`); } } From 301c15bf0642574ae627eb25cc64f39e674b06d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 02:37:05 +0300 Subject: [PATCH 03/16] feat(traefik): add helper to remove certificates from the ACME store --- apps/dokploy/__test__/traefik/acme.test.ts | 64 ++++++++++++++++++++++ packages/server/src/index.ts | 1 + packages/server/src/utils/traefik/acme.ts | 44 +++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 apps/dokploy/__test__/traefik/acme.test.ts create mode 100644 packages/server/src/utils/traefik/acme.ts diff --git a/apps/dokploy/__test__/traefik/acme.test.ts b/apps/dokploy/__test__/traefik/acme.test.ts new file mode 100644 index 0000000000..77c15f5462 --- /dev/null +++ b/apps/dokploy/__test__/traefik/acme.test.ts @@ -0,0 +1,64 @@ +import type { AcmeStore } from "@dokploy/server"; +import { removeAcmeCertificates } from "@dokploy/server"; +import { describe, expect, it } from "vitest"; + +const buildStore = (mains: string[]): AcmeStore => ({ + letsencrypt: { + Account: { Email: "test@example.com" }, + Certificates: mains.map((main) => ({ + domain: { main }, + certificate: "cert", + key: "key", + })), + }, +}); + +describe("removeAcmeCertificates", () => { + it("removes only the requested hosts", () => { + const store = buildStore(["a.example.com", "b.example.com"]); + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.removed).toEqual(["a.example.com"]); + expect(result.store.letsencrypt?.Certificates).toHaveLength(1); + expect(result.store.letsencrypt?.Certificates?.[0]?.domain.main).toBe( + "b.example.com", + ); + }); + + it("keeps the Account object intact", () => { + const store = buildStore(["a.example.com"]); + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.store.letsencrypt?.Account).toEqual({ + Email: "test@example.com", + }); + expect(result.store.letsencrypt?.Certificates).toEqual([]); + }); + + it("reports nothing removed when the host is absent", () => { + const store = buildStore(["a.example.com"]); + + const result = removeAcmeCertificates(store, ["other.example.com"]); + + expect(result.removed).toEqual([]); + expect(result.store.letsencrypt?.Certificates).toHaveLength(1); + }); + + it("tolerates a resolver with no Certificates array", () => { + const store: AcmeStore = { letsencrypt: { Account: {} } }; + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.removed).toEqual([]); + }); + + it("does not mutate the input store", () => { + const store = buildStore(["a.example.com"]); + + removeAcmeCertificates(store, ["a.example.com"]); + + expect(store.letsencrypt?.Certificates).toHaveLength(1); + }); +}); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index e81ebba2a6..994bae94dc 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -128,6 +128,7 @@ export * from "./utils/schedules/utils"; export * from "./utils/servers/remote-docker"; export * from "./utils/startup/cancel-deployments"; export * from "./utils/tracking/hubspot"; +export * from "./utils/traefik/acme"; export * from "./utils/traefik/application"; export * from "./utils/traefik/domain"; export * from "./utils/traefik/file-types"; diff --git a/packages/server/src/utils/traefik/acme.ts b/packages/server/src/utils/traefik/acme.ts new file mode 100644 index 0000000000..e15c5eada0 --- /dev/null +++ b/packages/server/src/utils/traefik/acme.ts @@ -0,0 +1,44 @@ +export interface AcmeCertificate { + domain: { main: string; sans?: string[] }; + certificate: string; + key: string; + Store?: string; +} + +export interface AcmeResolver { + Account?: unknown; + Certificates?: AcmeCertificate[] | null; +} + +export type AcmeStore = Record; + +/** + * Returns a copy of the ACME store with the certificates for the given hosts + * removed. The Account object of each resolver is preserved, so Traefik keeps + * its ACME registration. + */ +export const removeAcmeCertificates = ( + store: AcmeStore, + hosts: string[], +): { store: AcmeStore; removed: string[] } => { + const targets = new Set(hosts); + const removed: string[] = []; + const next: AcmeStore = {}; + + for (const [resolverName, resolver] of Object.entries(store)) { + const certificates = resolver.Certificates ?? []; + const kept: AcmeCertificate[] = []; + + for (const certificate of certificates) { + if (targets.has(certificate.domain.main)) { + removed.push(certificate.domain.main); + } else { + kept.push(certificate); + } + } + + next[resolverName] = { ...resolver, Certificates: kept }; + } + + return { store: next, removed }; +}; From ca1af290536d32200fe2adfd12137523cae4b8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 02:41:53 +0300 Subject: [PATCH 04/16] fix(traefik): remove the stale ACME certificate when a domain leaves Let's Encrypt --- apps/dokploy/__test__/traefik/acme.test.ts | 25 ++++++++ apps/dokploy/server/api/routers/domain.ts | 28 ++++++++- packages/server/src/services/domain.ts | 25 +++++++- packages/server/src/utils/traefik/acme.ts | 66 ++++++++++++++++++++++ 4 files changed, 142 insertions(+), 2 deletions(-) diff --git a/apps/dokploy/__test__/traefik/acme.test.ts b/apps/dokploy/__test__/traefik/acme.test.ts index 77c15f5462..9e2329ee64 100644 --- a/apps/dokploy/__test__/traefik/acme.test.ts +++ b/apps/dokploy/__test__/traefik/acme.test.ts @@ -62,3 +62,28 @@ describe("removeAcmeCertificates", () => { expect(store.letsencrypt?.Certificates).toHaveLength(1); }); }); + +describe("removeAcmeCertificates with multiple resolvers", () => { + it("only touches the resolver that holds the host", () => { + const store: AcmeStore = { + letsencrypt: { + Account: {}, + Certificates: [ + { domain: { main: "a.example.com" }, certificate: "c", key: "k" }, + ], + }, + other: { + Account: {}, + Certificates: [ + { domain: { main: "b.example.com" }, certificate: "c", key: "k" }, + ], + }, + }; + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.removed).toEqual(["a.example.com"]); + expect(result.store.letsencrypt?.Certificates).toEqual([]); + expect(result.store.other?.Certificates).toHaveLength(1); + }); +}); diff --git a/apps/dokploy/server/api/routers/domain.ts b/apps/dokploy/server/api/routers/domain.ts index 8210fcf8a5..ef6a1dfbda 100644 --- a/apps/dokploy/server/api/routers/domain.ts +++ b/apps/dokploy/server/api/routers/domain.ts @@ -1,5 +1,6 @@ import { createDomain, + type Domain, findApplicationById, findDomainById, findDomainsByApplicationId, @@ -8,7 +9,9 @@ import { findServerById, generateTraefikMeDomain, getWebServerSettings, + hasOtherLetsencryptDomainForHost, manageDomain, + purgeAcmeCertificates, removeDomain, removeDomainById, updateDomainById, @@ -31,6 +34,22 @@ import { apiUpdateDomain, } from "@/server/db/schema"; +const purgeStaleCertificate = async ( + domain: Domain, + serverId?: string | null, +): Promise => { + if (domain.certificateType === "letsencrypt") return false; + + const stillInUse = await hasOtherLetsencryptDomainForHost( + domain.host, + domain.domainId, + ); + if (stillInUse) return false; + + const removed = await purgeAcmeCertificates([domain.host], serverId); + return removed.length > 0; +}; + export const domainRouter = createTRPCRouter({ create: protectedProcedure .input(apiCreateDomain) @@ -126,9 +145,15 @@ export const domainRouter = createTRPCRouter({ resourceId: domain.domainId, resourceName: domain.host, }); + let traefikReloadRequired = false; + if (domain.applicationId) { const application = await findApplicationById(domain.applicationId); await manageDomain(application, domain); + traefikReloadRequired = await purgeStaleCertificate( + domain, + application.serverId, + ); } else if (domain.previewDeploymentId) { const previewDeployment = await findPreviewDeploymentById( domain.previewDeploymentId, @@ -139,7 +164,8 @@ export const domainRouter = createTRPCRouter({ application.appName = previewDeployment.appName; await manageDomain(application, domain); } - return result; + + return { ...result, traefikReloadRequired }; }), one: protectedProcedure.input(apiFindDomain).query(async ({ input, ctx }) => { const domain = await findDomainById(input.domainId); diff --git a/packages/server/src/services/domain.ts b/packages/server/src/services/domain.ts index c651af9fbb..6cc8e794c4 100644 --- a/packages/server/src/services/domain.ts +++ b/packages/server/src/services/domain.ts @@ -5,7 +5,7 @@ import { getWebServerSettings } from "@dokploy/server/services/web-server-settin import { generateRandomDomain } from "@dokploy/server/templates"; import { manageDomain } from "@dokploy/server/utils/traefik/domain"; import { TRPCError } from "@trpc/server"; -import { eq } from "drizzle-orm"; +import { and, eq, ne } from "drizzle-orm"; import type { z } from "zod"; import { type apiCreateDomain, domains } from "../db/schema"; import { findApplicationById } from "./application"; @@ -146,6 +146,29 @@ export const removeDomainById = async (domainId: string) => { return result[0]; }; +/** + * True when another domain record still serves this host with Let's Encrypt. + * Removing the shared certificate would break that domain. + */ +export const hasOtherLetsencryptDomainForHost = async ( + host: string, + excludeDomainId: string, +): Promise => { + const rows = await db + .select({ domainId: domains.domainId }) + .from(domains) + .where( + and( + eq(domains.host, host), + eq(domains.certificateType, "letsencrypt"), + ne(domains.domainId, excludeDomainId), + ), + ) + .limit(1); + + return rows.length > 0; +}; + export const getDomainHost = (domain: Domain) => { return `${domain.https ? "https" : "http"}://${domain.host}`; }; diff --git a/packages/server/src/utils/traefik/acme.ts b/packages/server/src/utils/traefik/acme.ts index e15c5eada0..66cfff6773 100644 --- a/packages/server/src/utils/traefik/acme.ts +++ b/packages/server/src/utils/traefik/acme.ts @@ -1,3 +1,9 @@ +import fs from "node:fs"; +import path from "node:path"; +import { paths } from "@dokploy/server/constants"; +import { encodeBase64 } from "../docker/utils"; +import { execAsyncRemote } from "../process/execAsync"; + export interface AcmeCertificate { domain: { main: string; sans?: string[] }; certificate: string; @@ -42,3 +48,63 @@ export const removeAcmeCertificates = ( return { store: next, removed }; }; + +const acmeJsonPath = (isRemote: boolean) => { + const { DYNAMIC_TRAEFIK_PATH } = paths(isRemote); + return path.join(DYNAMIC_TRAEFIK_PATH, "acme.json"); +}; + +/** + * Removes the ACME certificates for the given hosts and returns the hosts that + * were actually removed. The caller is responsible for reloading Traefik: + * acme.json is only read at startup, so an on-disk change has no effect until + * the container restarts. + */ +export const purgeAcmeCertificates = async ( + hosts: string[], + serverId?: string | null, +): Promise => { + if (hosts.length === 0) return []; + + const filePath = acmeJsonPath(!!serverId); + + let raw: string; + if (serverId) { + const { stdout } = await execAsyncRemote( + serverId, + `cat ${filePath} 2>/dev/null || true`, + ); + raw = stdout; + } else { + if (!fs.existsSync(filePath)) return []; + raw = fs.readFileSync(filePath, "utf8"); + } + + if (!raw.trim()) return []; + + let parsed: AcmeStore; + try { + parsed = JSON.parse(raw) as AcmeStore; + } catch { + // A malformed store is Traefik's to repair, not ours to overwrite. + return []; + } + + const { store, removed } = removeAcmeCertificates(parsed, hosts); + if (removed.length === 0) return []; + + const serialized = JSON.stringify(store, null, 2); + + if (serverId) { + // Traefik refuses to start if acme.json is more permissive than 0600. + await execAsyncRemote( + serverId, + `echo "${encodeBase64(serialized)}" | base64 -d > ${filePath}; chmod 600 ${filePath}`, + ); + } else { + fs.writeFileSync(filePath, serialized, "utf8"); + fs.chmodSync(filePath, 0o600); + } + + return removed; +}; From 1db7ddb3dc6db78d4c1cab274f2a2c4e638cb9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 02:48:22 +0300 Subject: [PATCH 05/16] fix(traefik): reconcile router TLS config for existing domains on startup --- .../__test__/traefik/reconciliation.test.ts | 51 +++++++++++ apps/dokploy/server/server.ts | 2 + packages/server/src/index.ts | 1 + packages/server/src/services/domain.ts | 19 +++- .../src/setup/domain-tls-reconciliation.ts | 91 +++++++++++++++++++ 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 apps/dokploy/__test__/traefik/reconciliation.test.ts create mode 100644 packages/server/src/setup/domain-tls-reconciliation.ts diff --git a/apps/dokploy/__test__/traefik/reconciliation.test.ts b/apps/dokploy/__test__/traefik/reconciliation.test.ts new file mode 100644 index 0000000000..210e9fbfe7 --- /dev/null +++ b/apps/dokploy/__test__/traefik/reconciliation.test.ts @@ -0,0 +1,51 @@ +import type { FileConfig } from "@dokploy/server"; +import { routerNeedsTlsFix } from "@dokploy/server"; +import { describe, expect, it } from "vitest"; + +const configWithRouter = ( + tls: Record | undefined, +): FileConfig => ({ + http: { + routers: { + "my-app-router-websecure-1": { + rule: "Host(`example.com`)", + service: "my-app-service-1", + entryPoints: ["websecure"], + ...(tls === undefined ? {} : { tls }), + }, + }, + services: {}, + }, +}); + +describe("routerNeedsTlsFix", () => { + it("is true when the websecure router has no tls key", () => { + expect(routerNeedsTlsFix(configWithRouter(undefined), "my-app", 1)).toBe( + true, + ); + }); + + it("is false when the router already has an empty tls block", () => { + expect(routerNeedsTlsFix(configWithRouter({}), "my-app", 1)).toBe(false); + }); + + it("is false when the router has a cert resolver", () => { + expect( + routerNeedsTlsFix( + configWithRouter({ certResolver: "letsencrypt" }), + "my-app", + 1, + ), + ).toBe(false); + }); + + it("is false when the router does not exist", () => { + expect(routerNeedsTlsFix(configWithRouter(undefined), "other-app", 1)).toBe( + false, + ); + }); + + it("is false for an empty config", () => { + expect(routerNeedsTlsFix({}, "my-app", 1)).toBe(false); + }); +}); diff --git a/apps/dokploy/server/server.ts b/apps/dokploy/server/server.ts index 5fe048c922..435ab06b17 100644 --- a/apps/dokploy/server/server.ts +++ b/apps/dokploy/server/server.ts @@ -6,6 +6,7 @@ import { IS_CLOUD, initCancelDeployments, initCronJobs, + initDomainTlsReconciliation, initEnterpriseBackupCronJobs, initializeNetwork, initSchedules, @@ -64,6 +65,7 @@ void app.prepare().then(async () => { await initCronJobs(); await initSchedules(); await initCancelDeployments(); + await initDomainTlsReconciliation(); await initVolumeBackupsCronJobs(); await sendDokployRestartNotifications(); } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 994bae94dc..10f71bbe1f 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -53,6 +53,7 @@ export * from "./services/user"; export * from "./services/volume-backups"; export * from "./services/web-server-settings"; export * from "./setup/config-paths"; +export * from "./setup/domain-tls-reconciliation"; export * from "./setup/forward-auth-setup"; export * from "./setup/monitoring-setup"; export * from "./setup/postgres-setup"; diff --git a/packages/server/src/services/domain.ts b/packages/server/src/services/domain.ts index 6cc8e794c4..c63b03bef0 100644 --- a/packages/server/src/services/domain.ts +++ b/packages/server/src/services/domain.ts @@ -5,7 +5,7 @@ import { getWebServerSettings } from "@dokploy/server/services/web-server-settin import { generateRandomDomain } from "@dokploy/server/templates"; import { manageDomain } from "@dokploy/server/utils/traefik/domain"; import { TRPCError } from "@trpc/server"; -import { and, eq, ne } from "drizzle-orm"; +import { and, eq, isNotNull, ne } from "drizzle-orm"; import type { z } from "zod"; import { type apiCreateDomain, domains } from "../db/schema"; import { findApplicationById } from "./application"; @@ -233,3 +233,20 @@ export const validateDomain = async ( }; } }; + +/** + * Domains that may still carry a router config written before the TLS + * override fix. Only application domains are returned; Compose domains are + * configured through Docker labels and are regenerated on deploy. + */ +export const findDomainsNeedingTlsReconciliation = async () => { + return await db + .select() + .from(domains) + .where( + and( + ne(domains.certificateType, "letsencrypt"), + isNotNull(domains.applicationId), + ), + ); +}; diff --git a/packages/server/src/setup/domain-tls-reconciliation.ts b/packages/server/src/setup/domain-tls-reconciliation.ts new file mode 100644 index 0000000000..4e84dd2244 --- /dev/null +++ b/packages/server/src/setup/domain-tls-reconciliation.ts @@ -0,0 +1,91 @@ +import { findApplicationById } from "../services/application"; +import { findDomainsNeedingTlsReconciliation } from "../services/domain"; +import { reloadDockerResource } from "../services/settings"; +import { purgeAcmeCertificates } from "../utils/traefik/acme"; +import { + loadOrCreateConfig, + loadOrCreateConfigRemote, +} from "../utils/traefik/application"; +import { manageDomain } from "../utils/traefik/domain"; +import type { FileConfig } from "../utils/traefik/file-types"; + +/** + * A websecure router written before the TLS override fix has no `tls` key at + * all, so Traefik applies the entrypoint's certResolver default to it. + */ +export const routerNeedsTlsFix = ( + config: FileConfig, + appName: string, + uniqueConfigKey: number, +): boolean => { + const router = + config.http?.routers?.[`${appName}-router-websecure-${uniqueConfigKey}`]; + if (!router) return false; + return router.tls === undefined; +}; + +/** + * One-shot pass that regenerates router configs written before the fix. + * Idempotent: once a router carries `tls: {}` it is skipped on later starts. + */ +export const initDomainTlsReconciliation = async () => { + const domains = await findDomainsNeedingTlsReconciliation(); + if (domains.length === 0) return; + + const reloadTargets = new Set(); + + const byApplication = new Map(); + for (const domain of domains) { + if (!domain.applicationId) continue; + const bucket = byApplication.get(domain.applicationId) ?? []; + bucket.push(domain); + byApplication.set(domain.applicationId, bucket); + } + + for (const [applicationId, appDomains] of byApplication) { + try { + const application = await findApplicationById(applicationId); + const config = application.serverId + ? await loadOrCreateConfigRemote( + application.serverId, + application.appName, + ) + : loadOrCreateConfig(application.appName); + + const stale = appDomains.filter((domain) => + routerNeedsTlsFix(config, application.appName, domain.uniqueConfigKey), + ); + if (stale.length === 0) continue; + + for (const domain of stale) { + await manageDomain(application, domain); + } + + const removed = await purgeAcmeCertificates( + stale.map((domain) => domain.host), + application.serverId, + ); + if (removed.length > 0) { + reloadTargets.add(application.serverId ?? ""); + } + + console.log( + `Reconciled TLS config for ${stale.length} domain(s) on ${application.appName}`, + ); + } catch (error) { + // One unreachable remote server must not stop the rest. + console.error( + `TLS reconciliation failed for application ${applicationId}:`, + error, + ); + } + } + + for (const serverId of reloadTargets) { + try { + await reloadDockerResource("dokploy-traefik", serverId || undefined); + } catch (error) { + console.error("TLS reconciliation could not reload Traefik:", error); + } + } +}; From 6c2f6466d1ec902eb2d1ad590f3487da3998f867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 02:54:59 +0300 Subject: [PATCH 06/16] test(traefik): cover startup TLS reconciliation behaviour --- .../__test__/traefik/reconciliation.test.ts | 273 +++++++++++++++++- 1 file changed, 270 insertions(+), 3 deletions(-) diff --git a/apps/dokploy/__test__/traefik/reconciliation.test.ts b/apps/dokploy/__test__/traefik/reconciliation.test.ts index 210e9fbfe7..e8c461ba2e 100644 --- a/apps/dokploy/__test__/traefik/reconciliation.test.ts +++ b/apps/dokploy/__test__/traefik/reconciliation.test.ts @@ -1,6 +1,83 @@ -import type { FileConfig } from "@dokploy/server"; -import { routerNeedsTlsFix } from "@dokploy/server"; -import { describe, expect, it } from "vitest"; +import type { Domain, FileConfig } from "@dokploy/server"; +import { + initDomainTlsReconciliation, + routerNeedsTlsFix, +} from "@dokploy/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const findDomainsNeedingTlsReconciliationMock = vi.fn(); +vi.mock("@dokploy/server/services/domain", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/domain") + >("@dokploy/server/services/domain"); + return { + ...actual, + findDomainsNeedingTlsReconciliation: () => + findDomainsNeedingTlsReconciliationMock(), + }; +}); + +const findApplicationByIdMock = vi.fn(); +vi.mock("@dokploy/server/services/application", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/application") + >("@dokploy/server/services/application"); + return { + ...actual, + findApplicationById: (applicationId: string) => + findApplicationByIdMock(applicationId), + }; +}); + +const reloadDockerResourceMock = vi.fn(); +vi.mock("@dokploy/server/services/settings", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/settings") + >("@dokploy/server/services/settings"); + return { + ...actual, + reloadDockerResource: (resourceName: string, serverId?: string) => + reloadDockerResourceMock(resourceName, serverId), + }; +}); + +const purgeAcmeCertificatesMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/acme", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/acme") + >("@dokploy/server/utils/traefik/acme"); + return { + ...actual, + purgeAcmeCertificates: (hosts: string[], serverId?: string | null) => + purgeAcmeCertificatesMock(hosts, serverId), + }; +}); + +const loadOrCreateConfigMock = vi.fn(); +const loadOrCreateConfigRemoteMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/application", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/application") + >("@dokploy/server/utils/traefik/application"); + return { + ...actual, + loadOrCreateConfig: (appName: string) => loadOrCreateConfigMock(appName), + loadOrCreateConfigRemote: (serverId: string, appName: string) => + loadOrCreateConfigRemoteMock(serverId, appName), + }; +}); + +const manageDomainMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/domain", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/domain") + >("@dokploy/server/utils/traefik/domain"); + return { + ...actual, + manageDomain: (app: unknown, domain: unknown) => + manageDomainMock(app, domain), + }; +}); const configWithRouter = ( tls: Record | undefined, @@ -49,3 +126,193 @@ describe("routerNeedsTlsFix", () => { expect(routerNeedsTlsFix({}, "my-app", 1)).toBe(false); }); }); + +const buildDomain = (overrides: Partial): Domain => + ({ + domainId: overrides.host ?? "domain-id", + host: "example.com", + https: false, + port: 3000, + customEntrypoint: null, + path: "/", + serviceName: null, + domainType: "application", + uniqueConfigKey: 1, + createdAt: new Date().toISOString(), + composeId: null, + customCertResolver: null, + applicationId: "app-1", + previewDeploymentId: null, + certificateType: "none", + internalPath: "/", + stripPath: false, + middlewares: [], + forwardAuthEnabled: false, + ...overrides, + }) as Domain; + +const routerConfigFor = ( + appName: string, + keys: number[], + tlsByKey: Record | undefined>, +): FileConfig => ({ + http: { + routers: Object.fromEntries( + keys.map((key) => [ + `${appName}-router-websecure-${key}`, + { + rule: "Host(`example.com`)", + service: `${appName}-service-${key}`, + entryPoints: ["websecure"], + ...(tlsByKey[key] === undefined ? {} : { tls: tlsByKey[key] }), + }, + ]), + ), + services: {}, + }, +}); + +describe("initDomainTlsReconciliation", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + it("skips applications already fixed, regenerates once per stale domain but loads config once per app, reloads at most once per server only when a certificate was removed, and isolates a failing application from the rest", async () => { + // app-1: local, two stale domains sharing one router config load. + const appOneDomainA = buildDomain({ + domainId: "a1", + host: "a1.example.com", + applicationId: "app-1", + uniqueConfigKey: 1, + }); + const appOneDomainB = buildDomain({ + domainId: "a2", + host: "a2.example.com", + applicationId: "app-1", + uniqueConfigKey: 2, + }); + // app-2: local, router already carries `tls: {}` -> nothing to do. + const appTwoDomain = buildDomain({ + domainId: "b1", + host: "b1.example.com", + applicationId: "app-2", + uniqueConfigKey: 1, + }); + // app-3: remote, config load throws -> must not block the others. + const appThreeDomain = buildDomain({ + domainId: "c1", + host: "c1.example.com", + applicationId: "app-3", + uniqueConfigKey: 1, + }); + // app-4: remote, one stale domain but no certificate actually removed. + const appFourDomain = buildDomain({ + domainId: "d1", + host: "d1.example.com", + applicationId: "app-4", + uniqueConfigKey: 1, + }); + + findDomainsNeedingTlsReconciliationMock.mockResolvedValue([ + appThreeDomain, + appOneDomainA, + appOneDomainB, + appTwoDomain, + appFourDomain, + ]); + + const applicationsById: Record = { + "app-1": { applicationId: "app-1", appName: "app-one", serverId: null }, + "app-2": { applicationId: "app-2", appName: "app-two", serverId: null }, + "app-3": { + applicationId: "app-3", + appName: "app-three", + serverId: "server-x", + }, + "app-4": { + applicationId: "app-4", + appName: "app-four", + serverId: "server-y", + }, + }; + findApplicationByIdMock.mockImplementation( + async (applicationId: string) => applicationsById[applicationId], + ); + + loadOrCreateConfigMock.mockImplementation((appName: string) => { + if (appName === "app-one") { + return routerConfigFor("app-one", [1, 2], { + 1: undefined, + 2: undefined, + }); + } + if (appName === "app-two") { + return routerConfigFor("app-two", [1], { 1: {} }); + } + throw new Error(`unexpected local config load for ${appName}`); + }); + + loadOrCreateConfigRemoteMock.mockImplementation( + async (_serverId: string, appName: string) => { + if (appName === "app-three") { + throw new Error("ssh connection refused"); + } + if (appName === "app-four") { + return routerConfigFor("app-four", [1], { 1: undefined }); + } + throw new Error(`unexpected remote config load for ${appName}`); + }, + ); + + manageDomainMock.mockResolvedValue(undefined); + + purgeAcmeCertificatesMock.mockImplementation( + async (hosts: string[], serverId?: string | null) => { + // app-4's server reports nothing was actually removed. + if (serverId === "server-y") return []; + return hosts; + }, + ); + + reloadDockerResourceMock.mockResolvedValue(undefined); + + await initDomainTlsReconciliation(); + + // (a) app-2's router already has `tls: {}` -> zero regeneration calls for it. + const manageDomainHosts = manageDomainMock.mock.calls.map( + (call) => (call[1] as Domain).host, + ); + expect(manageDomainHosts).not.toContain("b1.example.com"); + + // (b) app-1 has two stale domains -> regeneration invoked once per + // stale domain, but the config is loaded only once for that app. + expect(manageDomainHosts.filter((h) => h.startsWith("a"))).toEqual([ + "a1.example.com", + "a2.example.com", + ]); + expect( + loadOrCreateConfigMock.mock.calls.filter((call) => call[0] === "app-one"), + ).toHaveLength(1); + + // isolation: app-3 threw while loading its config, but app-1 (and + // app-4) were still processed and app-3 never reached manageDomain. + expect(manageDomainHosts).not.toContain("c1.example.com"); + expect(manageDomainHosts).toContain("d1.example.com"); + + // reload: requested at most once per server, only when a + // certificate was actually removed. app-1 (local, serverId + // undefined) had removals -> one reload. app-4's server reported no + // removals -> no reload for "server-y". + expect(reloadDockerResourceMock).toHaveBeenCalledTimes(1); + expect(reloadDockerResourceMock).toHaveBeenCalledWith( + "dokploy-traefik", + undefined, + ); + expect(reloadDockerResourceMock).not.toHaveBeenCalledWith( + "dokploy-traefik", + "server-y", + ); + }); +}); From 2b6e3dbeb399353c57d897e35404320e6ceb44c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 03:10:51 +0300 Subject: [PATCH 07/16] fix(traefik): write acme.json atomically when purging certificates The store was rewritten in place on both the local and the remote path. The remote command truncated acme.json with `>` before writing and used `;` to chain chmod, so a dropped SSH stream, a failing base64 or a full disk left an empty, world-readable store behind and destroyed every Let's Encrypt certificate on that server. `fs.writeFileSync` had the same truncate-in-place exposure. Both paths now write a sibling temp file, set mode 0600 on it and rename it over the target, which is atomic within a directory. The remote commands are chained with `&&` so a failure stops the sequence, and the temp file is removed on failure. --- packages/server/src/utils/traefik/acme.ts | 25 +++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/server/src/utils/traefik/acme.ts b/packages/server/src/utils/traefik/acme.ts index 66cfff6773..d11737ccfd 100644 --- a/packages/server/src/utils/traefik/acme.ts +++ b/packages/server/src/utils/traefik/acme.ts @@ -95,15 +95,32 @@ export const purgeAcmeCertificates = async ( const serialized = JSON.stringify(store, null, 2); + // Never write in place: a truncated acme.json costs every Let's Encrypt + // certificate on the server. Write a sibling temp file, lock it down to + // 0600 (Traefik refuses to start on anything more permissive) and rename + // it over the target, which is atomic within the same directory. + const tempPath = `${filePath}.dokploy.tmp`; + if (serverId) { - // Traefik refuses to start if acme.json is more permissive than 0600. await execAsyncRemote( serverId, - `echo "${encodeBase64(serialized)}" | base64 -d > ${filePath}; chmod 600 ${filePath}`, + `umask 077 && printf '%s' "${encodeBase64(serialized)}" | base64 -d > ${tempPath} && chmod 600 ${tempPath} && mv -f ${tempPath} ${filePath} || { rm -f ${tempPath}; exit 1; }`, ); } else { - fs.writeFileSync(filePath, serialized, "utf8"); - fs.chmodSync(filePath, 0o600); + try { + fs.writeFileSync(tempPath, serialized, { encoding: "utf8", mode: 0o600 }); + // writeFileSync's mode is subject to the process umask, and it is + // ignored entirely when the temp file already exists. + fs.chmodSync(tempPath, 0o600); + fs.renameSync(tempPath, filePath); + } catch (error) { + try { + fs.unlinkSync(tempPath); + } catch { + // The temp file may never have been created. + } + throw error; + } } return removed; From efc8103f0f0368e9c1e4350ce388aa6e0bbc367c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 03:10:58 +0300 Subject: [PATCH 08/16] refactor(traefik): move purgeStaleCertificate into the server package The helper lived inside the tRPC router, so the shared-host guard, the only thing preventing the deletion of a certificate another domain still needs, had no test coverage. It now lives next to the domain services it uses and is exported, with unit tests for the purge, the guard, the letsencrypt short circuit and the failure path. A purge failure no longer fails the mutation either: the domain row was already updated and the router already regenerated, so an unreachable server made the user see an error for an update that succeeded. Failures are logged and reported as "no reload required". --- .../traefik/domain-certificate.test.ts | 100 ++++++++++++++++++ apps/dokploy/server/api/routers/domain.ts | 20 +--- packages/server/src/index.ts | 1 + .../server/src/services/domain-certificate.ts | 37 +++++++ 4 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 apps/dokploy/__test__/traefik/domain-certificate.test.ts create mode 100644 packages/server/src/services/domain-certificate.ts diff --git a/apps/dokploy/__test__/traefik/domain-certificate.test.ts b/apps/dokploy/__test__/traefik/domain-certificate.test.ts new file mode 100644 index 0000000000..d0e2bd0197 --- /dev/null +++ b/apps/dokploy/__test__/traefik/domain-certificate.test.ts @@ -0,0 +1,100 @@ +import type { Domain } from "@dokploy/server"; +import { purgeStaleCertificate } from "@dokploy/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Stands in for the `domains` lookup that answers "is this host still served +// by another Let's Encrypt domain?". +const hasOtherLetsencryptDomainForHostMock = vi.fn(); +vi.mock("@dokploy/server/services/domain", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/domain") + >("@dokploy/server/services/domain"); + return { + ...actual, + hasOtherLetsencryptDomainForHost: (host: string, excludeId: string) => + hasOtherLetsencryptDomainForHostMock(host, excludeId), + }; +}); + +const purgeAcmeCertificatesMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/acme", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/acme") + >("@dokploy/server/utils/traefik/acme"); + return { + ...actual, + purgeAcmeCertificates: (hosts: string[], serverId?: string | null) => + purgeAcmeCertificatesMock(hosts, serverId), + }; +}); + +const buildDomain = (overrides: Partial = {}): Domain => + ({ + domainId: "domain-1", + host: "example.com", + certificateType: "none", + applicationId: "app-1", + uniqueConfigKey: 1, + ...overrides, + }) as Domain; + +describe("purgeStaleCertificate", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + it("purges the host when no other Let's Encrypt domain uses it", async () => { + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(false); + purgeAcmeCertificatesMock.mockResolvedValue(["example.com"]); + + const result = await purgeStaleCertificate(buildDomain(), "server-1"); + + expect(purgeAcmeCertificatesMock).toHaveBeenCalledWith( + ["example.com"], + "server-1", + ); + expect(result).toBe(true); + }); + + it("keeps the host when another Let's Encrypt domain still uses it", async () => { + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(true); + + const result = await purgeStaleCertificate(buildDomain(), "server-1"); + + expect(purgeAcmeCertificatesMock).not.toHaveBeenCalled(); + expect(result).toBe(false); + }); + + it("never purges a domain that still uses Let's Encrypt", async () => { + const result = await purgeStaleCertificate( + buildDomain({ certificateType: "letsencrypt" }), + "server-1", + ); + + expect(hasOtherLetsencryptDomainForHostMock).not.toHaveBeenCalled(); + expect(purgeAcmeCertificatesMock).not.toHaveBeenCalled(); + expect(result).toBe(false); + }); + + it("reports no reload instead of throwing when the purge fails", async () => { + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(false); + purgeAcmeCertificatesMock.mockRejectedValue( + new Error("ssh connection refused"), + ); + + await expect( + purgeStaleCertificate(buildDomain(), "server-1"), + ).resolves.toBe(false); + }); + + it("reports no reload instead of throwing when the lookup fails", async () => { + hasOtherLetsencryptDomainForHostMock.mockRejectedValue( + new Error("database unavailable"), + ); + + await expect( + purgeStaleCertificate(buildDomain(), "server-1"), + ).resolves.toBe(false); + }); +}); diff --git a/apps/dokploy/server/api/routers/domain.ts b/apps/dokploy/server/api/routers/domain.ts index ef6a1dfbda..779ee23c8b 100644 --- a/apps/dokploy/server/api/routers/domain.ts +++ b/apps/dokploy/server/api/routers/domain.ts @@ -1,6 +1,5 @@ import { createDomain, - type Domain, findApplicationById, findDomainById, findDomainsByApplicationId, @@ -9,9 +8,8 @@ import { findServerById, generateTraefikMeDomain, getWebServerSettings, - hasOtherLetsencryptDomainForHost, manageDomain, - purgeAcmeCertificates, + purgeStaleCertificate, removeDomain, removeDomainById, updateDomainById, @@ -34,22 +32,6 @@ import { apiUpdateDomain, } from "@/server/db/schema"; -const purgeStaleCertificate = async ( - domain: Domain, - serverId?: string | null, -): Promise => { - if (domain.certificateType === "letsencrypt") return false; - - const stillInUse = await hasOtherLetsencryptDomainForHost( - domain.host, - domain.domainId, - ); - if (stillInUse) return false; - - const removed = await purgeAcmeCertificates([domain.host], serverId); - return removed.length > 0; -}; - export const domainRouter = createTRPCRouter({ create: protectedProcedure .input(apiCreateDomain) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 10f71bbe1f..efe4c00147 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -18,6 +18,7 @@ export * from "./services/deployment"; export * from "./services/destination"; export * from "./services/docker"; export * from "./services/domain"; +export * from "./services/domain-certificate"; export * from "./services/environment"; export * from "./services/git-provider"; export * from "./services/gitea"; diff --git a/packages/server/src/services/domain-certificate.ts b/packages/server/src/services/domain-certificate.ts new file mode 100644 index 0000000000..a752dcc01f --- /dev/null +++ b/packages/server/src/services/domain-certificate.ts @@ -0,0 +1,37 @@ +import { purgeAcmeCertificates } from "../utils/traefik/acme"; +import { type Domain, hasOtherLetsencryptDomainForHost } from "./domain"; + +/** + * Removes the Let's Encrypt certificate a domain no longer needs after it moved + * away from the `letsencrypt` provider, and reports whether Traefik has to be + * reloaded for the change to take effect. + * + * The host is kept when another domain record still serves it with + * `certificateType: "letsencrypt"`, otherwise that domain would lose its + * certificate. Failures are logged and reported as "no reload required": the + * domain itself was already updated, so the caller must not fail because of a + * best-effort cleanup. + */ +export const purgeStaleCertificate = async ( + domain: Domain, + serverId?: string | null, +): Promise => { + if (domain.certificateType === "letsencrypt") return false; + + try { + const stillInUse = await hasOtherLetsencryptDomainForHost( + domain.host, + domain.domainId, + ); + if (stillInUse) return false; + + const removed = await purgeAcmeCertificates([domain.host], serverId); + return removed.length > 0; + } catch (error) { + console.error( + `Could not purge the stale ACME certificate for ${domain.host}:`, + error, + ); + return false; + } +}; From c72f65231a7791b160f60722ddd3dfda4de0f92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 03:11:07 +0300 Subject: [PATCH 09/16] fix(traefik): guard and reorder the startup TLS reconciliation The startup pass purged ACME certificates without the shared-host guard the mutation path applies, so a host still served by another domain with certificateType "letsencrypt" lost its certificate at boot and Traefik was restarted on top of it. It now reuses hasOtherLetsencryptDomainForHost and only purges genuinely unused hosts. The purge also ran after manageDomain. Once the router carries its `tls` key, routerNeedsTlsFix is false, so a purge that threw was never retried on a later boot. Purging first leaves the router untouched on failure and the next boot retries the whole domain. --- .../src/setup/domain-tls-reconciliation.ts | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/server/src/setup/domain-tls-reconciliation.ts b/packages/server/src/setup/domain-tls-reconciliation.ts index 4e84dd2244..f547209d59 100644 --- a/packages/server/src/setup/domain-tls-reconciliation.ts +++ b/packages/server/src/setup/domain-tls-reconciliation.ts @@ -1,5 +1,8 @@ import { findApplicationById } from "../services/application"; -import { findDomainsNeedingTlsReconciliation } from "../services/domain"; +import { + findDomainsNeedingTlsReconciliation, + hasOtherLetsencryptDomainForHost, +} from "../services/domain"; import { reloadDockerResource } from "../services/settings"; import { purgeAcmeCertificates } from "../utils/traefik/acme"; import { @@ -57,18 +60,32 @@ export const initDomainTlsReconciliation = async () => { ); if (stale.length === 0) continue; + // A host still served by another Let's Encrypt domain keeps its + // certificate, exactly as the mutation path does. + const purgeableHosts: string[] = []; for (const domain of stale) { - await manageDomain(application, domain); + const stillInUse = await hasOtherLetsencryptDomainForHost( + domain.host, + domain.domainId, + ); + if (!stillInUse) purgeableHosts.push(domain.host); } - const removed = await purgeAcmeCertificates( - stale.map((domain) => domain.host), - application.serverId, - ); + // Purge before regenerating: once the router carries its `tls` key + // `routerNeedsTlsFix` is false, so a purge that failed afterwards + // would never be retried on a later boot. + const removed = + purgeableHosts.length > 0 + ? await purgeAcmeCertificates(purgeableHosts, application.serverId) + : []; if (removed.length > 0) { reloadTargets.add(application.serverId ?? ""); } + for (const domain of stale) { + await manageDomain(application, domain); + } + console.log( `Reconciled TLS config for ${stale.length} domain(s) on ${application.appName}`, ); From a1b20e5665c945214d27350b040816a4abe7fc59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 03:11:07 +0300 Subject: [PATCH 10/16] feat(ui): tell the user when Traefik must be restarted after a cert change The domain update mutation already returned traefikReloadRequired but no caller read it. Traefik only reads acme.json at startup, so without a restart the purge has no effect and can be undone when Traefik next rewrites the store from memory. The dialog now raises a toast when the field is true. --- .../dashboard/application/domains/handle-domain.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx b/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx index be8901921f..d3a29fd3c5 100644 --- a/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx +++ b/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx @@ -299,9 +299,19 @@ export const AddDomain = ({ id, type, domainId = "", children }: Props) => { ...data, customEntrypoint: data.useCustomEntrypoint ? data.customEntrypoint : null, }) - .then(async () => { + .then(async (result) => { toast.success(dictionary.success); + if ( + result && + "traefikReloadRequired" in result && + result.traefikReloadRequired + ) { + toast.info( + "Restart Traefik to apply the certificate change for this domain.", + ); + } + if (data.domainType === "application") { await utils.domain.byApplicationId.invalidate({ applicationId: id, From 1348e921c1ada132c1338f05993cead4c3041f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 03:11:07 +0300 Subject: [PATCH 11/16] test(traefik): split the reconciliation test per behaviour The four reconciliation behaviours were asserted inside one large `it` with a sentence-long title, so a failure said nothing about which one broke. They are now separate cases over a shared beforeEach fixture, plus coverage for the shared-host guard and the purge-before-regenerate ordering. --- .../__test__/traefik/reconciliation.test.ts | 176 +++++++++++------- 1 file changed, 109 insertions(+), 67 deletions(-) diff --git a/apps/dokploy/__test__/traefik/reconciliation.test.ts b/apps/dokploy/__test__/traefik/reconciliation.test.ts index e8c461ba2e..e771b132c1 100644 --- a/apps/dokploy/__test__/traefik/reconciliation.test.ts +++ b/apps/dokploy/__test__/traefik/reconciliation.test.ts @@ -6,6 +6,7 @@ import { import { beforeEach, describe, expect, it, vi } from "vitest"; const findDomainsNeedingTlsReconciliationMock = vi.fn(); +const hasOtherLetsencryptDomainForHostMock = vi.fn(); vi.mock("@dokploy/server/services/domain", async () => { const actual = await vi.importActual< typeof import("@dokploy/server/services/domain") @@ -14,6 +15,8 @@ vi.mock("@dokploy/server/services/domain", async () => { ...actual, findDomainsNeedingTlsReconciliation: () => findDomainsNeedingTlsReconciliationMock(), + hasOtherLetsencryptDomainForHost: (host: string, excludeId: string) => + hasOtherLetsencryptDomainForHostMock(host, excludeId), }; }); @@ -173,47 +176,64 @@ const routerConfigFor = ( }); describe("initDomainTlsReconciliation", () => { - beforeEach(() => { - vi.clearAllMocks(); - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); + // Shared fixture: + // app-1: local, two stale domains sharing one router config load. + // app-2: local, router already carries `tls: {}` -> nothing to do. + // app-3: remote, config load throws -> must not block the others. + // app-4: remote, one stale domain but no certificate actually removed. + const appOneDomainA = buildDomain({ + domainId: "a1", + host: "a1.example.com", + applicationId: "app-1", + uniqueConfigKey: 1, + }); + const appOneDomainB = buildDomain({ + domainId: "a2", + host: "a2.example.com", + applicationId: "app-1", + uniqueConfigKey: 2, + }); + const appTwoDomain = buildDomain({ + domainId: "b1", + host: "b1.example.com", + applicationId: "app-2", + uniqueConfigKey: 1, + }); + const appThreeDomain = buildDomain({ + domainId: "c1", + host: "c1.example.com", + applicationId: "app-3", + uniqueConfigKey: 1, + }); + const appFourDomain = buildDomain({ + domainId: "d1", + host: "d1.example.com", + applicationId: "app-4", + uniqueConfigKey: 1, }); - it("skips applications already fixed, regenerates once per stale domain but loads config once per app, reloads at most once per server only when a certificate was removed, and isolates a failing application from the rest", async () => { - // app-1: local, two stale domains sharing one router config load. - const appOneDomainA = buildDomain({ - domainId: "a1", - host: "a1.example.com", - applicationId: "app-1", - uniqueConfigKey: 1, - }); - const appOneDomainB = buildDomain({ - domainId: "a2", - host: "a2.example.com", - applicationId: "app-1", - uniqueConfigKey: 2, - }); - // app-2: local, router already carries `tls: {}` -> nothing to do. - const appTwoDomain = buildDomain({ - domainId: "b1", - host: "b1.example.com", - applicationId: "app-2", - uniqueConfigKey: 1, - }); - // app-3: remote, config load throws -> must not block the others. - const appThreeDomain = buildDomain({ - domainId: "c1", - host: "c1.example.com", + const applicationsById: Record = { + "app-1": { applicationId: "app-1", appName: "app-one", serverId: null }, + "app-2": { applicationId: "app-2", appName: "app-two", serverId: null }, + "app-3": { applicationId: "app-3", - uniqueConfigKey: 1, - }); - // app-4: remote, one stale domain but no certificate actually removed. - const appFourDomain = buildDomain({ - domainId: "d1", - host: "d1.example.com", + appName: "app-three", + serverId: "server-x", + }, + "app-4": { applicationId: "app-4", - uniqueConfigKey: 1, - }); + appName: "app-four", + serverId: "server-y", + }, + }; + + const managedHosts = () => + manageDomainMock.mock.calls.map((call) => (call[1] as Domain).host); + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); findDomainsNeedingTlsReconciliationMock.mockResolvedValue([ appThreeDomain, @@ -223,20 +243,6 @@ describe("initDomainTlsReconciliation", () => { appFourDomain, ]); - const applicationsById: Record = { - "app-1": { applicationId: "app-1", appName: "app-one", serverId: null }, - "app-2": { applicationId: "app-2", appName: "app-two", serverId: null }, - "app-3": { - applicationId: "app-3", - appName: "app-three", - serverId: "server-x", - }, - "app-4": { - applicationId: "app-4", - appName: "app-four", - serverId: "server-y", - }, - }; findApplicationByIdMock.mockImplementation( async (applicationId: string) => applicationsById[applicationId], ); @@ -268,6 +274,9 @@ describe("initDomainTlsReconciliation", () => { manageDomainMock.mockResolvedValue(undefined); + // No host is shared with another Let's Encrypt domain by default. + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(false); + purgeAcmeCertificatesMock.mockImplementation( async (hosts: string[], serverId?: string | null) => { // app-4's server reports nothing was actually removed. @@ -277,34 +286,31 @@ describe("initDomainTlsReconciliation", () => { ); reloadDockerResourceMock.mockResolvedValue(undefined); + }); + it("does not regenerate an application whose router is already fixed", async () => { await initDomainTlsReconciliation(); - // (a) app-2's router already has `tls: {}` -> zero regeneration calls for it. - const manageDomainHosts = manageDomainMock.mock.calls.map( - (call) => (call[1] as Domain).host, - ); - expect(manageDomainHosts).not.toContain("b1.example.com"); + expect(managedHosts()).not.toContain("b1.example.com"); + }); - // (b) app-1 has two stale domains -> regeneration invoked once per - // stale domain, but the config is loaded only once for that app. - expect(manageDomainHosts.filter((h) => h.startsWith("a"))).toEqual([ + it("regenerates once per stale domain while loading the application config once", async () => { + await initDomainTlsReconciliation(); + + expect(managedHosts().filter((host) => host.startsWith("a"))).toEqual([ "a1.example.com", "a2.example.com", ]); expect( loadOrCreateConfigMock.mock.calls.filter((call) => call[0] === "app-one"), ).toHaveLength(1); + }); - // isolation: app-3 threw while loading its config, but app-1 (and - // app-4) were still processed and app-3 never reached manageDomain. - expect(manageDomainHosts).not.toContain("c1.example.com"); - expect(manageDomainHosts).toContain("d1.example.com"); + it("requests a reload at most once per server and only when a certificate was removed", async () => { + await initDomainTlsReconciliation(); - // reload: requested at most once per server, only when a - // certificate was actually removed. app-1 (local, serverId - // undefined) had removals -> one reload. app-4's server reported no - // removals -> no reload for "server-y". + // app-1 (local, serverId undefined) had removals -> exactly one + // reload. app-4's server reported no removals -> no reload at all. expect(reloadDockerResourceMock).toHaveBeenCalledTimes(1); expect(reloadDockerResourceMock).toHaveBeenCalledWith( "dokploy-traefik", @@ -315,4 +321,40 @@ describe("initDomainTlsReconciliation", () => { "server-y", ); }); + + it("keeps processing the other applications when one fails", async () => { + await initDomainTlsReconciliation(); + + // app-3 threw while loading its config and never reached + // manageDomain, but app-1 and app-4 were still processed. + expect(managedHosts()).not.toContain("c1.example.com"); + expect(managedHosts()).toContain("a1.example.com"); + expect(managedHosts()).toContain("d1.example.com"); + }); + + it("keeps the certificate of a host another Let's Encrypt domain still uses", async () => { + hasOtherLetsencryptDomainForHostMock.mockImplementation( + async (host: string) => host === "a1.example.com", + ); + + await initDomainTlsReconciliation(); + + const purgedHosts = purgeAcmeCertificatesMock.mock.calls.flatMap( + (call) => call[0] as string[], + ); + expect(purgedHosts).not.toContain("a1.example.com"); + expect(purgedHosts).toContain("a2.example.com"); + // The router is still regenerated, only the certificate is spared. + expect(managedHosts()).toContain("a1.example.com"); + }); + + it("purges the stale certificate before regenerating the router", async () => { + await initDomainTlsReconciliation(); + + const firstPurge = + purgeAcmeCertificatesMock.mock.invocationCallOrder[0] ?? Number.NaN; + const firstManage = + manageDomainMock.mock.invocationCallOrder[0] ?? Number.NaN; + expect(firstPurge).toBeLessThan(firstManage); + }); }); From 40816d3ee5ae29db2717535f680cca16ccd9bc14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 03:49:48 +0300 Subject: [PATCH 12/16] fix(ui): show the Traefik restart hint inside the success toast Raising it as a second toast collapsed it onto the success one, since the Toaster does not set expand, and no other place in the app fires two toasts for a single action. It is now the success toast's description, with a longer duration because the message asks the user to go and restart Traefik rather than just confirming what happened. --- .../application/domains/handle-domain.tsx | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx b/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx index d3a29fd3c5..663ea36889 100644 --- a/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx +++ b/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx @@ -300,17 +300,20 @@ export const AddDomain = ({ id, type, domainId = "", children }: Props) => { customEntrypoint: data.useCustomEntrypoint ? data.customEntrypoint : null, }) .then(async (result) => { - toast.success(dictionary.success); - - if ( + const traefikReloadRequired = result && "traefikReloadRequired" in result && - result.traefikReloadRequired - ) { - toast.info( - "Restart Traefik to apply the certificate change for this domain.", - ); - } + result.traefikReloadRequired; + + toast.success(dictionary.success, { + ...(traefikReloadRequired && { + description: + "Restart Traefik to apply the certificate change for this domain.", + // The default duration is tuned for one-line confirmations; this + // one asks the user to go and do something, so give them longer. + duration: 10000, + }), + }); if (data.domainType === "application") { await utils.domain.byApplicationId.invalidate({ From 2f47efd18bbc9d7d4adb2e1490d4a498c0bd0b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 04:05:26 +0300 Subject: [PATCH 13/16] fix(traefik): serialise ACME purges per server A purge is a read-modify-write over a file Traefik owns. On a remote server the read is an SSH round trip, so two domains switched off Let's Encrypt at the same time both read the original store and the later write puts back whatever the earlier one removed, leaving a stale certificate that keeps shadowing the Origin CA one. Purges for a server now run one at a time. The chain survives a failed purge so one broken SSH connection cannot block later ones. --- .../__test__/traefik/acme-purge.test.ts | 125 ++++++++++++++++++ packages/server/src/utils/traefik/acme.ts | 33 +++++ 2 files changed, 158 insertions(+) create mode 100644 apps/dokploy/__test__/traefik/acme-purge.test.ts diff --git a/apps/dokploy/__test__/traefik/acme-purge.test.ts b/apps/dokploy/__test__/traefik/acme-purge.test.ts new file mode 100644 index 0000000000..cda546974a --- /dev/null +++ b/apps/dokploy/__test__/traefik/acme-purge.test.ts @@ -0,0 +1,125 @@ +import { purgeAcmeCertificates } from "@dokploy/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const execAsyncRemoteMock = vi.fn(); +vi.mock("@dokploy/server/utils/process/execAsync", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/process/execAsync") + >("@dokploy/server/utils/process/execAsync"); + return { + ...actual, + execAsyncRemote: (serverId: string, command: string) => + execAsyncRemoteMock(serverId, command), + }; +}); + +const certificatesFor = (mains: string[]) => + JSON.stringify({ + letsencrypt: { + Account: { Email: "test@localhost.com" }, + Certificates: mains.map((main) => ({ + domain: { main }, + certificate: "cert", + key: "key", + })), + }, + }); + +const mainsIn = (raw: string): string[] => + (JSON.parse(raw).letsencrypt.Certificates ?? []).map( + (certificate: { domain: { main: string } }) => certificate.domain.main, + ); + +describe("purgeAcmeCertificates on a remote server", () => { + /** One acme.json per remote host, as on a real fleet. */ + let remoteStores: Map; + /** Delay applied to the read, to give a concurrent purge room to interleave. */ + let readDelayMs: number; + + const storeOf = (serverId: string) => remoteStores.get(serverId) ?? ""; + + beforeEach(() => { + vi.clearAllMocks(); + remoteStores = new Map([ + ["server-one", certificatesFor(["a.example.com", "b.example.com"])], + ["server-two", certificatesFor(["a.example.com", "b.example.com"])], + ]); + readDelayMs = 0; + + execAsyncRemoteMock.mockImplementation( + async (serverId: string, command: string) => { + if (command.startsWith("cat ")) { + // The remote reads the file when the command runs; the contents + // only reach us after the round trip. Snapshotting before the + // delay is what makes a concurrent purge observable. + const snapshot = storeOf(serverId); + await new Promise((resolve) => setTimeout(resolve, readDelayMs)); + return { stdout: snapshot, stderr: "" }; + } + + const payload = command.match(/printf '%s' "([^"]+)"/)?.[1]; + if (!payload) throw new Error(`unrecognised command: ${command}`); + remoteStores.set( + serverId, + Buffer.from(payload, "base64").toString("utf8"), + ); + return { stdout: "", stderr: "" }; + }, + ); + }); + + it("removes the requested host and leaves the others", async () => { + const removed = await purgeAcmeCertificates( + ["a.example.com"], + "server-one", + ); + + expect(removed).toEqual(["a.example.com"]); + expect(mainsIn(storeOf("server-one"))).toEqual(["b.example.com"]); + }); + + // Without serialisation both purges read the original store and the later + // write puts back whatever the earlier one removed. + it("does not lose a removal when two purges run concurrently", async () => { + readDelayMs = 10; + + const [firstRemoved, secondRemoved] = await Promise.all([ + purgeAcmeCertificates(["a.example.com"], "server-one"), + purgeAcmeCertificates(["b.example.com"], "server-one"), + ]); + + expect(firstRemoved).toEqual(["a.example.com"]); + expect(secondRemoved).toEqual(["b.example.com"]); + expect(mainsIn(storeOf("server-one"))).toEqual([]); + }); + + it("keeps serialising after a failed purge", async () => { + readDelayMs = 5; + const failing = execAsyncRemoteMock.getMockImplementation(); + execAsyncRemoteMock.mockImplementationOnce(async () => { + throw new Error("ssh dropped"); + }); + + const results = await Promise.allSettled([ + purgeAcmeCertificates(["a.example.com"], "server-one"), + purgeAcmeCertificates(["b.example.com"], "server-one"), + ]); + + expect(failing).toBeDefined(); + expect(results[0]?.status).toBe("rejected"); + expect(results[1]).toEqual({ + status: "fulfilled", + value: ["b.example.com"], + }); + expect(mainsIn(storeOf("server-one"))).toEqual(["a.example.com"]); + }); + + it("runs purges for different servers independently", async () => { + const removed = await Promise.all([ + purgeAcmeCertificates(["a.example.com"], "server-one"), + purgeAcmeCertificates(["a.example.com"], "server-two"), + ]); + + expect(removed).toEqual([["a.example.com"], ["a.example.com"]]); + }); +}); diff --git a/packages/server/src/utils/traefik/acme.ts b/packages/server/src/utils/traefik/acme.ts index d11737ccfd..fd6f01b69b 100644 --- a/packages/server/src/utils/traefik/acme.ts +++ b/packages/server/src/utils/traefik/acme.ts @@ -54,6 +54,30 @@ const acmeJsonPath = (isRemote: boolean) => { return path.join(DYNAMIC_TRAEFIK_PATH, "acme.json"); }; +const purgeChains = new Map>(); + +/** + * Runs purges for one server one at a time. A purge is a read-modify-write over + * a file Traefik owns, so two concurrent calls for the same server would both + * read the old store and the later write would resurrect whatever the earlier + * one removed. Serialising in process is the practical boundary here, since a + * server's acme.json is only ever written by the Dokploy instance managing it. + */ +const withPurgeLock = (key: string, task: () => Promise): Promise => { + const previous = purgeChains.get(key) ?? Promise.resolve(); + // Run whether or not the previous purge succeeded, otherwise one failure + // would block every later purge for that server. + const result = previous.then(task, task); + purgeChains.set( + key, + result.then( + () => undefined, + () => undefined, + ), + ); + return result; +}; + /** * Removes the ACME certificates for the given hosts and returns the hosts that * were actually removed. The caller is responsible for reloading Traefik: @@ -66,6 +90,15 @@ export const purgeAcmeCertificates = async ( ): Promise => { if (hosts.length === 0) return []; + return withPurgeLock(serverId ?? "", () => + purgeAcmeCertificatesUnsynchronised(hosts, serverId), + ); +}; + +const purgeAcmeCertificatesUnsynchronised = async ( + hosts: string[], + serverId?: string | null, +): Promise => { const filePath = acmeJsonPath(!!serverId); let raw: string; From bc84642b45894b8fbd3bc3ef5359de56377ba7eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 04:05:26 +0300 Subject: [PATCH 14/16] fix(traefik): stop a failed reconciliation from aborting startup The domain query ran outside the per-application error isolation, so a database failure propagated to the shared catch in server.ts. Everything sequenced after it was skipped: the backup cron jobs, the restart notifications, the enterprise cron jobs and the deployment worker, all while the HTTP server was already listening. The pass is best effort and now never rejects. --- .../__test__/traefik/reconciliation.test.ts | 18 ++++++++++++++++++ .../src/setup/domain-tls-reconciliation.ts | 12 ++++++++++++ 2 files changed, 30 insertions(+) diff --git a/apps/dokploy/__test__/traefik/reconciliation.test.ts b/apps/dokploy/__test__/traefik/reconciliation.test.ts index e771b132c1..6d6e9069a8 100644 --- a/apps/dokploy/__test__/traefik/reconciliation.test.ts +++ b/apps/dokploy/__test__/traefik/reconciliation.test.ts @@ -357,4 +357,22 @@ describe("initDomainTlsReconciliation", () => { manageDomainMock.mock.invocationCallOrder[0] ?? Number.NaN; expect(firstPurge).toBeLessThan(firstManage); }); + + // It runs in the startup sequence ahead of the backup cron jobs, the restart + // notifications and the deployment worker, all of which share a single catch. + // A rejection here would leave every one of them uninitialised. + it("does not reject when the domain query fails", async () => { + findDomainsNeedingTlsReconciliationMock.mockRejectedValue( + new Error("database unreachable"), + ); + + await expect(initDomainTlsReconciliation()).resolves.toBeUndefined(); + expect(manageDomainMock).not.toHaveBeenCalled(); + }); + + it("does not reject when regenerating every application fails", async () => { + findApplicationByIdMock.mockRejectedValue(new Error("boom")); + + await expect(initDomainTlsReconciliation()).resolves.toBeUndefined(); + }); }); diff --git a/packages/server/src/setup/domain-tls-reconciliation.ts b/packages/server/src/setup/domain-tls-reconciliation.ts index f547209d59..2a2cbb2c4b 100644 --- a/packages/server/src/setup/domain-tls-reconciliation.ts +++ b/packages/server/src/setup/domain-tls-reconciliation.ts @@ -30,8 +30,20 @@ export const routerNeedsTlsFix = ( /** * One-shot pass that regenerates router configs written before the fix. * Idempotent: once a router carries `tls: {}` it is skipped on later starts. + * + * Never rejects. It runs inside the startup sequence, ahead of the backup cron + * jobs, the restart notifications and the deployment worker, so a failure here + * must not stop any of those from being brought up. */ export const initDomainTlsReconciliation = async () => { + try { + await reconcileDomainTls(); + } catch (error) { + console.error("TLS reconciliation could not run:", error); + } +}; + +const reconcileDomainTls = async () => { const domains = await findDomainsNeedingTlsReconciliation(); if (domains.length === 0) return; From d46f6e4d2bb17168c13a6db178ad15ddbb20f40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Tue, 4 Aug 2026 04:14:39 +0300 Subject: [PATCH 15/16] fix(traefik): re-check acme.json before swapping the purged store in Traefik owns acme.json and rewrites it in full whenever it issues or renews a certificate, with no locking protocol to join. A snapshot taken before such a write would have dropped the new certificate on rename. The store is now re-read and compared just before the swap, and the purge starts over if it changed. That narrows the window to the final round trip rather than closing it, which is as far as this can go without stopping Traefik to edit its own state. If a purged certificate is reinstated by a write we lost the race to, the startup reconciliation purges it again on the next boot. --- .../__test__/traefik/acme-purge.test.ts | 39 ++++++ packages/server/src/utils/traefik/acme.ts | 117 +++++++++++------- 2 files changed, 111 insertions(+), 45 deletions(-) diff --git a/apps/dokploy/__test__/traefik/acme-purge.test.ts b/apps/dokploy/__test__/traefik/acme-purge.test.ts index cda546974a..e61f5a894a 100644 --- a/apps/dokploy/__test__/traefik/acme-purge.test.ts +++ b/apps/dokploy/__test__/traefik/acme-purge.test.ts @@ -114,6 +114,45 @@ describe("purgeAcmeCertificates on a remote server", () => { expect(mainsIn(storeOf("server-one"))).toEqual(["a.example.com"]); }); + // Traefik owns acme.json and rewrites it in full when it issues or renews a + // certificate. Swapping in a snapshot taken before that write would drop the + // new certificate from disk. + it("does not drop a certificate Traefik writes while the purge is in flight", async () => { + const passthrough = execAsyncRemoteMock.getMockImplementation(); + let reads = 0; + + execAsyncRemoteMock.mockImplementation( + async (serverId: string, command: string) => { + const result = await passthrough?.(serverId, command); + if (command.startsWith("cat ")) { + reads += 1; + if (reads === 1) { + // Traefik issues a certificate right after our first read. + const store = JSON.parse(storeOf(serverId)); + store.letsencrypt.Certificates.push({ + domain: { main: "fresh.example.com" }, + certificate: "cert", + key: "key", + }); + remoteStores.set(serverId, JSON.stringify(store)); + } + } + return result; + }, + ); + + const removed = await purgeAcmeCertificates( + ["a.example.com"], + "server-one", + ); + + expect(removed).toEqual(["a.example.com"]); + expect(mainsIn(storeOf("server-one"))).toEqual([ + "b.example.com", + "fresh.example.com", + ]); + }); + it("runs purges for different servers independently", async () => { const removed = await Promise.all([ purgeAcmeCertificates(["a.example.com"], "server-one"), diff --git a/packages/server/src/utils/traefik/acme.ts b/packages/server/src/utils/traefik/acme.ts index fd6f01b69b..443638a9c1 100644 --- a/packages/server/src/utils/traefik/acme.ts +++ b/packages/server/src/utils/traefik/acme.ts @@ -95,66 +95,93 @@ export const purgeAcmeCertificates = async ( ); }; -const purgeAcmeCertificatesUnsynchronised = async ( - hosts: string[], +const readAcmeStoreRaw = async ( + filePath: string, serverId?: string | null, -): Promise => { - const filePath = acmeJsonPath(!!serverId); - - let raw: string; +): Promise => { if (serverId) { const { stdout } = await execAsyncRemote( serverId, `cat ${filePath} 2>/dev/null || true`, ); - raw = stdout; - } else { - if (!fs.existsSync(filePath)) return []; - raw = fs.readFileSync(filePath, "utf8"); - } - - if (!raw.trim()) return []; - - let parsed: AcmeStore; - try { - parsed = JSON.parse(raw) as AcmeStore; - } catch { - // A malformed store is Traefik's to repair, not ours to overwrite. - return []; + return stdout; } + if (!fs.existsSync(filePath)) return null; + return fs.readFileSync(filePath, "utf8"); +}; - const { store, removed } = removeAcmeCertificates(parsed, hosts); - if (removed.length === 0) return []; +const PURGE_ATTEMPTS = 3; - const serialized = JSON.stringify(store, null, 2); +const purgeAcmeCertificatesUnsynchronised = async ( + hosts: string[], + serverId?: string | null, +): Promise => { + const filePath = acmeJsonPath(!!serverId); - // Never write in place: a truncated acme.json costs every Let's Encrypt - // certificate on the server. Write a sibling temp file, lock it down to - // 0600 (Traefik refuses to start on anything more permissive) and rename - // it over the target, which is atomic within the same directory. - const tempPath = `${filePath}.dokploy.tmp`; + for (let attempt = 1; attempt <= PURGE_ATTEMPTS; attempt++) { + const raw = await readAcmeStoreRaw(filePath, serverId); + if (raw === null || !raw.trim()) return []; - if (serverId) { - await execAsyncRemote( - serverId, - `umask 077 && printf '%s' "${encodeBase64(serialized)}" | base64 -d > ${tempPath} && chmod 600 ${tempPath} && mv -f ${tempPath} ${filePath} || { rm -f ${tempPath}; exit 1; }`, - ); - } else { + let parsed: AcmeStore; try { - fs.writeFileSync(tempPath, serialized, { encoding: "utf8", mode: 0o600 }); - // writeFileSync's mode is subject to the process umask, and it is - // ignored entirely when the temp file already exists. - fs.chmodSync(tempPath, 0o600); - fs.renameSync(tempPath, filePath); - } catch (error) { + parsed = JSON.parse(raw) as AcmeStore; + } catch { + // A malformed store is Traefik's to repair, not ours to overwrite. + return []; + } + + const { store, removed } = removeAcmeCertificates(parsed, hosts); + if (removed.length === 0) return []; + + // Traefik owns this file and rewrites it in full whenever it issues or + // renews a certificate, without any locking protocol we can join. Check + // the store still looks the way we read it before swapping ours in, so a + // certificate Traefik wrote while we were working is not dropped. This + // narrows the window to the final round trip rather than closing it, + // which is the best available short of stopping Traefik to edit its + // store. If the certificate we purge is reinstated by such a write, the + // startup reconciliation purges it again on the next boot. + const current = await readAcmeStoreRaw(filePath, serverId); + if (current !== raw) continue; + + const serialized = JSON.stringify(store, null, 2); + + // Never write in place: a truncated acme.json costs every Let's Encrypt + // certificate on the server. Write a sibling temp file, lock it down to + // 0600 (Traefik refuses to start on anything more permissive) and rename + // it over the target, which is atomic within the same directory. + const tempPath = `${filePath}.dokploy.tmp`; + + if (serverId) { + await execAsyncRemote( + serverId, + `umask 077 && printf '%s' "${encodeBase64(serialized)}" | base64 -d > ${tempPath} && chmod 600 ${tempPath} && mv -f ${tempPath} ${filePath} || { rm -f ${tempPath}; exit 1; }`, + ); + } else { try { - fs.unlinkSync(tempPath); - } catch { - // The temp file may never have been created. + fs.writeFileSync(tempPath, serialized, { + encoding: "utf8", + mode: 0o600, + }); + // writeFileSync's mode is subject to the process umask, and it is + // ignored entirely when the temp file already exists. + fs.chmodSync(tempPath, 0o600); + fs.renameSync(tempPath, filePath); + } catch (error) { + try { + fs.unlinkSync(tempPath); + } catch { + // The temp file may never have been created. + } + throw error; } - throw error; } + + return removed; } - return removed; + console.warn( + `Skipped purging ${hosts.join(", ")} from acme.json: Traefik kept rewriting it`, + ); + return []; }; From 9db157c5a176b67936fcad8d9d487049cbbf1987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=BCldal=C4=B1?= Date: Fri, 7 Aug 2026 00:24:09 +0300 Subject: [PATCH 16/16] fix(traefik): purge stale acme.json entries independent of router state The startup pass only purged an application's stale certificates when its routers still needed regeneration. Once a router carried `tls: {}` routerNeedsTlsFix went false and the application was skipped entirely, so a purge that had exhausted its retry budget or lost a race against Traefik was never revisited. Split reconcileDomainTls into two phases: phase 1 regenerates router configs as before, phase 2 purges every domain's host regardless of whether its router needed regeneration. Phase 2 groups domains by server rather than by application, since acme.json is one file per server, cutting repeated reads (and remote SSH round trips) down to one per server per pass. --- .../__test__/traefik/reconciliation.test.ts | 32 ++++-- .../src/setup/domain-tls-reconciliation.ts | 101 +++++++++++++----- 2 files changed, 95 insertions(+), 38 deletions(-) diff --git a/apps/dokploy/__test__/traefik/reconciliation.test.ts b/apps/dokploy/__test__/traefik/reconciliation.test.ts index 6d6e9069a8..9769f271a7 100644 --- a/apps/dokploy/__test__/traefik/reconciliation.test.ts +++ b/apps/dokploy/__test__/traefik/reconciliation.test.ts @@ -178,7 +178,8 @@ const routerConfigFor = ( describe("initDomainTlsReconciliation", () => { // Shared fixture: // app-1: local, two stale domains sharing one router config load. - // app-2: local, router already carries `tls: {}` -> nothing to do. + // app-2: local, router already carries `tls: {}` -> not regenerated, + // but its host must still be purged (the defect being fixed). // app-3: remote, config load throws -> must not block the others. // app-4: remote, one stale domain but no certificate actually removed. const appOneDomainA = buildDomain({ @@ -309,13 +310,20 @@ describe("initDomainTlsReconciliation", () => { it("requests a reload at most once per server and only when a certificate was removed", async () => { await initDomainTlsReconciliation(); - // app-1 (local, serverId undefined) had removals -> exactly one - // reload. app-4's server reported no removals -> no reload at all. - expect(reloadDockerResourceMock).toHaveBeenCalledTimes(1); + // The local server (serverId undefined) batches app-1's and app-2's + // hosts into a single purge call and reloads once. server-x (app-3) + // also has removals and reloads once, independently of phase 1 having + // failed for that application. server-y (app-4) reported no removals, + // so it never reloads. + expect(reloadDockerResourceMock).toHaveBeenCalledTimes(2); expect(reloadDockerResourceMock).toHaveBeenCalledWith( "dokploy-traefik", undefined, ); + expect(reloadDockerResourceMock).toHaveBeenCalledWith( + "dokploy-traefik", + "server-x", + ); expect(reloadDockerResourceMock).not.toHaveBeenCalledWith( "dokploy-traefik", "server-y", @@ -348,14 +356,18 @@ describe("initDomainTlsReconciliation", () => { expect(managedHosts()).toContain("a1.example.com"); }); - it("purges the stale certificate before regenerating the router", async () => { + // This is the defect being fixed: once a router carries `tls: {}`, + // `routerNeedsTlsFix` is false and the application is never regenerated, + // but a lingering acme.json entry for its host must still be cleaned up + // on every boot, not just the one where the router itself was fixed. + it("purges a stale host even when the application's routers are already correct", async () => { await initDomainTlsReconciliation(); - const firstPurge = - purgeAcmeCertificatesMock.mock.invocationCallOrder[0] ?? Number.NaN; - const firstManage = - manageDomainMock.mock.invocationCallOrder[0] ?? Number.NaN; - expect(firstPurge).toBeLessThan(firstManage); + const purgedHosts = purgeAcmeCertificatesMock.mock.calls.flatMap( + (call) => call[0] as string[], + ); + expect(purgedHosts).toContain("b1.example.com"); + expect(managedHosts()).not.toContain("b1.example.com"); }); // It runs in the startup sequence ahead of the backup cron jobs, the restart diff --git a/packages/server/src/setup/domain-tls-reconciliation.ts b/packages/server/src/setup/domain-tls-reconciliation.ts index 2a2cbb2c4b..e3d0b991b9 100644 --- a/packages/server/src/setup/domain-tls-reconciliation.ts +++ b/packages/server/src/setup/domain-tls-reconciliation.ts @@ -47,8 +47,6 @@ const reconcileDomainTls = async () => { const domains = await findDomainsNeedingTlsReconciliation(); if (domains.length === 0) return; - const reloadTargets = new Set(); - const byApplication = new Map(); for (const domain of domains) { if (!domain.applicationId) continue; @@ -57,9 +55,31 @@ const reconcileDomainTls = async () => { byApplication.set(domain.applicationId, bucket); } + // Resolved once, up front, since both phases below need it: phase 1 to + // regenerate routers, phase 2 to know which server each domain's + // application lives on. An application that fails to resolve here is + // excluded from both phases. + const applications = new Map< + string, + Awaited> + >(); + for (const applicationId of byApplication.keys()) { + try { + applications.set(applicationId, await findApplicationById(applicationId)); + } catch (error) { + console.error( + `TLS reconciliation could not resolve application ${applicationId}:`, + error, + ); + } + } + + // Phase 1: regenerate router configs written before the TLS override fix. for (const [applicationId, appDomains] of byApplication) { + const application = applications.get(applicationId); + if (!application) continue; + try { - const application = await findApplicationById(applicationId); const config = application.serverId ? await loadOrCreateConfigRemote( application.serverId, @@ -72,28 +92,6 @@ const reconcileDomainTls = async () => { ); if (stale.length === 0) continue; - // A host still served by another Let's Encrypt domain keeps its - // certificate, exactly as the mutation path does. - const purgeableHosts: string[] = []; - for (const domain of stale) { - const stillInUse = await hasOtherLetsencryptDomainForHost( - domain.host, - domain.domainId, - ); - if (!stillInUse) purgeableHosts.push(domain.host); - } - - // Purge before regenerating: once the router carries its `tls` key - // `routerNeedsTlsFix` is false, so a purge that failed afterwards - // would never be retried on a later boot. - const removed = - purgeableHosts.length > 0 - ? await purgeAcmeCertificates(purgeableHosts, application.serverId) - : []; - if (removed.length > 0) { - reloadTargets.add(application.serverId ?? ""); - } - for (const domain of stale) { await manageDomain(application, domain); } @@ -110,11 +108,58 @@ const reconcileDomainTls = async () => { } } - for (const serverId of reloadTargets) { + // Phase 2: purge stale acme.json entries, independent of whether the + // router for that domain still needed regeneration. A router already + // carrying `tls: {}` is exactly the state `routerNeedsTlsFix` treats as + // "nothing to do" in phase 1, so it is the only place left where a + // certificate purge that was skipped or lost a race against Traefik on an + // earlier boot gets retried. Grouped per server, not per application, + // because acme.json is one file per server: this keeps the read/rewrite + // and any remote SSH round trip to once per server for the whole pass. + const byServer = new Map(); + for (const domain of domains) { + if (!domain.applicationId) continue; + const application = applications.get(domain.applicationId); + if (!application) continue; + const serverKey = application.serverId ?? ""; + const bucket = byServer.get(serverKey) ?? []; + bucket.push(domain); + byServer.set(serverKey, bucket); + } + + for (const [serverKey, serverDomains] of byServer) { + const serverId = serverKey || undefined; try { - await reloadDockerResource("dokploy-traefik", serverId || undefined); + // A host still served by another Let's Encrypt domain keeps its + // certificate, exactly as the mutation path does. + const hostToDomainId = new Map(); + for (const domain of serverDomains) { + if (!hostToDomainId.has(domain.host)) { + hostToDomainId.set(domain.host, domain.domainId); + } + } + + const purgeableHosts: string[] = []; + for (const [host, domainId] of hostToDomainId) { + const stillInUse = await hasOtherLetsencryptDomainForHost( + host, + domainId, + ); + if (!stillInUse) purgeableHosts.push(host); + } + + if (purgeableHosts.length === 0) continue; + + const removed = await purgeAcmeCertificates(purgeableHosts, serverId); + if (removed.length > 0) { + await reloadDockerResource("dokploy-traefik", serverId); + } } catch (error) { - console.error("TLS reconciliation could not reload Traefik:", error); + // One unreachable remote server must not stop the rest. + console.error( + `TLS reconciliation could not purge certificates for server ${serverKey || "local"}:`, + error, + ); } } };