diff --git a/apps/dokploy/__test__/compose/domain/labels.test.ts b/apps/dokploy/__test__/compose/domain/labels.test.ts index 57e018bafe..4731c048d2 100644 --- a/apps/dokploy/__test__/compose/domain/labels.test.ts +++ b/apps/dokploy/__test__/compose/domain/labels.test.ts @@ -544,4 +544,20 @@ describe("createDomainLabels", () => { // Should not contain redirect-to-https since there's only one router expect(middlewareLabel).toBeUndefined(); }); + + it("should add tls=true for certificateType custom without a resolver", async () => { + const customDomain = { + ...baseDomain, + https: true, + certificateType: "custom" as const, + customCertResolver: null, + }; + const labels = await createDomainLabels(appName, customDomain, "websecure"); + expect(labels).toContain( + "traefik.http.routers.test-app-1-websecure.tls=true", + ); + expect(labels).not.toContain( + "traefik.http.routers.test-app-1-websecure.tls.certresolver=letsencrypt", + ); + }); }); diff --git a/apps/dokploy/__test__/traefik/acme-purge.test.ts b/apps/dokploy/__test__/traefik/acme-purge.test.ts new file mode 100644 index 0000000000..e61f5a894a --- /dev/null +++ b/apps/dokploy/__test__/traefik/acme-purge.test.ts @@ -0,0 +1,164 @@ +import { purgeAcmeCertificates } from "@dokploy/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const execAsyncRemoteMock = vi.fn(); +vi.mock("@dokploy/server/utils/process/execAsync", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/process/execAsync") + >("@dokploy/server/utils/process/execAsync"); + return { + ...actual, + execAsyncRemote: (serverId: string, command: string) => + execAsyncRemoteMock(serverId, command), + }; +}); + +const certificatesFor = (mains: string[]) => + JSON.stringify({ + letsencrypt: { + Account: { Email: "test@localhost.com" }, + Certificates: mains.map((main) => ({ + domain: { main }, + certificate: "cert", + key: "key", + })), + }, + }); + +const mainsIn = (raw: string): string[] => + (JSON.parse(raw).letsencrypt.Certificates ?? []).map( + (certificate: { domain: { main: string } }) => certificate.domain.main, + ); + +describe("purgeAcmeCertificates on a remote server", () => { + /** One acme.json per remote host, as on a real fleet. */ + let remoteStores: Map; + /** Delay applied to the read, to give a concurrent purge room to interleave. */ + let readDelayMs: number; + + const storeOf = (serverId: string) => remoteStores.get(serverId) ?? ""; + + beforeEach(() => { + vi.clearAllMocks(); + remoteStores = new Map([ + ["server-one", certificatesFor(["a.example.com", "b.example.com"])], + ["server-two", certificatesFor(["a.example.com", "b.example.com"])], + ]); + readDelayMs = 0; + + execAsyncRemoteMock.mockImplementation( + async (serverId: string, command: string) => { + if (command.startsWith("cat ")) { + // The remote reads the file when the command runs; the contents + // only reach us after the round trip. Snapshotting before the + // delay is what makes a concurrent purge observable. + const snapshot = storeOf(serverId); + await new Promise((resolve) => setTimeout(resolve, readDelayMs)); + return { stdout: snapshot, stderr: "" }; + } + + const payload = command.match(/printf '%s' "([^"]+)"/)?.[1]; + if (!payload) throw new Error(`unrecognised command: ${command}`); + remoteStores.set( + serverId, + Buffer.from(payload, "base64").toString("utf8"), + ); + return { stdout: "", stderr: "" }; + }, + ); + }); + + it("removes the requested host and leaves the others", async () => { + const removed = await purgeAcmeCertificates( + ["a.example.com"], + "server-one", + ); + + expect(removed).toEqual(["a.example.com"]); + expect(mainsIn(storeOf("server-one"))).toEqual(["b.example.com"]); + }); + + // Without serialisation both purges read the original store and the later + // write puts back whatever the earlier one removed. + it("does not lose a removal when two purges run concurrently", async () => { + readDelayMs = 10; + + const [firstRemoved, secondRemoved] = await Promise.all([ + purgeAcmeCertificates(["a.example.com"], "server-one"), + purgeAcmeCertificates(["b.example.com"], "server-one"), + ]); + + expect(firstRemoved).toEqual(["a.example.com"]); + expect(secondRemoved).toEqual(["b.example.com"]); + expect(mainsIn(storeOf("server-one"))).toEqual([]); + }); + + it("keeps serialising after a failed purge", async () => { + readDelayMs = 5; + const failing = execAsyncRemoteMock.getMockImplementation(); + execAsyncRemoteMock.mockImplementationOnce(async () => { + throw new Error("ssh dropped"); + }); + + const results = await Promise.allSettled([ + purgeAcmeCertificates(["a.example.com"], "server-one"), + purgeAcmeCertificates(["b.example.com"], "server-one"), + ]); + + expect(failing).toBeDefined(); + expect(results[0]?.status).toBe("rejected"); + expect(results[1]).toEqual({ + status: "fulfilled", + value: ["b.example.com"], + }); + expect(mainsIn(storeOf("server-one"))).toEqual(["a.example.com"]); + }); + + // Traefik owns acme.json and rewrites it in full when it issues or renews a + // certificate. Swapping in a snapshot taken before that write would drop the + // new certificate from disk. + it("does not drop a certificate Traefik writes while the purge is in flight", async () => { + const passthrough = execAsyncRemoteMock.getMockImplementation(); + let reads = 0; + + execAsyncRemoteMock.mockImplementation( + async (serverId: string, command: string) => { + const result = await passthrough?.(serverId, command); + if (command.startsWith("cat ")) { + reads += 1; + if (reads === 1) { + // Traefik issues a certificate right after our first read. + const store = JSON.parse(storeOf(serverId)); + store.letsencrypt.Certificates.push({ + domain: { main: "fresh.example.com" }, + certificate: "cert", + key: "key", + }); + remoteStores.set(serverId, JSON.stringify(store)); + } + } + return result; + }, + ); + + const removed = await purgeAcmeCertificates( + ["a.example.com"], + "server-one", + ); + + expect(removed).toEqual(["a.example.com"]); + expect(mainsIn(storeOf("server-one"))).toEqual([ + "b.example.com", + "fresh.example.com", + ]); + }); + + it("runs purges for different servers independently", async () => { + const removed = await Promise.all([ + purgeAcmeCertificates(["a.example.com"], "server-one"), + purgeAcmeCertificates(["a.example.com"], "server-two"), + ]); + + expect(removed).toEqual([["a.example.com"], ["a.example.com"]]); + }); +}); diff --git a/apps/dokploy/__test__/traefik/acme.test.ts b/apps/dokploy/__test__/traefik/acme.test.ts new file mode 100644 index 0000000000..9e2329ee64 --- /dev/null +++ b/apps/dokploy/__test__/traefik/acme.test.ts @@ -0,0 +1,89 @@ +import type { AcmeStore } from "@dokploy/server"; +import { removeAcmeCertificates } from "@dokploy/server"; +import { describe, expect, it } from "vitest"; + +const buildStore = (mains: string[]): AcmeStore => ({ + letsencrypt: { + Account: { Email: "test@example.com" }, + Certificates: mains.map((main) => ({ + domain: { main }, + certificate: "cert", + key: "key", + })), + }, +}); + +describe("removeAcmeCertificates", () => { + it("removes only the requested hosts", () => { + const store = buildStore(["a.example.com", "b.example.com"]); + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.removed).toEqual(["a.example.com"]); + expect(result.store.letsencrypt?.Certificates).toHaveLength(1); + expect(result.store.letsencrypt?.Certificates?.[0]?.domain.main).toBe( + "b.example.com", + ); + }); + + it("keeps the Account object intact", () => { + const store = buildStore(["a.example.com"]); + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.store.letsencrypt?.Account).toEqual({ + Email: "test@example.com", + }); + expect(result.store.letsencrypt?.Certificates).toEqual([]); + }); + + it("reports nothing removed when the host is absent", () => { + const store = buildStore(["a.example.com"]); + + const result = removeAcmeCertificates(store, ["other.example.com"]); + + expect(result.removed).toEqual([]); + expect(result.store.letsencrypt?.Certificates).toHaveLength(1); + }); + + it("tolerates a resolver with no Certificates array", () => { + const store: AcmeStore = { letsencrypt: { Account: {} } }; + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.removed).toEqual([]); + }); + + it("does not mutate the input store", () => { + const store = buildStore(["a.example.com"]); + + removeAcmeCertificates(store, ["a.example.com"]); + + expect(store.letsencrypt?.Certificates).toHaveLength(1); + }); +}); + +describe("removeAcmeCertificates with multiple resolvers", () => { + it("only touches the resolver that holds the host", () => { + const store: AcmeStore = { + letsencrypt: { + Account: {}, + Certificates: [ + { domain: { main: "a.example.com" }, certificate: "c", key: "k" }, + ], + }, + other: { + Account: {}, + Certificates: [ + { domain: { main: "b.example.com" }, certificate: "c", key: "k" }, + ], + }, + }; + + const result = removeAcmeCertificates(store, ["a.example.com"]); + + expect(result.removed).toEqual(["a.example.com"]); + expect(result.store.letsencrypt?.Certificates).toEqual([]); + expect(result.store.other?.Certificates).toHaveLength(1); + }); +}); diff --git a/apps/dokploy/__test__/traefik/domain-certificate.test.ts b/apps/dokploy/__test__/traefik/domain-certificate.test.ts new file mode 100644 index 0000000000..d0e2bd0197 --- /dev/null +++ b/apps/dokploy/__test__/traefik/domain-certificate.test.ts @@ -0,0 +1,100 @@ +import type { Domain } from "@dokploy/server"; +import { purgeStaleCertificate } from "@dokploy/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Stands in for the `domains` lookup that answers "is this host still served +// by another Let's Encrypt domain?". +const hasOtherLetsencryptDomainForHostMock = vi.fn(); +vi.mock("@dokploy/server/services/domain", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/domain") + >("@dokploy/server/services/domain"); + return { + ...actual, + hasOtherLetsencryptDomainForHost: (host: string, excludeId: string) => + hasOtherLetsencryptDomainForHostMock(host, excludeId), + }; +}); + +const purgeAcmeCertificatesMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/acme", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/acme") + >("@dokploy/server/utils/traefik/acme"); + return { + ...actual, + purgeAcmeCertificates: (hosts: string[], serverId?: string | null) => + purgeAcmeCertificatesMock(hosts, serverId), + }; +}); + +const buildDomain = (overrides: Partial = {}): Domain => + ({ + domainId: "domain-1", + host: "example.com", + certificateType: "none", + applicationId: "app-1", + uniqueConfigKey: 1, + ...overrides, + }) as Domain; + +describe("purgeStaleCertificate", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + it("purges the host when no other Let's Encrypt domain uses it", async () => { + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(false); + purgeAcmeCertificatesMock.mockResolvedValue(["example.com"]); + + const result = await purgeStaleCertificate(buildDomain(), "server-1"); + + expect(purgeAcmeCertificatesMock).toHaveBeenCalledWith( + ["example.com"], + "server-1", + ); + expect(result).toBe(true); + }); + + it("keeps the host when another Let's Encrypt domain still uses it", async () => { + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(true); + + const result = await purgeStaleCertificate(buildDomain(), "server-1"); + + expect(purgeAcmeCertificatesMock).not.toHaveBeenCalled(); + expect(result).toBe(false); + }); + + it("never purges a domain that still uses Let's Encrypt", async () => { + const result = await purgeStaleCertificate( + buildDomain({ certificateType: "letsencrypt" }), + "server-1", + ); + + expect(hasOtherLetsencryptDomainForHostMock).not.toHaveBeenCalled(); + expect(purgeAcmeCertificatesMock).not.toHaveBeenCalled(); + expect(result).toBe(false); + }); + + it("reports no reload instead of throwing when the purge fails", async () => { + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(false); + purgeAcmeCertificatesMock.mockRejectedValue( + new Error("ssh connection refused"), + ); + + await expect( + purgeStaleCertificate(buildDomain(), "server-1"), + ).resolves.toBe(false); + }); + + it("reports no reload instead of throwing when the lookup fails", async () => { + hasOtherLetsencryptDomainForHostMock.mockRejectedValue( + new Error("database unavailable"), + ); + + await expect( + purgeStaleCertificate(buildDomain(), "server-1"), + ).resolves.toBe(false); + }); +}); diff --git a/apps/dokploy/__test__/traefik/reconciliation.test.ts b/apps/dokploy/__test__/traefik/reconciliation.test.ts new file mode 100644 index 0000000000..9769f271a7 --- /dev/null +++ b/apps/dokploy/__test__/traefik/reconciliation.test.ts @@ -0,0 +1,390 @@ +import type { Domain, FileConfig } from "@dokploy/server"; +import { + initDomainTlsReconciliation, + routerNeedsTlsFix, +} from "@dokploy/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const findDomainsNeedingTlsReconciliationMock = vi.fn(); +const hasOtherLetsencryptDomainForHostMock = vi.fn(); +vi.mock("@dokploy/server/services/domain", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/domain") + >("@dokploy/server/services/domain"); + return { + ...actual, + findDomainsNeedingTlsReconciliation: () => + findDomainsNeedingTlsReconciliationMock(), + hasOtherLetsencryptDomainForHost: (host: string, excludeId: string) => + hasOtherLetsencryptDomainForHostMock(host, excludeId), + }; +}); + +const findApplicationByIdMock = vi.fn(); +vi.mock("@dokploy/server/services/application", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/application") + >("@dokploy/server/services/application"); + return { + ...actual, + findApplicationById: (applicationId: string) => + findApplicationByIdMock(applicationId), + }; +}); + +const reloadDockerResourceMock = vi.fn(); +vi.mock("@dokploy/server/services/settings", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/services/settings") + >("@dokploy/server/services/settings"); + return { + ...actual, + reloadDockerResource: (resourceName: string, serverId?: string) => + reloadDockerResourceMock(resourceName, serverId), + }; +}); + +const purgeAcmeCertificatesMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/acme", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/acme") + >("@dokploy/server/utils/traefik/acme"); + return { + ...actual, + purgeAcmeCertificates: (hosts: string[], serverId?: string | null) => + purgeAcmeCertificatesMock(hosts, serverId), + }; +}); + +const loadOrCreateConfigMock = vi.fn(); +const loadOrCreateConfigRemoteMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/application", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/application") + >("@dokploy/server/utils/traefik/application"); + return { + ...actual, + loadOrCreateConfig: (appName: string) => loadOrCreateConfigMock(appName), + loadOrCreateConfigRemote: (serverId: string, appName: string) => + loadOrCreateConfigRemoteMock(serverId, appName), + }; +}); + +const manageDomainMock = vi.fn(); +vi.mock("@dokploy/server/utils/traefik/domain", async () => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/traefik/domain") + >("@dokploy/server/utils/traefik/domain"); + return { + ...actual, + manageDomain: (app: unknown, domain: unknown) => + manageDomainMock(app, domain), + }; +}); + +const configWithRouter = ( + tls: Record | undefined, +): FileConfig => ({ + http: { + routers: { + "my-app-router-websecure-1": { + rule: "Host(`example.com`)", + service: "my-app-service-1", + entryPoints: ["websecure"], + ...(tls === undefined ? {} : { tls }), + }, + }, + services: {}, + }, +}); + +describe("routerNeedsTlsFix", () => { + it("is true when the websecure router has no tls key", () => { + expect(routerNeedsTlsFix(configWithRouter(undefined), "my-app", 1)).toBe( + true, + ); + }); + + it("is false when the router already has an empty tls block", () => { + expect(routerNeedsTlsFix(configWithRouter({}), "my-app", 1)).toBe(false); + }); + + it("is false when the router has a cert resolver", () => { + expect( + routerNeedsTlsFix( + configWithRouter({ certResolver: "letsencrypt" }), + "my-app", + 1, + ), + ).toBe(false); + }); + + it("is false when the router does not exist", () => { + expect(routerNeedsTlsFix(configWithRouter(undefined), "other-app", 1)).toBe( + false, + ); + }); + + it("is false for an empty config", () => { + expect(routerNeedsTlsFix({}, "my-app", 1)).toBe(false); + }); +}); + +const buildDomain = (overrides: Partial): Domain => + ({ + domainId: overrides.host ?? "domain-id", + host: "example.com", + https: false, + port: 3000, + customEntrypoint: null, + path: "/", + serviceName: null, + domainType: "application", + uniqueConfigKey: 1, + createdAt: new Date().toISOString(), + composeId: null, + customCertResolver: null, + applicationId: "app-1", + previewDeploymentId: null, + certificateType: "none", + internalPath: "/", + stripPath: false, + middlewares: [], + forwardAuthEnabled: false, + ...overrides, + }) as Domain; + +const routerConfigFor = ( + appName: string, + keys: number[], + tlsByKey: Record | undefined>, +): FileConfig => ({ + http: { + routers: Object.fromEntries( + keys.map((key) => [ + `${appName}-router-websecure-${key}`, + { + rule: "Host(`example.com`)", + service: `${appName}-service-${key}`, + entryPoints: ["websecure"], + ...(tlsByKey[key] === undefined ? {} : { tls: tlsByKey[key] }), + }, + ]), + ), + services: {}, + }, +}); + +describe("initDomainTlsReconciliation", () => { + // Shared fixture: + // app-1: local, two stale domains sharing one router config load. + // app-2: local, router already carries `tls: {}` -> not regenerated, + // but its host must still be purged (the defect being fixed). + // app-3: remote, config load throws -> must not block the others. + // app-4: remote, one stale domain but no certificate actually removed. + const appOneDomainA = buildDomain({ + domainId: "a1", + host: "a1.example.com", + applicationId: "app-1", + uniqueConfigKey: 1, + }); + const appOneDomainB = buildDomain({ + domainId: "a2", + host: "a2.example.com", + applicationId: "app-1", + uniqueConfigKey: 2, + }); + const appTwoDomain = buildDomain({ + domainId: "b1", + host: "b1.example.com", + applicationId: "app-2", + uniqueConfigKey: 1, + }); + const appThreeDomain = buildDomain({ + domainId: "c1", + host: "c1.example.com", + applicationId: "app-3", + uniqueConfigKey: 1, + }); + const appFourDomain = buildDomain({ + domainId: "d1", + host: "d1.example.com", + applicationId: "app-4", + uniqueConfigKey: 1, + }); + + const applicationsById: Record = { + "app-1": { applicationId: "app-1", appName: "app-one", serverId: null }, + "app-2": { applicationId: "app-2", appName: "app-two", serverId: null }, + "app-3": { + applicationId: "app-3", + appName: "app-three", + serverId: "server-x", + }, + "app-4": { + applicationId: "app-4", + appName: "app-four", + serverId: "server-y", + }, + }; + + const managedHosts = () => + manageDomainMock.mock.calls.map((call) => (call[1] as Domain).host); + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + findDomainsNeedingTlsReconciliationMock.mockResolvedValue([ + appThreeDomain, + appOneDomainA, + appOneDomainB, + appTwoDomain, + appFourDomain, + ]); + + findApplicationByIdMock.mockImplementation( + async (applicationId: string) => applicationsById[applicationId], + ); + + loadOrCreateConfigMock.mockImplementation((appName: string) => { + if (appName === "app-one") { + return routerConfigFor("app-one", [1, 2], { + 1: undefined, + 2: undefined, + }); + } + if (appName === "app-two") { + return routerConfigFor("app-two", [1], { 1: {} }); + } + throw new Error(`unexpected local config load for ${appName}`); + }); + + loadOrCreateConfigRemoteMock.mockImplementation( + async (_serverId: string, appName: string) => { + if (appName === "app-three") { + throw new Error("ssh connection refused"); + } + if (appName === "app-four") { + return routerConfigFor("app-four", [1], { 1: undefined }); + } + throw new Error(`unexpected remote config load for ${appName}`); + }, + ); + + manageDomainMock.mockResolvedValue(undefined); + + // No host is shared with another Let's Encrypt domain by default. + hasOtherLetsencryptDomainForHostMock.mockResolvedValue(false); + + purgeAcmeCertificatesMock.mockImplementation( + async (hosts: string[], serverId?: string | null) => { + // app-4's server reports nothing was actually removed. + if (serverId === "server-y") return []; + return hosts; + }, + ); + + reloadDockerResourceMock.mockResolvedValue(undefined); + }); + + it("does not regenerate an application whose router is already fixed", async () => { + await initDomainTlsReconciliation(); + + expect(managedHosts()).not.toContain("b1.example.com"); + }); + + it("regenerates once per stale domain while loading the application config once", async () => { + await initDomainTlsReconciliation(); + + expect(managedHosts().filter((host) => host.startsWith("a"))).toEqual([ + "a1.example.com", + "a2.example.com", + ]); + expect( + loadOrCreateConfigMock.mock.calls.filter((call) => call[0] === "app-one"), + ).toHaveLength(1); + }); + + it("requests a reload at most once per server and only when a certificate was removed", async () => { + await initDomainTlsReconciliation(); + + // The local server (serverId undefined) batches app-1's and app-2's + // hosts into a single purge call and reloads once. server-x (app-3) + // also has removals and reloads once, independently of phase 1 having + // failed for that application. server-y (app-4) reported no removals, + // so it never reloads. + expect(reloadDockerResourceMock).toHaveBeenCalledTimes(2); + expect(reloadDockerResourceMock).toHaveBeenCalledWith( + "dokploy-traefik", + undefined, + ); + expect(reloadDockerResourceMock).toHaveBeenCalledWith( + "dokploy-traefik", + "server-x", + ); + expect(reloadDockerResourceMock).not.toHaveBeenCalledWith( + "dokploy-traefik", + "server-y", + ); + }); + + it("keeps processing the other applications when one fails", async () => { + await initDomainTlsReconciliation(); + + // app-3 threw while loading its config and never reached + // manageDomain, but app-1 and app-4 were still processed. + expect(managedHosts()).not.toContain("c1.example.com"); + expect(managedHosts()).toContain("a1.example.com"); + expect(managedHosts()).toContain("d1.example.com"); + }); + + it("keeps the certificate of a host another Let's Encrypt domain still uses", async () => { + hasOtherLetsencryptDomainForHostMock.mockImplementation( + async (host: string) => host === "a1.example.com", + ); + + await initDomainTlsReconciliation(); + + const purgedHosts = purgeAcmeCertificatesMock.mock.calls.flatMap( + (call) => call[0] as string[], + ); + expect(purgedHosts).not.toContain("a1.example.com"); + expect(purgedHosts).toContain("a2.example.com"); + // The router is still regenerated, only the certificate is spared. + expect(managedHosts()).toContain("a1.example.com"); + }); + + // This is the defect being fixed: once a router carries `tls: {}`, + // `routerNeedsTlsFix` is false and the application is never regenerated, + // but a lingering acme.json entry for its host must still be cleaned up + // on every boot, not just the one where the router itself was fixed. + it("purges a stale host even when the application's routers are already correct", async () => { + await initDomainTlsReconciliation(); + + const purgedHosts = purgeAcmeCertificatesMock.mock.calls.flatMap( + (call) => call[0] as string[], + ); + expect(purgedHosts).toContain("b1.example.com"); + expect(managedHosts()).not.toContain("b1.example.com"); + }); + + // It runs in the startup sequence ahead of the backup cron jobs, the restart + // notifications and the deployment worker, all of which share a single catch. + // A rejection here would leave every one of them uninitialised. + it("does not reject when the domain query fails", async () => { + findDomainsNeedingTlsReconciliationMock.mockRejectedValue( + new Error("database unreachable"), + ); + + await expect(initDomainTlsReconciliation()).resolves.toBeUndefined(); + expect(manageDomainMock).not.toHaveBeenCalled(); + }); + + it("does not reject when regenerating every application fails", async () => { + findApplicationByIdMock.mockRejectedValue(new Error("boom")); + + await expect(initDomainTlsReconciliation()).resolves.toBeUndefined(); + }); +}); diff --git a/apps/dokploy/__test__/traefik/traefik.test.ts b/apps/dokploy/__test__/traefik/traefik.test.ts index 379c63a00c..818f8488ae 100644 --- a/apps/dokploy/__test__/traefik/traefik.test.ts +++ b/apps/dokploy/__test__/traefik/traefik.test.ts @@ -347,6 +347,32 @@ test("Web entrypoint with empty middlewares array", async () => { /** Certificates */ +test("CertificateType none on websecure sets an empty tls block", async () => { + const router = await createRouterConfig( + baseApp, + { ...baseDomain, certificateType: "none" }, + "websecure", + ); + + // Must be defined, otherwise Traefik applies the websecure entrypoint's + // certResolver default and requests a Let's Encrypt certificate anyway. + expect(router.tls).toEqual({}); +}); + +test("CertificateType custom without a resolver sets an empty tls block", async () => { + const router = await createRouterConfig( + baseApp, + { + ...baseDomain, + certificateType: "custom", + customCertResolver: null, + }, + "websecure", + ); + + expect(router.tls).toEqual({}); +}); + test("CertificateType on websecure entrypoint", async () => { const router = await createRouterConfig( baseApp, diff --git a/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx b/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx index 8bb763add4..2ca864d8fc 100644 --- a/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx +++ b/apps/dokploy/components/dashboard/application/domains/handle-domain.tsx @@ -300,13 +300,30 @@ export const AddDomain = ({ id, type, domainId = "", children }: Props) => { ...data, customEntrypoint: data.useCustomEntrypoint ? data.customEntrypoint : null, }) - .then(async () => { - toast.success( - dictionary.success, - data.domainType === "compose" - ? { description: COMPOSE_REDEPLOY_TOAST } - : undefined, - ); + .then(async (result) => { + const traefikReloadRequired = + result && + "traefikReloadRequired" in result && + result.traefikReloadRequired; + + // A compose domain is rendered as a docker label and only reaches + // Traefik on the next deployment. An application domain that just left + // Let's Encrypt needs a restart instead, because Traefik reads + // acme.json only at startup. A domain is never in both cases. + let hint: { description: string; duration?: number } | undefined; + if (data.domainType === "compose") { + hint = { description: COMPOSE_REDEPLOY_TOAST }; + } else if (traefikReloadRequired) { + hint = { + description: + "Restart Traefik to apply the certificate change for this domain.", + // The default duration is tuned for one-line confirmations; this + // one asks the user to go and do something, so give them longer. + duration: 10000, + }; + } + + toast.success(dictionary.success, hint); if (data.domainType === "application") { await utils.domain.byApplicationId.invalidate({ diff --git a/apps/dokploy/server/api/routers/domain.ts b/apps/dokploy/server/api/routers/domain.ts index 505db50c67..a3fe126842 100644 --- a/apps/dokploy/server/api/routers/domain.ts +++ b/apps/dokploy/server/api/routers/domain.ts @@ -9,6 +9,7 @@ import { generateTraefikMeDomain, getWebServerSettings, manageDomain, + purgeStaleCertificate, removeDomain, removeDomainById, updateDomainById, @@ -126,9 +127,15 @@ export const domainRouter = createTRPCRouter({ resourceId: domain.domainId, resourceName: domain.host, }); + let traefikReloadRequired = false; + if (domain.applicationId) { const application = await findApplicationById(domain.applicationId); await manageDomain(application, domain); + traefikReloadRequired = await purgeStaleCertificate( + domain, + application.serverId, + ); } else if (domain.previewDeploymentId) { const previewDeployment = await findPreviewDeploymentById( domain.previewDeploymentId, @@ -139,7 +146,8 @@ export const domainRouter = createTRPCRouter({ application.appName = previewDeployment.appName; await manageDomain(application, domain); } - return result; + + return { ...result, traefikReloadRequired }; }), toggleEnable: protectedProcedure .input(apiFindDomain) diff --git a/apps/dokploy/server/server.ts b/apps/dokploy/server/server.ts index 5fe048c922..435ab06b17 100644 --- a/apps/dokploy/server/server.ts +++ b/apps/dokploy/server/server.ts @@ -6,6 +6,7 @@ import { IS_CLOUD, initCancelDeployments, initCronJobs, + initDomainTlsReconciliation, initEnterpriseBackupCronJobs, initializeNetwork, initSchedules, @@ -64,6 +65,7 @@ void app.prepare().then(async () => { await initCronJobs(); await initSchedules(); await initCancelDeployments(); + await initDomainTlsReconciliation(); await initVolumeBackupsCronJobs(); await sendDokployRestartNotifications(); } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 766ccdd6fb..1e3c9f40e3 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -21,6 +21,7 @@ export * from "./services/docker"; export * from "./services/docker-image"; export * from "./services/docker-volume"; export * from "./services/domain"; +export * from "./services/domain-certificate"; export * from "./services/environment"; export * from "./services/git-provider"; export * from "./services/gitea"; @@ -59,6 +60,7 @@ export * from "./services/vault-provider"; export * from "./services/volume-backups"; export * from "./services/web-server-settings"; export * from "./setup/config-paths"; +export * from "./setup/domain-tls-reconciliation"; export * from "./setup/forward-auth-setup"; export * from "./setup/monitoring-setup"; export * from "./setup/postgres-setup"; @@ -134,6 +136,7 @@ export * from "./utils/schedules/utils"; export * from "./utils/servers/remote-docker"; export * from "./utils/startup/cancel-deployments"; export * from "./utils/tracking/hubspot"; +export * from "./utils/traefik/acme"; export * from "./utils/traefik/application"; export * from "./utils/traefik/domain"; export * from "./utils/traefik/file-types"; diff --git a/packages/server/src/services/domain-certificate.ts b/packages/server/src/services/domain-certificate.ts new file mode 100644 index 0000000000..a752dcc01f --- /dev/null +++ b/packages/server/src/services/domain-certificate.ts @@ -0,0 +1,37 @@ +import { purgeAcmeCertificates } from "../utils/traefik/acme"; +import { type Domain, hasOtherLetsencryptDomainForHost } from "./domain"; + +/** + * Removes the Let's Encrypt certificate a domain no longer needs after it moved + * away from the `letsencrypt` provider, and reports whether Traefik has to be + * reloaded for the change to take effect. + * + * The host is kept when another domain record still serves it with + * `certificateType: "letsencrypt"`, otherwise that domain would lose its + * certificate. Failures are logged and reported as "no reload required": the + * domain itself was already updated, so the caller must not fail because of a + * best-effort cleanup. + */ +export const purgeStaleCertificate = async ( + domain: Domain, + serverId?: string | null, +): Promise => { + if (domain.certificateType === "letsencrypt") return false; + + try { + const stillInUse = await hasOtherLetsencryptDomainForHost( + domain.host, + domain.domainId, + ); + if (stillInUse) return false; + + const removed = await purgeAcmeCertificates([domain.host], serverId); + return removed.length > 0; + } catch (error) { + console.error( + `Could not purge the stale ACME certificate for ${domain.host}:`, + error, + ); + return false; + } +}; diff --git a/packages/server/src/services/domain.ts b/packages/server/src/services/domain.ts index c651af9fbb..c63b03bef0 100644 --- a/packages/server/src/services/domain.ts +++ b/packages/server/src/services/domain.ts @@ -5,7 +5,7 @@ import { getWebServerSettings } from "@dokploy/server/services/web-server-settin import { generateRandomDomain } from "@dokploy/server/templates"; import { manageDomain } from "@dokploy/server/utils/traefik/domain"; import { TRPCError } from "@trpc/server"; -import { eq } from "drizzle-orm"; +import { and, eq, isNotNull, ne } from "drizzle-orm"; import type { z } from "zod"; import { type apiCreateDomain, domains } from "../db/schema"; import { findApplicationById } from "./application"; @@ -146,6 +146,29 @@ export const removeDomainById = async (domainId: string) => { return result[0]; }; +/** + * True when another domain record still serves this host with Let's Encrypt. + * Removing the shared certificate would break that domain. + */ +export const hasOtherLetsencryptDomainForHost = async ( + host: string, + excludeDomainId: string, +): Promise => { + const rows = await db + .select({ domainId: domains.domainId }) + .from(domains) + .where( + and( + eq(domains.host, host), + eq(domains.certificateType, "letsencrypt"), + ne(domains.domainId, excludeDomainId), + ), + ) + .limit(1); + + return rows.length > 0; +}; + export const getDomainHost = (domain: Domain) => { return `${domain.https ? "https" : "http"}://${domain.host}`; }; @@ -210,3 +233,20 @@ export const validateDomain = async ( }; } }; + +/** + * Domains that may still carry a router config written before the TLS + * override fix. Only application domains are returned; Compose domains are + * configured through Docker labels and are regenerated on deploy. + */ +export const findDomainsNeedingTlsReconciliation = async () => { + return await db + .select() + .from(domains) + .where( + and( + ne(domains.certificateType, "letsencrypt"), + isNotNull(domains.applicationId), + ), + ); +}; diff --git a/packages/server/src/setup/domain-tls-reconciliation.ts b/packages/server/src/setup/domain-tls-reconciliation.ts new file mode 100644 index 0000000000..e3d0b991b9 --- /dev/null +++ b/packages/server/src/setup/domain-tls-reconciliation.ts @@ -0,0 +1,165 @@ +import { findApplicationById } from "../services/application"; +import { + findDomainsNeedingTlsReconciliation, + hasOtherLetsencryptDomainForHost, +} from "../services/domain"; +import { reloadDockerResource } from "../services/settings"; +import { purgeAcmeCertificates } from "../utils/traefik/acme"; +import { + loadOrCreateConfig, + loadOrCreateConfigRemote, +} from "../utils/traefik/application"; +import { manageDomain } from "../utils/traefik/domain"; +import type { FileConfig } from "../utils/traefik/file-types"; + +/** + * A websecure router written before the TLS override fix has no `tls` key at + * all, so Traefik applies the entrypoint's certResolver default to it. + */ +export const routerNeedsTlsFix = ( + config: FileConfig, + appName: string, + uniqueConfigKey: number, +): boolean => { + const router = + config.http?.routers?.[`${appName}-router-websecure-${uniqueConfigKey}`]; + if (!router) return false; + return router.tls === undefined; +}; + +/** + * One-shot pass that regenerates router configs written before the fix. + * Idempotent: once a router carries `tls: {}` it is skipped on later starts. + * + * Never rejects. It runs inside the startup sequence, ahead of the backup cron + * jobs, the restart notifications and the deployment worker, so a failure here + * must not stop any of those from being brought up. + */ +export const initDomainTlsReconciliation = async () => { + try { + await reconcileDomainTls(); + } catch (error) { + console.error("TLS reconciliation could not run:", error); + } +}; + +const reconcileDomainTls = async () => { + const domains = await findDomainsNeedingTlsReconciliation(); + if (domains.length === 0) return; + + const byApplication = new Map(); + for (const domain of domains) { + if (!domain.applicationId) continue; + const bucket = byApplication.get(domain.applicationId) ?? []; + bucket.push(domain); + byApplication.set(domain.applicationId, bucket); + } + + // Resolved once, up front, since both phases below need it: phase 1 to + // regenerate routers, phase 2 to know which server each domain's + // application lives on. An application that fails to resolve here is + // excluded from both phases. + const applications = new Map< + string, + Awaited> + >(); + for (const applicationId of byApplication.keys()) { + try { + applications.set(applicationId, await findApplicationById(applicationId)); + } catch (error) { + console.error( + `TLS reconciliation could not resolve application ${applicationId}:`, + error, + ); + } + } + + // Phase 1: regenerate router configs written before the TLS override fix. + for (const [applicationId, appDomains] of byApplication) { + const application = applications.get(applicationId); + if (!application) continue; + + try { + const config = application.serverId + ? await loadOrCreateConfigRemote( + application.serverId, + application.appName, + ) + : loadOrCreateConfig(application.appName); + + const stale = appDomains.filter((domain) => + routerNeedsTlsFix(config, application.appName, domain.uniqueConfigKey), + ); + if (stale.length === 0) continue; + + for (const domain of stale) { + await manageDomain(application, domain); + } + + console.log( + `Reconciled TLS config for ${stale.length} domain(s) on ${application.appName}`, + ); + } catch (error) { + // One unreachable remote server must not stop the rest. + console.error( + `TLS reconciliation failed for application ${applicationId}:`, + error, + ); + } + } + + // Phase 2: purge stale acme.json entries, independent of whether the + // router for that domain still needed regeneration. A router already + // carrying `tls: {}` is exactly the state `routerNeedsTlsFix` treats as + // "nothing to do" in phase 1, so it is the only place left where a + // certificate purge that was skipped or lost a race against Traefik on an + // earlier boot gets retried. Grouped per server, not per application, + // because acme.json is one file per server: this keeps the read/rewrite + // and any remote SSH round trip to once per server for the whole pass. + const byServer = new Map(); + for (const domain of domains) { + if (!domain.applicationId) continue; + const application = applications.get(domain.applicationId); + if (!application) continue; + const serverKey = application.serverId ?? ""; + const bucket = byServer.get(serverKey) ?? []; + bucket.push(domain); + byServer.set(serverKey, bucket); + } + + for (const [serverKey, serverDomains] of byServer) { + const serverId = serverKey || undefined; + try { + // A host still served by another Let's Encrypt domain keeps its + // certificate, exactly as the mutation path does. + const hostToDomainId = new Map(); + for (const domain of serverDomains) { + if (!hostToDomainId.has(domain.host)) { + hostToDomainId.set(domain.host, domain.domainId); + } + } + + const purgeableHosts: string[] = []; + for (const [host, domainId] of hostToDomainId) { + const stillInUse = await hasOtherLetsencryptDomainForHost( + host, + domainId, + ); + if (!stillInUse) purgeableHosts.push(host); + } + + if (purgeableHosts.length === 0) continue; + + const removed = await purgeAcmeCertificates(purgeableHosts, serverId); + if (removed.length > 0) { + await reloadDockerResource("dokploy-traefik", serverId); + } + } catch (error) { + // One unreachable remote server must not stop the rest. + console.error( + `TLS reconciliation could not purge certificates for server ${serverKey || "local"}:`, + error, + ); + } + } +}; diff --git a/packages/server/src/utils/docker/domain.ts b/packages/server/src/utils/docker/domain.ts index 0d83d4a48d..5565bd7f4f 100644 --- a/packages/server/src/utils/docker/domain.ts +++ b/packages/server/src/utils/docker/domain.ts @@ -500,9 +500,10 @@ export const createDomainLabels = ( labels.push( `traefik.http.routers.${routerName}.tls.certresolver=${customCertResolver}`, ); - } else if (certificateType === "none" && https) { + } else if (https) { // No cert resolver, but HTTPS is enabled (default/custom certificate): - // explicitly enable TLS so Traefik serves the router over HTTPS. + // explicitly enable TLS so Traefik serves the router over HTTPS and + // does not inherit the entrypoint's certResolver default. labels.push(`traefik.http.routers.${routerName}.tls=true`); } } diff --git a/packages/server/src/utils/traefik/acme.ts b/packages/server/src/utils/traefik/acme.ts new file mode 100644 index 0000000000..443638a9c1 --- /dev/null +++ b/packages/server/src/utils/traefik/acme.ts @@ -0,0 +1,187 @@ +import fs from "node:fs"; +import path from "node:path"; +import { paths } from "@dokploy/server/constants"; +import { encodeBase64 } from "../docker/utils"; +import { execAsyncRemote } from "../process/execAsync"; + +export interface AcmeCertificate { + domain: { main: string; sans?: string[] }; + certificate: string; + key: string; + Store?: string; +} + +export interface AcmeResolver { + Account?: unknown; + Certificates?: AcmeCertificate[] | null; +} + +export type AcmeStore = Record; + +/** + * Returns a copy of the ACME store with the certificates for the given hosts + * removed. The Account object of each resolver is preserved, so Traefik keeps + * its ACME registration. + */ +export const removeAcmeCertificates = ( + store: AcmeStore, + hosts: string[], +): { store: AcmeStore; removed: string[] } => { + const targets = new Set(hosts); + const removed: string[] = []; + const next: AcmeStore = {}; + + for (const [resolverName, resolver] of Object.entries(store)) { + const certificates = resolver.Certificates ?? []; + const kept: AcmeCertificate[] = []; + + for (const certificate of certificates) { + if (targets.has(certificate.domain.main)) { + removed.push(certificate.domain.main); + } else { + kept.push(certificate); + } + } + + next[resolverName] = { ...resolver, Certificates: kept }; + } + + return { store: next, removed }; +}; + +const acmeJsonPath = (isRemote: boolean) => { + const { DYNAMIC_TRAEFIK_PATH } = paths(isRemote); + return path.join(DYNAMIC_TRAEFIK_PATH, "acme.json"); +}; + +const purgeChains = new Map>(); + +/** + * Runs purges for one server one at a time. A purge is a read-modify-write over + * a file Traefik owns, so two concurrent calls for the same server would both + * read the old store and the later write would resurrect whatever the earlier + * one removed. Serialising in process is the practical boundary here, since a + * server's acme.json is only ever written by the Dokploy instance managing it. + */ +const withPurgeLock = (key: string, task: () => Promise): Promise => { + const previous = purgeChains.get(key) ?? Promise.resolve(); + // Run whether or not the previous purge succeeded, otherwise one failure + // would block every later purge for that server. + const result = previous.then(task, task); + purgeChains.set( + key, + result.then( + () => undefined, + () => undefined, + ), + ); + return result; +}; + +/** + * Removes the ACME certificates for the given hosts and returns the hosts that + * were actually removed. The caller is responsible for reloading Traefik: + * acme.json is only read at startup, so an on-disk change has no effect until + * the container restarts. + */ +export const purgeAcmeCertificates = async ( + hosts: string[], + serverId?: string | null, +): Promise => { + if (hosts.length === 0) return []; + + return withPurgeLock(serverId ?? "", () => + purgeAcmeCertificatesUnsynchronised(hosts, serverId), + ); +}; + +const readAcmeStoreRaw = async ( + filePath: string, + serverId?: string | null, +): Promise => { + if (serverId) { + const { stdout } = await execAsyncRemote( + serverId, + `cat ${filePath} 2>/dev/null || true`, + ); + return stdout; + } + if (!fs.existsSync(filePath)) return null; + return fs.readFileSync(filePath, "utf8"); +}; + +const PURGE_ATTEMPTS = 3; + +const purgeAcmeCertificatesUnsynchronised = async ( + hosts: string[], + serverId?: string | null, +): Promise => { + const filePath = acmeJsonPath(!!serverId); + + for (let attempt = 1; attempt <= PURGE_ATTEMPTS; attempt++) { + const raw = await readAcmeStoreRaw(filePath, serverId); + if (raw === null || !raw.trim()) return []; + + let parsed: AcmeStore; + try { + parsed = JSON.parse(raw) as AcmeStore; + } catch { + // A malformed store is Traefik's to repair, not ours to overwrite. + return []; + } + + const { store, removed } = removeAcmeCertificates(parsed, hosts); + if (removed.length === 0) return []; + + // Traefik owns this file and rewrites it in full whenever it issues or + // renews a certificate, without any locking protocol we can join. Check + // the store still looks the way we read it before swapping ours in, so a + // certificate Traefik wrote while we were working is not dropped. This + // narrows the window to the final round trip rather than closing it, + // which is the best available short of stopping Traefik to edit its + // store. If the certificate we purge is reinstated by such a write, the + // startup reconciliation purges it again on the next boot. + const current = await readAcmeStoreRaw(filePath, serverId); + if (current !== raw) continue; + + const serialized = JSON.stringify(store, null, 2); + + // Never write in place: a truncated acme.json costs every Let's Encrypt + // certificate on the server. Write a sibling temp file, lock it down to + // 0600 (Traefik refuses to start on anything more permissive) and rename + // it over the target, which is atomic within the same directory. + const tempPath = `${filePath}.dokploy.tmp`; + + if (serverId) { + await execAsyncRemote( + serverId, + `umask 077 && printf '%s' "${encodeBase64(serialized)}" | base64 -d > ${tempPath} && chmod 600 ${tempPath} && mv -f ${tempPath} ${filePath} || { rm -f ${tempPath}; exit 1; }`, + ); + } else { + try { + fs.writeFileSync(tempPath, serialized, { + encoding: "utf8", + mode: 0o600, + }); + // writeFileSync's mode is subject to the process umask, and it is + // ignored entirely when the temp file already exists. + fs.chmodSync(tempPath, 0o600); + fs.renameSync(tempPath, filePath); + } catch (error) { + try { + fs.unlinkSync(tempPath); + } catch { + // The temp file may never have been created. + } + throw error; + } + } + + return removed; + } + + console.warn( + `Skipped purging ${hosts.join(", ")} from acme.json: Traefik kept rewriting it`, + ); + return []; +}; diff --git a/packages/server/src/utils/traefik/domain.ts b/packages/server/src/utils/traefik/domain.ts index b23796301e..8c347f972a 100644 --- a/packages/server/src/utils/traefik/domain.ts +++ b/packages/server/src/utils/traefik/domain.ts @@ -224,8 +224,11 @@ export const createRouterConfig = async ( routerConfig.tls = { certResolver: "letsencrypt" }; } else if (certificateType === "custom" && domain.customCertResolver) { routerConfig.tls = { certResolver: domain.customCertResolver }; - } else if (certificateType === "none") { - routerConfig.tls = undefined; + } else { + // An empty object still enables TLS, but marks the router's TLS as + // non-nil so Traefik does not fall back to the entrypoint's + // certResolver default. Same handling as forward-auth.ts. + routerConfig.tls = {}; } }