From 96e84f399734dceb07ed40707b0a7682bf7a78f8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 25 Aug 2026 07:13:55 -0700 Subject: [PATCH 1/5] fix(certbot): reissue when the configured domains change --- CHANGELOG.md | 1 + dstack/certbot/src/acme_client.rs | 155 +++++++++++++++++++++++++++++- dstack/certbot/src/bot.rs | 44 +++++++-- 3 files changed, 188 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3d358564..d65e8c1a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - certbot: a certificate covering both a name and its wildcard (`example.com` and `*.example.com`) could never be issued over dns-01. The two authorizations are answered under one `_acme-challenge.example.com`, each with its own TXT value, and the publish step cleared every TXT record at that name before writing its own -- so the second authorization deleted the record answering the first, and the order failed with `Correct value not found for DNS challenge`. Clearing leftovers from an aborted run is now done once per challenge name per issuance, and the records for one name accumulate instead of replacing each other; cleanup afterwards is unchanged, deleting each record this run created by id +- certbot: editing `domains` in `certbot.toml` had no effect once a certificate existed. Issuance was skipped whenever `live/cert.pem` was present, whatever names it carried, and renewal read its name list back off that certificate rather than the configuration -- so an added or removed name never reached the CA, and the mismatch survived every renewal. The live certificate's DNS names are now compared against the configured list (as sets, case- and trailing-dot-insensitive) and a mismatch reissues, logging both lists. A reissue that fails does not take the renewal check down with it: a name the CA will not validate is reported on every cycle, while the certificate actually being served keeps renewing, and the failure is still what the run returns unless the renewal committed something of its own - gateway: a node removed via `RemoveNode` silently rejoined the cluster the next time it started, because every node re-registers its own sync address on boot. Once tombstone GC is collecting, that comeback is worse than an annoyance: a stale data directory diverges from every digest, and the divergence repair's full re-exchange resurrects records whose deletes the cluster already collected. Removal now writes a durable marker — a live record, so the GC can never eat it — that every gateway's sync endpoints enforce; a removed node's envelopes are refused until an operator re-admits it with `SetNodeUrl`. The refused node counts HTTP 403 sync rejections (`dstack_gateway_sync_rejected_total`), including removal lockouts and app-identity mismatches. Every gateway also exposes `dstack_gateway_node_last_seen_timestamp_seconds` per known node, so a long-offline gateway is a one-line alert instead of an ack-watermark puzzle - gateway: deleted KV records left a tombstone that nothing ever collected, so every deregistered CVM stayed on disk for the life of the deployment. Tombstones every peer has acknowledged are now dropped once every `tombstone_gc_writes` replicated writes (default 10000, zero disables); the trigger counts replicated writes rather than reading a clock, so nodes in a cluster collect in the same window without depending on time synchronization. A `SetTombstoneGcConfig` admin RPC stores an operator override in the KV itself, replicating one pace to every node - gateway: `Admin.RemoveCvm` now reports the outcome of the `inst/` tombstone alone. A failure to delete associated override or telemetry records is logged instead of failing the call, so a removal that did take effect is no longer reported as failed — which also aborted the local routing cleanup that follows it. Re-issuing a removal still sweeps up override and telemetry records orphaned by an earlier partial failure diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs index acc1a8907..6288303b9 100644 --- a/dstack/certbot/src/acme_client.rs +++ b/dstack/certbot/src/acme_client.rs @@ -290,7 +290,10 @@ impl AcmeClient { Ok(()) } - /// Auto renew given certificate + /// Issue a certificate for `domains` unless the live one is already for + /// exactly those names. + /// + /// Returns whether a certificate was issued. pub async fn create_cert_if_needed( &self, domains: &[String], @@ -299,7 +302,28 @@ impl AcmeClient { backup_dir: impl AsRef, ) -> Result { if live_cert_pem_path.as_ref().exists() && live_key_pem_path.as_ref().exists() { - return Ok(false); + // The live certificate is only "the certificate we were asked for" + // if it carries the configured names. Treating its mere existence as + // sufficient pinned the name list at whatever the first issuance + // used: an operator adding a name to the configuration got no new + // certificate, and renewal read its names back off the old one, so + // the edit never took effect at all. + let reason = match fs::read_to_string(live_cert_pem_path.as_ref()) { + Ok(live_cert_pem) => reissue_reason(&live_cert_pem, domains), + // `exists()` has already passed, so this is a permission + // problem or a file replaced mid-flight rather than a missing + // certificate. Either way the certificate cannot be checked + // against the configuration, which is a reissue for the same + // reason an unparseable one is. + Err(err) => Some(format!( + "cannot read {}: {err:#}", + live_cert_pem_path.as_ref().display() + )), + }; + match reason { + None => return Ok(false), + Some(reason) => info!("reissuing: {reason}"), + } } let key_pem = if live_key_pem_path.as_ref().exists() { debug!("using existing cert key pair"); @@ -798,6 +822,39 @@ pub(crate) fn read_pem(cert_pem: &str) -> Result { .context("no certificate in pem") } +/// Why the live certificate has to be reissued, or `None` if it is the one the +/// configuration asks for. +/// +/// A certificate whose names cannot be read counts as a reissue: it cannot be +/// checked against the configuration, so leaving it in place would keep serving +/// something this process can no longer reason about. +fn reissue_reason(live_cert_pem: &str, domains: &[String]) -> Option { + match extract_subject_alt_names(live_cert_pem) { + Ok(names) if names_match(&names, domains) => None, + Ok(names) => Some(format!( + "the live certificate covers {}, the configuration asks for {}", + names.join(", "), + domains.join(", ") + )), + Err(err) => Some(format!("cannot read the live certificate's names: {err:#}")), + } +} + +/// Whether a certificate's DNS names are exactly the configured ones. +/// +/// Compared as sets: the order the CA returns names in is its own business, and +/// DNS names are case-insensitive and may carry a trailing root dot. Equality +/// rather than containment, so narrowing the configured list reissues too. +fn names_match(cert_names: &[String], domains: &[String]) -> bool { + fn normalized(names: &[String]) -> BTreeSet { + names + .iter() + .map(|name| name.trim_end_matches('.').to_ascii_lowercase()) + .collect() + } + normalized(cert_names) == normalized(domains) +} + fn extract_subject_alt_names(cert_pem: &str) -> Result> { let pem = read_pem(cert_pem)?; let cert = pem.parse_x509().context("Invalid x509 certificate")?; @@ -977,3 +1034,97 @@ mod purge_tests { assert!(needs_purge(&published, "_acme-challenge.example.org")); } } + +/// The owned name list the functions under test take, spelled once. +#[cfg(test)] +fn names(list: &[&str]) -> Vec { + list.iter().map(|name| name.to_string()).collect() +} + +#[cfg(test)] +mod names_match_tests { + use super::{names, names_match}; + + /// The CA returns the names in whatever order it likes -- the staging CA + /// puts the wildcard first -- and DNS names are case-insensitive and may + /// carry the root dot. None of that is a configuration change. + #[test] + fn the_same_names_written_differently_are_the_same_names() { + assert!(names_match( + &names(&["*.example.com", "Example.com."]), + &names(&["example.com", "*.example.com"]), + )); + } + + /// Adding a name to the configuration is what has to trigger a reissue -- + /// this is the case that silently did nothing before. + #[test] + fn an_added_name_does_not_match() { + assert!(!names_match( + &names(&["example.com"]), + &names(&["example.com", "*.example.com"]), + )); + } + + /// Dropping a name must reissue as well: a certificate covering more than + /// the configuration asks for is not the certificate that was asked for. + #[test] + fn a_dropped_name_does_not_match() { + assert!(!names_match( + &names(&["example.com", "*.example.com"]), + &names(&["example.com"]), + )); + } +} + +#[cfg(test)] +mod reissue_reason_tests { + use super::{names, reissue_reason}; + use rcgen::{CertificateParams, KeyPair}; + + /// A certificate carrying exactly `sans`, in the order given -- the CA's own + /// order is not the configuration's, which is what the decision has to + /// tolerate. + fn cert_with_names(sans: &[&str]) -> String { + let key = KeyPair::generate().expect("failed to generate key"); + CertificateParams::new(names(sans)) + .expect("failed to build certificate params") + .self_signed(&key) + .expect("failed to self-sign") + .pem() + } + + #[test] + fn the_configured_certificate_is_kept() { + let cert = cert_with_names(&["*.example.com", "example.com"]); + assert_eq!( + reissue_reason(&cert, &names(&["example.com", "*.example.com"])), + None + ); + } + + /// The case that silently did nothing before: a name added to the + /// configuration has to reach the CA. + #[test] + fn a_certificate_missing_a_configured_name_is_reissued() { + let cert = cert_with_names(&["example.com"]); + let reason = reissue_reason(&cert, &names(&["example.com", "*.example.com"])) + .expect("an added name must reissue"); + // Both lists are in the message: the operator has to be able to see + // what is being replaced and why. + assert!(reason.contains("covers example.com"), "{reason}"); + assert!( + reason.contains("asks for example.com, *.example.com"), + "{reason}" + ); + } + + /// A certificate whose names cannot be read cannot be checked against the + /// configuration either, so it is replaced rather than served on. + #[test] + fn an_unreadable_certificate_is_reissued() { + let reason = reissue_reason("not a certificate", &names(&["example.com"])) + .expect("an unreadable certificate must reissue"); + assert!(reason.contains("cannot read"), "{reason}"); + } +} diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs index 06eb86cf3..0daf499e3 100644 --- a/dstack/certbot/src/bot.rs +++ b/dstack/certbot/src/bot.rs @@ -190,7 +190,8 @@ impl CertBot { } async fn renew_inner(&self, force: bool) -> Result { - let created = self + let live_cert_exists = self.config.cert_file.exists() && self.config.key_file.exists(); + let issued = self .acme_client .create_cert_if_needed( &self.config.cert_subject_alt_names, @@ -198,11 +199,30 @@ impl CertBot { &self.config.key_file, &self.config.cert_dir, ) - .await?; - if created { - info!("created new certificate"); - return Ok(true); - } + .await; + // A live certificate that does not cover the configured names is + // reissued above, and that reissuance keeps failing for as long as the + // configuration names something the CA will not validate -- a typo, a + // zone the DNS credentials cannot write. Failing the run right here + // would take the renewal check below down with it, so one name the + // operator got wrong would stop renewing the certificate that is + // actually being served, until it expires. The renewal still runs; the + // error is reported, and returned below unless the renewal committed + // something of its own. + let reissue_error = match issued { + Ok(true) => { + info!("created new certificate"); + return Ok(true); + } + Ok(false) => None, + // Nothing is being served yet, so there is no renewal to protect + // and `auto_renew` has no certificate to read. + Err(err) if !live_cert_exists => return Err(err), + Err(err) => { + error!("failed to issue a certificate for the configured domains: {err:#}"); + Some(err) + } + }; info!("checking if certificate needs to be renewed"); let renewed = self .acme_client @@ -215,21 +235,25 @@ impl CertBot { ) .await?; - match renewed { - true => { + match (renewed, reissue_error) { + (true, _) => { info!( "renewed certificate for {}", self.config.cert_file.display() ); + Ok(true) } - false => { + // The renewal committed nothing, so the reissue failure is the + // whole outcome of this run and `renew --once` must report it. + (false, Some(err)) => Err(err), + (false, None) => { info!( "certificate {} is up to date", self.config.cert_file.display() ); + Ok(false) } } - Ok(renewed) } /// Set CAA record for the domain. From da36db82d34f0f8dc66bedb5d74be31800cecb3e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 25 Aug 2026 20:23:30 -0700 Subject: [PATCH 2/5] fix(certbot-cli): attach the config template's comments to the right keys `certbot cfg` walks the serialized document and the struct's doc comments in lockstep, by position. An `Option` that is `None` serializes to nothing, so `cf_api_url` and `renewed_hook` are absent from the document and every comment after the first of them describes the key above the one it belongs to: # Optional Cloudflare-compatible API base URL dns_txt_ttl = 60 # TTL for DNS TXT challenge records in seconds auto_set_caa = true Look the comment up by key name instead. A key with no doc comment keeps none rather than borrowing its neighbour's. --- CHANGELOG.md | 1 + dstack/certbot/cli/src/main.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d65e8c1a3..e66ad168a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - certbot: a certificate covering both a name and its wildcard (`example.com` and `*.example.com`) could never be issued over dns-01. The two authorizations are answered under one `_acme-challenge.example.com`, each with its own TXT value, and the publish step cleared every TXT record at that name before writing its own -- so the second authorization deleted the record answering the first, and the order failed with `Correct value not found for DNS challenge`. Clearing leftovers from an aborted run is now done once per challenge name per issuance, and the records for one name accumulate instead of replacing each other; cleanup afterwards is unchanged, deleting each record this run created by id - certbot: editing `domains` in `certbot.toml` had no effect once a certificate existed. Issuance was skipped whenever `live/cert.pem` was present, whatever names it carried, and renewal read its name list back off that certificate rather than the configuration -- so an added or removed name never reached the CA, and the mismatch survived every renewal. The live certificate's DNS names are now compared against the configured list (as sets, case- and trailing-dot-insensitive) and a mismatch reissues, logging both lists. A reissue that fails does not take the renewal check down with it: a name the CA will not validate is reported on every cycle, while the certificate actually being served keeps renewing, and the failure is still what the run returns unless the renewal committed something of its own +- certbot: `certbot cfg` attached every comment after `cf_api_url` to the wrong key, because an absent optional field shifts the generated document's keys out of step with the struct's. The template's documentation is now looked up by key name - gateway: a node removed via `RemoveNode` silently rejoined the cluster the next time it started, because every node re-registers its own sync address on boot. Once tombstone GC is collecting, that comeback is worse than an annoyance: a stale data directory diverges from every digest, and the divergence repair's full re-exchange resurrects records whose deletes the cluster already collected. Removal now writes a durable marker — a live record, so the GC can never eat it — that every gateway's sync endpoints enforce; a removed node's envelopes are refused until an operator re-admits it with `SetNodeUrl`. The refused node counts HTTP 403 sync rejections (`dstack_gateway_sync_rejected_total`), including removal lockouts and app-identity mismatches. Every gateway also exposes `dstack_gateway_node_last_seen_timestamp_seconds` per known node, so a long-offline gateway is a one-line alert instead of an ack-watermark puzzle - gateway: deleted KV records left a tombstone that nothing ever collected, so every deregistered CVM stayed on disk for the life of the deployment. Tombstones every peer has acknowledged are now dropped once every `tombstone_gc_writes` replicated writes (default 10000, zero disables); the trigger counts replicated writes rather than reading a clock, so nodes in a cluster collect in the same window without depending on time synchronization. A `SetTombstoneGcConfig` admin RPC stores an operator override in the KV itself, replicating one pace to every node - gateway: `Admin.RemoveCvm` now reports the outcome of the `inst/` tombstone alone. A failure to delete associated override or telemetry records is logged instead of failing the call, so a removal that did take effect is no longer reported as failed — which also aborted the local routing cleanup that follows it. Re-issuing a removal still sweeps up override and telemetry records orphaned by an earlier partial failure diff --git a/dstack/certbot/cli/src/main.rs b/dstack/certbot/cli/src/main.rs index 4a20aacf8..38ed8dd55 100644 --- a/dstack/certbot/cli/src/main.rs +++ b/dstack/certbot/cli/src/main.rs @@ -112,9 +112,16 @@ impl Config { fn to_commented_toml(&self) -> Result { let mut doc = to_document(self)?; - for (i, (mut key, _value)) in doc.iter_mut().enumerate() { + for (mut key, _value) in doc.iter_mut() { + // Look the doc comment up by name rather than by position: a `None` + // option serializes to nothing, so the document's keys are a subset + // of the struct's fields and indexing `FIELD_DOCS` positionally + // attaches every comment after the first absent key to the wrong + // one. + let Ok(docstring) = Self::get_field_docs(key.get()) else { + continue; + }; let decor = key.leaf_decor_mut(); - let docstring = Self::FIELD_DOCS[i]; let mut comment = String::new(); for line in docstring.lines() { From a896a2594d2f308d53b1faaf1bfccf6badc00352 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 25 Aug 2026 20:23:35 -0700 Subject: [PATCH 3/5] docs(certbot-cli): describe the CLI as a testing tool The CLI keeps the ACME account key and the certificate key in a plain directory on the host that runs it, so nothing about the issuance is attested. A deployed gateway issues its own certificates from the configuration it already holds and publishes the public keys for `ct_monitor` to check. Say so in a README, next to what the subcommands do and where the workdir keeps things. `init`'s help said it initializes the configuration file; it reads the configuration file and creates the ACME account. --- dstack/certbot/cli/README.md | 52 ++++++++++++++++++++++++++++++++++ dstack/certbot/cli/src/main.rs | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 dstack/certbot/cli/README.md diff --git a/dstack/certbot/cli/README.md b/dstack/certbot/cli/README.md new file mode 100644 index 000000000..70a0240d9 --- /dev/null +++ b/dstack/certbot/cli/README.md @@ -0,0 +1,52 @@ +# certbot + +A small ACME client that issues and renews certificates over dns-01, with Cloudflare +as the DNS provider. It is a testing and development tool: it exercises the `certbot` +crate that dstack-gateway builds on, and it obtains a certificate by hand for a local +or staging setup. + +It is not the production path. A deployed gateway issues its own certificates from the +domain configuration and DNS credentials it already holds, publishes the resulting +public keys over `/acme-info` for `ct_monitor` to check the CT logs against, and — when +that gateway runs in a CVM — keeps both the ACME account key and the certificate key +inside the enclave. This CLI keeps them in a plain directory on whatever host runs it, +so nothing about the issuance is attested and the key is only as protected as that +filesystem. Point it at Pebble or Let's Encrypt staging while working on the ACME code, +not at the certificate fronting a real deployment. + +## Usage + +Write a configuration template, fill in the Cloudflare token and the names, then run +the daemon: + +```bash +certbot cfg --write-to certbot.toml +$EDITOR certbot.toml +RUST_LOG=info,certbot=debug certbot renew -c certbot.toml +``` + +`renew` issues a certificate if none is live, reissues when the live one does not carry +the configured `domains`, and renews once the live one is within `renew_days_before` of +expiry. It loops every `renew_interval` seconds; `--once` runs a single pass and exits, +and `--force` renews whether or not expiry is near. + +`init` creates the ACME account — and, with `auto_set_caa`, the CAA records — without +issuing anything. `set-caa` writes the CAA records for the configured names on their +own, which is worth running after adding a name to `domains`: the automatic pass only +happens when the account is created. + +## Configuration + +`certbot cfg` prints every field with its documentation. Two optional fields are absent +from that template because they default to nothing: + +- `cf_api_url` — a Cloudflare-compatible API base URL, for pointing the DNS calls at a + mock server instead of Cloudflare. +- `renewed_hook` — a shell command run after a certificate is committed, e.g. to reload + whatever is serving it. + +`workdir` holds everything else. Each issuance lands in its own timestamped directory +under `backup/`, `live/cert.pem` and `live/key.pem` are symlinks to the one in force, +and the ACME account credentials sit in `credentials.json`. Editing `domains` takes +effect on the next run: the live certificate's names are compared against the +configuration, and a mismatch reissues. diff --git a/dstack/certbot/cli/src/main.rs b/dstack/certbot/cli/src/main.rs index 38ed8dd55..68348b5f9 100644 --- a/dstack/certbot/cli/src/main.rs +++ b/dstack/certbot/cli/src/main.rs @@ -28,7 +28,7 @@ enum Command { #[arg(long)] force: bool, }, - /// Initialize the configuration file + /// Create the ACME account described by the configuration file Init { /// Path to the configuration file #[arg(short, long, default_value = "certbot.toml")] From 4cab90ad27f6cff82536c388a4c5943c55b27dc7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 25 Aug 2026 20:57:28 -0700 Subject: [PATCH 4/5] docs(deployment): the gateway issues its own certificates Step 4 told the operator to configure `GATEWAY_CERT`/`GATEWAY_KEY` in `build-config.sh` and run `./certbot renew -c certbot.toml` on the host. That is not how a deployed gateway gets a certificate, and `build-config.sh` is not part of this guide's flow: `dstack-gateway` links the `certbot` crate and runs it inside the CVM, answering dns-01 with the DNS credential in its admin config and keeping the ACME account key and every certificate in the CVM's WaveKV store. Following the old text produced a certificate on the host that nothing reads. Describe what step 3 already set up -- `bootstrap-cluster.sh` calling SetCertbotConfig, CreateDnsCredential and AddZtDomain -- how to watch issuance land in `ListZtDomains`, how to pin CAA, and what switching from staging to production takes. Zero-trust HTTPS moves to the required part of the checklist: without a certificate for the domain the gateway cannot serve an app over TLS. --- docs/deployment.md | 68 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index dec73fe4b..7f774baba 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -42,10 +42,10 @@ If you skip the KMS allowlist step, the VM may boot and the onboard UI may still 1. Set up TDX host with dstack-vmm 2. Deploy KMS as CVM (with auth server, capture its attestation info, and allowlist the KMS `mrAggregated` before bootstrap) 3. Deploy Gateway as CVM +4. [Zero Trust HTTPS](#4-zero-trust-https) - the gateway cannot serve an app over TLS until it holds a certificate for the domain **Optional Add-ons:** -4. [Zero Trust HTTPS](#4-zero-trust-https-optional) 5. [Certificate Transparency monitoring](#5-certificate-transparency-monitoring-optional) 6. [Multi-node deployment](#6-multi-node-deployment-optional) 7. [On-chain governance](./onchain-governance.md) - Smart contract-based authorization @@ -355,29 +355,69 @@ Restart dstack-vmm to apply changes. --- -### 4. Zero Trust HTTPS (Optional) +### 4. Zero Trust HTTPS -Generate TLS certificates inside the TEE with automatic CAA record management. +The gateway issues its own certificates from inside the CVM. `dstack-gateway` +links the `certbot` crate directly: it answers dns-01 challenges with the +Cloudflare credential you give it, and keeps the ACME account key and every +certificate in the CVM's WaveKV store. Neither key is ever written to the host, +which is what makes the monitoring in step 5 worth running — every certificate +the CT logs show for your domain should carry a public key the gateway +published. Nothing on the host issues or holds these certificates; the `certbot` +CLI under `dstack/certbot/cli` is a testing tool for the same crate and has no +part in this path. -Configure in `build-config.sh`: +`bootstrap-cluster.sh` configures all of it, reading `CF_API_TOKEN`, +`SRV_DOMAIN` and `ACME_STAGING` from the `.env` you filled in during step 3 and +calling the gateway's admin API: ```bash -GATEWAY_CERT=${CERTBOT_WORKDIR}/live/cert.pem -GATEWAY_KEY=${CERTBOT_WORKDIR}/live/key.pem -CF_API_TOKEN= -ACME_URL=https://acme-v02.api.letsencrypt.org/directory +cd dstack/gateway/dstack-app/ +bash bootstrap-cluster.sh ``` -Run certbot: +| RPC | What it sets | +|---|---| +| `SetCertbotConfig` | ACME directory URL and the renewal schedule | +| `CreateDnsCredential` | the Cloudflare token used for dns-01, as the default credential | +| `AddZtDomain` | a domain to keep a wildcard certificate for | + +Run it once per cluster. Additional nodes receive all three through cluster +sync, so do not repeat it per node. + +Every name the gateway terminates TLS on needs its own ZT domain. The script +adds `SRV_DOMAIN`; if app URLs sit one level deeper — `-.gateway.example.com` +— then `gateway.example.com` needs an entry of its own, because a wildcard +certificate for `*.example.com` does not cover subdomains of subdomains. + +Certificates are requested on the next renewal round rather than the moment a +domain is added. Watch for them to arrive: ```bash -RUST_LOG=info,certbot=debug ./certbot renew -c certbot.toml +ADMIN_ADDR=127.0.0.1:9203 # GATEWAY_ADMIN_RPC_ADDR in .env +curl -sf -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + "http://$ADMIN_ADDR/prpc/ListZtDomains" | jq '.domains[] | {domain: .config.domain, cert: .cert_status}' ``` -This will: -- Create an ACME account -- Set CAA DNS records on Cloudflare -- Request and auto-renew certificates +`has_cert: true` with a `not_after` roughly 90 days out means the domain is +served. `POST /prpc/RenewCert` forces a round immediately instead of waiting +for `renew_interval_secs`. + +Pin issuance to your own ACME account with `POST /prpc/SetCaa`, which writes +CAA records naming Let's Encrypt and the gateway's account URI for every +configured domain. Any other account is then refused by the CA rather than +merely noticed after the fact by step 5. + +Start on Let's Encrypt staging (`ACME_STAGING=yes`), whose certificates are not +browser-trusted but whose rate limits leave room for mistakes. Switching to +production takes two calls: `SetCertbotConfig` with the production directory +URL, then `RotateAcmeCredentials` to register an account there and re-pin every +domain's CAA to it. Renewals refuse to run while the stored account and the +configured ACME URL disagree, so do not skip the rotation. + +For the same flow driven by hand, one curl at a time, see the +[Gateway service setup tutorial](./tutorials/gateway-service-setup.md#step-4-bootstrap-admin-api). +For a multi-node cluster, see [Cluster deployment](../dstack/gateway/docs/cluster-deployment.md). --- From 5fe49034313029cab499a7df6b47ccd73ac5ac3c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 25 Aug 2026 21:13:32 -0700 Subject: [PATCH 5/5] docs(gateway): the host-mode gateway issues its own certificates too The production setup guide walked the operator through editing `certbot.toml`, running `./certbot set-caa && ./certbot renew` by hand, and pointing `cert_chain`/`cert_key` at the result. A gateway running on the host links the same `certbot` crate as one running in a CVM and issues over dns-01 from its own process, keeping the ACME account key and every certificate in its WaveKV store -- `cert_chain`/`cert_key` only load a certificate something else produced. Configure the gateway instead: the proxy fields, the admin API the ACME settings are stored through, and a `data_dir` that survives restarts so a restart does not ask the CA for a fresh certificate. Then SetCertbotConfig, CreateDnsCredential and AddZtDomain, with what to watch in ListZtDomains, what SetCaa pins, and what switching from staging to production takes. Also: `core.admin.auth_token` is the current key name (`admin_token` is the accepted older one), URL Format becomes its own section rather than sitting inside a configuration step, and the alerting table listed `dstack_gateway_kv_persist_failures_total` twice. --- docs/dstack-gateway.md | 136 ++++++++++++++++++++++++++++++++--------- 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/docs/dstack-gateway.md b/docs/dstack-gateway.md index 58e59a1bc..fe4992b29 100644 --- a/docs/dstack-gateway.md +++ b/docs/dstack-gateway.md @@ -2,41 +2,125 @@ > **This guide is for self-hosted deployments** on your own TDX hardware. For cloud deployments, see [Quickstart](./quickstart.md). -To set up dstack-gateway for production, you need a wildcard domain and SSL certificate. +To set up dstack-gateway for production, you need a wildcard domain and a +Cloudflare API token. You do not need to obtain a certificate yourself: the +gateway links the `certbot` crate and runs ACME over dns-01 in its own process, +keeping the ACME account key and every certificate in its WaveKV store. The +`certbot` CLI under `dstack/certbot/cli` is a testing tool for the same crate +and has no part in this path. -## Step 1: Setup wildcard domain +## Step 1: Set up the wildcard domain Set up a second-level wildcard domain using Cloudflare; make sure to disable proxy mode and use **DNS Only**. ![add-wildcard-domain](./assets/tproxy-add-wildcard-domain.jpg) -## Step 2: Request a Wildcard Domain SSL Certificate with Certbot +Then create an API token that can edit this zone's DNS records. The gateway uses +it to publish the `_acme-challenge` TXT records that answer dns-01, and the CAA +records that pin issuance to its own ACME account. -You need to get a Cloudflare API Key and ensure the API can manage this domain. +## Step 2: Configure `gateway.toml` -Open your `certbot.toml`, and update these fields: +Focus on these fields in the `core.proxy` section: -- `acme_url`: change to `https://acme-v02.api.letsencrypt.org/directory` -- `cf_api_token`: Obtain from Cloudflare +- `base_domain`: the wildcard domain for the proxy +- `listen_addr` & `listen_port`: listen on `0.0.0.0` and preferably `443` in production. If using another port, specify it in the URL (see [URL Format](#url-format)) -## Step 3: Run Certbot Manually and Get First SSL Certificates +For example, if your base domain is `gateway.example.com`, app ID is ``, listening on `80`, and dstack-gateway is on port 7777, the URL would be `https://-80.gateway.example.com:7777` + +Leave `cert_chain` and `cert_key` unset. They load a certificate you already +have from disk at startup, for the case where something else issues it; the +gateway's own issuance does not use them and does not write them. + +Two more sections matter for certificates: + +```toml +[core.admin] +enabled = true +address = "127.0.0.1:9016" +auth_token = "" -```shell -./certbot set-caa -./certbot renew +[core.sync] +data_dir = "/var/lib/dstack-gateway/data" ``` -## Step 4: Update `gateway.toml` +The admin API is where the ACME settings, the Cloudflare token and the domain +list live — they are stored in the gateway's KV store and there is no file to +put them in, so certificates cannot be issued without it. See +[Admin API authentication](#admin-api-authentication) for the credential +options. `data_dir` is where the ACME account key and the issued certificates +are persisted; point it somewhere writable that survives restarts, or the +gateway asks the CA for a fresh certificate every time it starts and will run +into Let's Encrypt's rate limits. The section is named for cluster sync, but +this store is used whether or not `enabled` is set. + +Start the gateway. + +## Step 3: Give the gateway its ACME configuration + +Open the admin dashboard at `http://` and fill in **Certbot +Configuration**, **DNS Credentials** and **ZT-Domains**, or do the same over the +admin API: + +```bash +ADMIN_ADDR=127.0.0.1:9016 +AUTH=(-H "Authorization: Bearer $ADMIN_API_TOKEN") + +# Start on staging: its certificates are not browser-trusted, but its rate +# limits leave room for mistakes. +curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \ + -H "Content-Type: application/json" \ + -d '{"acme_url":"https://acme-staging-v02.api.letsencrypt.org/directory", + "renew_interval_secs":3600,"renew_before_expiration_secs":864000, + "renew_timeout_secs":300}' + +curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \ + -H "Content-Type: application/json" \ + -d '{"name":"cloudflare","provider_type":"cloudflare", + "cf_api_token":"'"$CF_API_TOKEN"'","set_as_default":true}' + +curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/AddZtDomain" \ + -H "Content-Type: application/json" \ + -d '{"domain":"gateway.example.com","port":443,"priority":100}' +``` -Focus on these five fields in the `core.proxy` section: +Add a ZT domain for every name the gateway terminates TLS on. An entry for +`gateway.example.com` gets a certificate covering `*.gateway.example.com`, which +is what app URLs live under — a certificate for `*.example.com` would not, since +a wildcard does not span a further label. `port` is the port that domain is +served on and `priority` breaks ties when more than one entry could be the +default base domain. -- `cert_chain` & `cert_key`: Point to the certificate paths from the previous step -- `base_domain`: The wildcard domain for proxy -- `listen_addr` & `listen_port`: Listen to `0.0.0.0` and preferably `443` in production. If using another port, specify it in the URL +Certificates are requested on the next renewal round rather than the moment a +domain is added. Watch for them to arrive: -For example, if your base domain is `gateway.example.com`, app ID is ``, listening on `80`, and dstack-gateway is on port 7777, the URL would be `https://-80.gateway.example.com:7777` +```bash +curl -sf "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" \ + | jq '.domains[] | {domain: .config.domain, cert: .cert_status}' +``` + +`has_cert: true` with a `not_after` roughly 90 days out means the domain is +served. `POST /prpc/RenewCert` forces a round immediately instead of waiting for +`renew_interval_secs`. + +Pin issuance with `POST /prpc/SetCaa`, which writes CAA records naming Let's +Encrypt and the gateway's ACME account URI for every configured domain, so no +other account can have a certificate issued for them. + +Once the gateway serves traffic on staging certificates, switch to production: +`SetCertbotConfig` with `https://acme-v02.api.letsencrypt.org/directory`, then +`RotateAcmeCredentials` to register an account there and re-pin every domain's +CAA to it. Renewals refuse to run while the stored account and the configured +ACME URL disagree, so do not skip the rotation. -### URL Format +## Step 4: Adjust Configuration in `vmm.toml` + +Open `vmm.toml` and adjust dstack-gateway configuration in the `gateway` section: + +- `base_domain`: Same as `base_domain` from `gateway.toml`'s `core.proxy` section +- `port`: Same as `listen_port` from `gateway.toml`'s `core.proxy` section + +## URL Format The gateway supports the following URL format: - `[-][].` @@ -57,13 +141,6 @@ Examples: Note: The `s` and `g` suffixes cannot be used together -## Step 5: Adjust Configuration in `vmm.toml` - -Open `vmm.toml` and adjust dstack-gateway configuration in the `gateway` section: - -- `base_domain`: Same as `base_domain` from `gateway.toml`'s `core.proxy` section -- `port`: Same as `listen_port` from `gateway.toml`'s `core.proxy` section - ## Admin API authentication The gateway exposes a separate admin API (used for sync, WireGuard peer management, and other operator RPCs). Configure it in the `core.admin` section of `gateway.toml`: @@ -73,7 +150,7 @@ The gateway exposes a separate admin API (used for sync, WireGuard peer manageme enabled = true address = "0.0.0.0:9016" # generate with: openssl rand -hex 32 -admin_token = "" +auth_token = "" # alternatively, an Apache bcrypt htpasswd file (htpasswd -B -c admin.htpasswd admin) # htpasswd_file = "/etc/dstack/gateway-admin.htpasswd" insecure_no_auth = false @@ -81,11 +158,11 @@ insecure_no_auth = false - `enabled`: enable the admin API server. - `address`: bind address/port for the admin API. -- `admin_token`: shared admin token. It can also be supplied via the environment variables `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` instead of the config file. -- `htpasswd_file`: path to an Apache bcrypt htpasswd file (create with `htpasswd -B -c admin.htpasswd admin`); only bcrypt entries are accepted. Can be used instead of, or alongside, `admin_token`. +- `auth_token`: shared admin token. It can also be supplied via the environment variables `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` instead of the config file. The older name `admin_token` is still accepted. +- `htpasswd_file`: path to an Apache bcrypt htpasswd file (create with `htpasswd -B -c admin.htpasswd admin`); only bcrypt entries are accepted. Can be used instead of, or alongside, `auth_token`. - `insecure_no_auth`: development-only escape hatch that disables admin authentication. Never enable it on a network-reachable admin interface. -The admin server is fail-closed: if it is enabled with no `admin_token` and no `htpasswd_file`, and `insecure_no_auth` is `false`, it refuses to start rather than exposing an unauthenticated admin API. +The admin server is fail-closed: if it is enabled with no `auth_token` and no `htpasswd_file`, and `insecure_no_auth` is `false`, it refuses to start rather than exposing an unauthenticated admin API. Clients authenticate by sending `Authorization: Bearer ` or the `X-Admin-Token: ` header. @@ -133,4 +210,3 @@ max(dstack_gateway_cluster_nodes_active) - min(dstack_gateway_cluster_nodes_acti | `dstack_gateway_kv_peer_buffered_logs` | Entries still buffered for a peer. Sustained growth means that peer stopped acknowledging and the two nodes are drifting apart. | | `dstack_gateway_cluster_cert_not_after_seconds` | Certificate expiry per domain; alert on `- time()` falling under the renewal window. Capped at 256 series — compare `dstack_gateway_cluster_cert_domains` to see whether the cap was hit. | | `dstack_gateway_kv_persist_failures_total` | Periodic snapshots are failing, so a restart replays a growing WAL. | -| `dstack_gateway_kv_persist_failures_total` | Periodic snapshots are failing, so a restart replays a growing WAL. |