From a752055542d6ae4fde27b102d0a06290ba4f843d Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 02:50:00 -0700
Subject: [PATCH 01/37] feat(certbot): add the dns-persist-01 validation method

dns-01 rewrites the zone on every order, so certbot holds a DNS provider
credential for the life of the deployment. dns-persist-01
(draft-ietf-acme-dns-persist-01) proves control with a `_validation-persist`
TXT record published once, out of band: the record names the CA and the ACME
account, nothing about it changes between orders, and certbot only ever reads
DNS.

`ValidationMethod` replaces the `Dns01Client` field on `AcmeClient`, so the
dns-01-only state -- the provider client and the TXT TTL -- lives in the one
variant that has any use for it, and dns-persist-01 cannot be constructed
holding a credential it would never call. `check_dns` widens from an exact
match on the key authorization to matching whatever the challenge expects,
and the cleanup pass skips records certbot did not create: a persistent
record is the operator's and outlives every order.

The self-check stays advisory for both methods. Our resolver is not the CA's,
and under dns-persist-01 our expectation can be stricter than the CA's -- the
challenge's `issuer-domain-names` are not exposed by instant-acme -- so a
record we cannot see is named in a warning and the order proceeds.

CAA content now names the challenge in use, because a record pinned to
`validationmethods=dns-01` refuses every dns-persist-01 order. The dns-01
string is unchanged byte for byte, pinned by a test, so records published by
earlier releases keep matching.

`required_dns_records` renders the whole one-time setup -- validation record
plus CAA -- from an account URI rather than a live client, so a caller holding
only stored credentials can render it without a round trip to the CA. The
record grammar is an RFC 8659 issue-value, and the parser mirrors what CAs
run down to the parts that reject rather than ignore (trailing semicolon,
repeated tag, whitespace in a value), so certbot never renders a record the
CA would refuse or accepts one it would.

The gateway keeps its dns-01 behaviour; the call sites move to the new
constructor unchanged.
---
 dstack/certbot/src/acme_client.rs         | 659 ++++++++++++++++++----
 dstack/certbot/src/bot.rs                 | 110 ++--
 dstack/certbot/src/dns_persist.rs         | 399 +++++++++++++
 dstack/certbot/src/lib.rs                 |   7 +-
 dstack/gateway/src/distributed_certbot.rs |  58 +-
 5 files changed, 1050 insertions(+), 183 deletions(-)
 create mode 100644 dstack/certbot/src/dns_persist.rs

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index 6288303b9..3cc73ab3d 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -16,6 +16,7 @@ use rcgen::{CertificateParams, DistinguishedName, KeyPair};
 use serde::{Deserialize, Serialize};
 use std::{
     collections::{BTreeMap, BTreeSet},
+    fmt,
     net::SocketAddr,
     path::{Path, PathBuf},
     time::Duration,
@@ -25,23 +26,245 @@ use tracing::{debug, error, info, warn};
 use x509_parser::prelude::{GeneralName, Pem};
 
 use super::dns01_client::{Dns01Api, Dns01Client};
+use super::dns_persist::{self, AuthorizationRecord};
 use super::http_client::ReqwestHttpClient;
 
+/// The ACME challenge used to prove control of the certificate's domains.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ChallengeKind {
+    /// RFC 8555 `dns-01`. certbot answers every order by writing a TXT record
+    /// through the DNS provider API, so it needs a credential with write access
+    /// to the zone.
+    #[default]
+    #[serde(rename = "dns-01")]
+    Dns01,
+    /// draft-ietf-acme-dns-persist-01 `dns-persist-01`. Control is proven by a
+    /// `_validation-persist` record published once, out of band; certbot needs no
+    /// DNS credential and the zone can be hosted anywhere.
+    ///
+    /// Experimental — see `docs/certbot-dns-persist-01.md`.
+    #[serde(rename = "dns-persist-01")]
+    DnsPersist01,
+}
+
+/// How the client proves control of a domain to the ACME server.
+///
+/// The two methods differ in who writes DNS. `dns-01` needs certbot to hold a
+/// provider credential with write access to the zone for the lifetime of the
+/// deployment; `dns-persist-01` moves that to a one-time record the operator
+/// publishes by hand, after which certbot only ever reads DNS.
+#[derive(Debug)]
+pub enum ValidationMethod {
+    /// RFC 8555 `dns-01`: certbot publishes a fresh `_acme-challenge` TXT record
+    /// through the provider API for every order and removes it afterwards.
+    Dns01 {
+        /// Provider client with write access to the zone.
+        client: Dns01Client,
+        /// TTL of the published records, in seconds (1 = auto, min 60 on Cloudflare).
+        txt_ttl: u32,
+    },
+    /// draft-ietf-acme-dns-persist-01 `dns-persist-01`: control is proven by a
+    /// `_validation-persist` TXT record naming the CA and this ACME account,
+    /// published once and left in place. certbot needs no provider credential,
+    /// and the zone can be hosted anywhere.
+    ///
+    /// Experimental: the draft is still changing and Let's Encrypt serves this
+    /// challenge on staging only. See `docs/certbot-dns-persist-01.md`.
+    DnsPersist01 {
+        /// Issuer Domain Name to name in the record and in CAA records. Must be
+        /// one of the `issuer-domain-names` the CA sends in the challenge —
+        /// `letsencrypt.org` for Let's Encrypt.
+        issuer_domain_name: String,
+    },
+}
+
+impl ValidationMethod {
+    /// Which challenge this method answers.
+    fn kind(&self) -> ChallengeKind {
+        match self {
+            Self::Dns01 { .. } => ChallengeKind::Dns01,
+            Self::DnsPersist01 { .. } => ChallengeKind::DnsPersist01,
+        }
+    }
+
+    /// The challenge type to look for in an authorization.
+    fn challenge_type(&self) -> ChallengeType {
+        match self {
+            Self::Dns01 { .. } => ChallengeType::Dns01,
+            // instant-acme has no variant for the draft challenge, so it lands in
+            // `Unknown`. Matching on the wire string is what selects it.
+            Self::DnsPersist01 { .. } => ChallengeType::Unknown(DNS_PERSIST_01.to_string()),
+        }
+    }
+
+    /// Issuer Domain Name to write into CAA records.
+    fn issuer_domain_name(&self) -> &str {
+        match self {
+            // Unconfigurable for dns-01, as it has always been: existing
+            // deployments have CAA records published under this name.
+            Self::Dns01 { .. } => dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+            Self::DnsPersist01 { issuer_domain_name } => issuer_domain_name,
+        }
+    }
+
+    /// The provider client, or an error naming why there isn't one.
+    fn dns01_client(&self) -> Result<&Dns01Client> {
+        match self {
+            Self::Dns01 { client, .. } => Ok(client),
+            Self::DnsPersist01 { .. } => bail!(
+                "dns-persist-01 holds no DNS provider credential, so certbot cannot write \
+                 records; publish them out of band (see the `dns-records` command)"
+            ),
+        }
+    }
+}
+
+/// Wire name of the draft challenge, as it appears in the authorization.
+const DNS_PERSIST_01: &str = "dns-persist-01";
+
+/// A DNS record that has to exist before the CA will issue.
+///
+/// Rendered as a zone-file line so it can be pasted into any provider.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RequiredRecord {
+    /// FQDN the record lives at.
+    pub name: String,
+    /// Record type, e.g. `TXT` or `CAA`.
+    pub record_type: String,
+    /// Record value, including any CAA flags and tag.
+    pub content: String,
+}
+
+impl fmt::Display for RequiredRecord {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}. IN {} {}", self.name, self.record_type, self.content)
+    }
+}
+
+/// The CAA `issue`/`issuewild` value that pins issuance to one account.
+///
+/// `validationmethods` names the challenge actually in use: a record left
+/// pinned to `dns-01` after switching to `dns-persist-01` refuses every order,
+/// and vice versa.
+fn caa_content(challenge: ChallengeKind, issuer_domain_name: &str, account_uri: &str) -> String {
+    let method = match challenge {
+        ChallengeKind::Dns01 => "dns-01",
+        ChallengeKind::DnsPersist01 => DNS_PERSIST_01,
+    };
+    format!("{issuer_domain_name};validationmethods={method};accounturi={account_uri}")
+}
+
+/// Every DNS record that has to exist for the CA to issue for `domains`.
+///
+/// Under `dns-01` certbot writes these itself and the list is informational.
+/// Under `dns-persist-01` it holds no credential and cannot write anything, so
+/// this list *is* the one-time setup an operator has to publish by hand.
+///
+/// Takes the account URI rather than a client so that callers holding only the
+/// stored credentials — an admin listing, say — can render the records without
+/// a round trip to the CA.
+pub fn required_dns_records(
+    challenge: ChallengeKind,
+    issuer_domain_name: &str,
+    account_uri: &str,
+    domains: &[String],
+) -> Vec<RequiredRecord> {
+    let caa_content = caa_content(challenge, issuer_domain_name, account_uri);
+    let mut records = Vec::new();
+    for base_name in base_names(domains) {
+        if challenge == ChallengeKind::DnsPersist01 {
+            // One record covers the base name and, with the wildcard policy,
+            // `*.<base name>`; only ask for the policy when a wildcard is
+            // actually requested, so the record grants no more than needed.
+            let record = AuthorizationRecord {
+                issuer_domain_name: issuer_domain_name.to_string(),
+                account_uri: account_uri.to_string(),
+                wildcard: domains
+                    .iter()
+                    .any(|name| name.strip_prefix("*.") == Some(base_name)),
+            };
+            records.push(RequiredRecord {
+                name: dns_persist::validation_domain(base_name),
+                record_type: "TXT".to_string(),
+                content: format!("\"{}\"", record.rdata()),
+            });
+        }
+        for tag in ["issue", "issuewild"] {
+            records.push(RequiredRecord {
+                name: base_name.to_string(),
+                record_type: "CAA".to_string(),
+                content: format!("0 {tag} \"{caa_content}\""),
+            });
+        }
+    }
+    records
+}
+
 /// A AcmeClient instance.
 pub struct AcmeClient {
     account: Account,
     credentials: Credentials,
-    dns01_client: Dns01Client,
+    validation: ValidationMethod,
     max_dns_wait: Duration,
-    /// TTL for DNS TXT records used in ACME challenges (in seconds).
-    dns_txt_ttl: u32,
 }
 
+/// One pending authorization and the DNS record that answers it.
 #[derive(Debug, Clone)]
 struct Challenge {
-    id: String,
+    /// Provider-assigned record id, used by the cleanup pass after the order
+    /// settles. `None` when certbot did not create the record and must not
+    /// delete it — `dns-persist-01` records belong to the operator.
+    id: Option<String>,
+    /// FQDN the TXT record lives at.
     acme_domain: String,
-    dns_value: String,
+    /// What a TXT record there has to say for the CA to accept the challenge.
+    expected: Expected,
+}
+
+/// The condition a challenge's TXT records have to meet.
+#[derive(Debug, Clone)]
+enum Expected {
+    /// `dns-01`: the key authorization digest, matched verbatim.
+    KeyAuthorization(String),
+    /// `dns-persist-01`: an issue-value naming our issuer and account.
+    Authorization(AuthorizationRecord),
+}
+
+impl Expected {
+    /// Whether the TXT records currently published at the challenge domain answer
+    /// the challenge.
+    fn satisfied_by(&self, published: &[String], now: u64) -> bool {
+        match self {
+            Self::KeyAuthorization(value) => published.iter().any(|txt| txt == value),
+            Self::Authorization(record) => record.satisfied_by_any(published, now),
+        }
+    }
+}
+
+impl Challenge {
+    /// A line naming what is missing at the challenge domain, phrased so an
+    /// operator can act on it: under `dns-persist-01` it is the exact record to
+    /// publish, under `dns-01` it is the value certbot just wrote.
+    fn unsettled_hint(&self) -> String {
+        format!(
+            "no TXT record at {} matches the expected value: {}",
+            self.acme_domain, self.expected
+        )
+    }
+}
+
+impl fmt::Display for Expected {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::KeyAuthorization(value) => f.write_str(value),
+            Self::Authorization(record) => write!(f, "{}", record.rdata()),
+        }
+    }
+}
+
+/// Current UNIX time, for comparing against a record's `persistUntil`.
+fn now_secs() -> u64 {
+    time::OffsetDateTime::now_utc().unix_timestamp().max(0) as u64
 }
 
 #[derive(Serialize, Deserialize)]
@@ -59,16 +282,26 @@ pub(crate) fn acme_matches(encoded_credentials: &str, acme_url: &str) -> bool {
     credentials.acme_url == acme_url
 }
 
+/// The names CAA and `dns-persist-01` records are published under, deduplicated.
+///
+/// A wildcard request is authorized from its base name, so `example.com` and
+/// `*.example.com` share one set of records.
+fn base_names(domains: &[String]) -> BTreeSet<&str> {
+    domains
+        .iter()
+        .map(|name| name.strip_prefix("*.").unwrap_or(name))
+        .collect()
+}
+
 fn caa_tag(content: &str) -> Option<&str> {
     content.split_whitespace().nth(1)
 }
 
 impl AcmeClient {
     pub async fn load(
-        dns01_client: Dns01Client,
+        validation: ValidationMethod,
         encoded_credentials: &str,
         max_dns_wait: Duration,
-        dns_txt_ttl: u32,
     ) -> Result<Self> {
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
         let http_client = Box::new(ReqwestHttpClient::new()?);
@@ -78,19 +311,17 @@ impl AcmeClient {
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
         Ok(Self {
             account,
-            dns01_client,
+            validation,
             credentials,
             max_dns_wait,
-            dns_txt_ttl,
         })
     }
 
     /// Create a new account.
     pub async fn new_account(
         acme_url: &str,
-        dns01_client: Dns01Client,
+        validation: ValidationMethod,
         max_dns_wait: Duration,
-        dns_txt_ttl: u32,
     ) -> Result<Self> {
         let http_client = Box::new(ReqwestHttpClient::new()?);
         let (account, credentials) = Account::builder_with_http(http_client)
@@ -112,10 +343,9 @@ impl AcmeClient {
         };
         Ok(Self {
             account,
-            dns01_client,
+            validation,
             credentials,
             max_dns_wait,
-            dns_txt_ttl,
         })
     }
 
@@ -129,27 +359,46 @@ impl AcmeClient {
         &self.credentials.account_id
     }
 
+    /// The CAA `issue`/`issuewild` value that pins issuance to this account.
+    fn caa_content(&self) -> String {
+        caa_content(
+            self.validation.kind(),
+            self.validation.issuer_domain_name(),
+            self.account_id(),
+        )
+    }
+
+    /// Every DNS record that has to exist for the CA to issue for `domains`.
+    ///
+    /// See [`required_dns_records`], which this fills in from the live account.
+    pub fn required_dns_records(&self, domains: &[String]) -> Vec<RequiredRecord> {
+        required_dns_records(
+            self.validation.kind(),
+            self.validation.issuer_domain_name(),
+            self.account_id(),
+            domains,
+        )
+    }
+
     pub async fn set_caa_records(&self, domains: &[String]) -> Result<()> {
-        let account_id = self.account_id();
-        let content = format!("letsencrypt.org;validationmethods=dns-01;accounturi={account_id}");
-        let base_names = domains
-            .iter()
-            .map(|name| name.strip_prefix("*.").unwrap_or(name))
-            .collect::<BTreeSet<_>>();
+        let dns01_client = self.validation.dns01_client().context(
+            "cannot set CAA records without DNS write access; publish the records \
+             printed by `certbot dns-records` instead",
+        )?;
+        let content = self.caa_content();
+        let base_names = base_names(domains);
 
         for base_name in base_names {
             // 1. Set ";" to guard timing gap between the operations.
             debug!("setting guard CAA records for {base_name}");
-            let guard0 = self
-                .dns01_client
+            let guard0 = dns01_client
                 .add_caa_record(base_name, 0, "issue", ";")
                 .await?;
-            let guard1 = self
-                .dns01_client
+            let guard1 = dns01_client
                 .add_caa_record(base_name, 0, "issuewild", ";")
                 .await?;
             // 2. Remove the existing constraints
-            for record in self.dns01_client.get_records(base_name).await? {
+            for record in dns01_client.get_records(base_name).await? {
                 if record.id == guard0 || record.id == guard1 {
                     continue;
                 }
@@ -161,22 +410,22 @@ impl AcmeClient {
                         "removing existing issuer CAA record {} {}",
                         record.name, record.content
                     );
-                    self.dns01_client.remove_record(&record.id).await?;
+                    dns01_client.remove_record(&record.id).await?;
                 }
             }
             // 3. Set the new constraints
             debug!("setting CAA records for {base_name}, 0 issue \"{content}\"");
-            self.dns01_client
+            dns01_client
                 .add_caa_record(base_name, 0, "issue", &content)
                 .await?;
             debug!("setting CAA records for {base_name}, 0 issuewild \"{content}\"");
-            self.dns01_client
+            dns01_client
                 .add_caa_record(base_name, 0, "issuewild", &content)
                 .await?;
             debug!("removing guard CAA records for {base_name}");
             // 4. Remove the guards
-            self.dns01_client.remove_record(&guard0).await?;
-            self.dns01_client.remove_record(&guard1).await?;
+            dns01_client.remove_record(&guard0).await?;
+            dns01_client.remove_record(&guard1).await?;
         }
         Ok(())
     }
@@ -190,10 +439,18 @@ impl AcmeClient {
         let result = self
             .request_new_certificate_inner(key, domains, &mut challenges)
             .await;
-        for challenge in &challenges {
-            debug!("removing dns record {}", challenge.id);
-            if let Err(err) = self.dns01_client.remove_record(&challenge.id).await {
-                error!("failed to remove dns record {}: {err}", challenge.id);
+        // Only records certbot created are cleaned up. A dns-persist-01 record
+        // is the operator's and outlives every order, so it carries no id.
+        for id in challenges
+            .iter()
+            .filter_map(|challenge| challenge.id.as_ref())
+        {
+            let Ok(dns01_client) = self.validation.dns01_client() else {
+                break;
+            };
+            debug!("removing dns record {id}");
+            if let Err(err) = dns01_client.remove_record(id).await {
+                error!("failed to remove dns record {id}: {err}");
             }
         }
         result
@@ -347,6 +604,7 @@ impl AcmeClient {
 
 impl AcmeClient {
     async fn authorize(&self, order: &mut Order, challenges: &mut Vec<Challenge>) -> Result<()> {
+        let challenge_type = self.validation.challenge_type();
         let mut authorizations = order.authorizations();
         while let Some(authz) = authorizations.next().await {
             let mut authz = authz.context("failed to get authorizations")?;
@@ -356,42 +614,61 @@ impl AcmeClient {
                 _ => bail!("unsupported authorization status: {:?}", authz.status),
             }
 
+            // Read before taking the challenge handle, which borrows the
+            // authorization for the rest of the iteration.
+            let wildcard = authz.wildcard;
             let challenge = authz
-                .challenge(ChallengeType::Dns01)
-                .context("no dns01 challenge found")?;
-
-            let acme_domain = challenge_domain(challenge.identifier())?;
-            let dns_value = challenge.key_authorization().dns_value();
-            // Clearing stale records is a per-name preparation step, not a
-            // per-authorization one. An order for `example.com` and
-            // `*.example.com` yields two authorizations that are both answered
-            // under `_acme-challenge.example.com`, each with its own value, and
-            // both values have to be live at validation time. Purging again for
-            // the second authorization would delete the record the first one
-            // just published, so one of the two challenges could never be
-            // answered and the order failed with "Correct value not found for
-            // DNS challenge".
-            if needs_purge(challenges, &acme_domain) {
-                debug!("removing existing TXT records for {acme_domain}");
-                self.dns01_client
-                    .remove_txt_records(&acme_domain)
-                    .await
-                    .context("failed to remove existing dns record")?;
-            }
-            debug!(
-                "creating TXT record for {acme_domain} with TTL {}s",
-                self.dns_txt_ttl
-            );
-            let id = self
-                .dns01_client
-                .add_txt_record(&acme_domain, &dns_value, self.dns_txt_ttl)
-                .await
-                .context("failed to create dns record")?;
-            challenges.push(Challenge {
-                id,
-                acme_domain,
-                dns_value,
-            });
+                .challenge(challenge_type.clone())
+                .with_context(|| format!("no {challenge_type:?} challenge found"))?;
+
+            let acme_domain = challenge_domain(self.validation.kind(), challenge.identifier())?;
+            let challenge = match &self.validation {
+                ValidationMethod::Dns01 {
+                    client, txt_ttl, ..
+                } => {
+                    let dns_value = challenge.key_authorization().dns_value();
+                    // Clearing stale records is a per-name preparation step, not a
+                    // per-authorization one. An order for `example.com` and
+                    // `*.example.com` yields two authorizations that are both answered
+                    // under `_acme-challenge.example.com`, each with its own value, and
+                    // both values have to be live at validation time. Purging again for
+                    // the second authorization would delete the record the first one
+                    // just published, so one of the two challenges could never be
+                    // answered and the order failed with "Correct value not found for
+                    // DNS challenge".
+                    if needs_purge(challenges, &acme_domain) {
+                        debug!("removing existing TXT records for {acme_domain}");
+                        client
+                            .remove_txt_records(&acme_domain)
+                            .await
+                            .context("failed to remove existing dns record")?;
+                    }
+                    debug!("creating TXT record for {acme_domain} with TTL {txt_ttl}s");
+                    let id = client
+                        .add_txt_record(&acme_domain, &dns_value, *txt_ttl)
+                        .await
+                        .context("failed to create dns record")?;
+                    Challenge {
+                        id: Some(id),
+                        acme_domain,
+                        expected: Expected::KeyAuthorization(dns_value),
+                    }
+                }
+                // Nothing to publish: the record is already in the zone, or the
+                // order is about to fail and say so. There is likewise nothing
+                // to purge -- the record is the operator's, and one persistent
+                // record answers every authorization under the name.
+                ValidationMethod::DnsPersist01 { issuer_domain_name } => Challenge {
+                    id: None,
+                    acme_domain,
+                    expected: Expected::Authorization(AuthorizationRecord {
+                        issuer_domain_name: issuer_domain_name.clone(),
+                        account_uri: self.account_id().to_string(),
+                        wildcard,
+                    }),
+                },
+            };
+            challenges.push(challenge);
         }
         Ok(())
     }
@@ -416,10 +693,10 @@ impl AcmeClient {
         // the name below it: for `_acme-challenge.a.example.com` the NS records
         // usually live on `example.com`. Querying the full name returns NODATA,
         // so walk up a label at a time until a name actually carries NS records.
-        let mut candidate = domain
-            .strip_prefix("_acme-challenge.")
-            .unwrap_or(domain)
-            .to_string();
+        // The leading label is dropped for any challenge prefix -- `_acme-challenge`
+        // for dns-01, `_validation-persist` for dns-persist-01 -- since an
+        // underscore label never carries NS records.
+        let mut candidate = strip_challenge_label(domain).to_string();
         let mut addrs = Vec::new();
         let mut zone = candidate.clone();
         loop {
@@ -452,7 +729,13 @@ impl AcmeClient {
         resolver_for(&addrs)
     }
 
-    /// Self check the TXT records for the given challenges.
+    /// Wait until every challenge's records are visible to us.
+    ///
+    /// Advisory, not a gate: our resolver is not the CA's, and under
+    /// `dns-persist-01` our expectation can even be stricter than the CA's --
+    /// the `issuer-domain-names` it would accept are not exposed by
+    /// instant-acme. A record we never see is reported and the order proceeds,
+    /// letting the CA decide.
     async fn check_dns(&self, challenges: &[Challenge]) -> Result<()> {
         let mut delay = Duration::from_millis(250);
         let mut tries = 1u8;
@@ -500,24 +783,26 @@ impl AcmeClient {
                     "DNS propagation timeout after {elapsed:?}, max wait time is {max:?}. proceeding anyway as ACME server may have different DNS view",
                     max = self.max_dns_wait
                 );
+                for challenge in &unsettled_challenges {
+                    warn!("{}", challenge.unsettled_hint());
+                }
                 break;
             }
 
             while let Some(challenge) = unsettled_challenges.pop() {
-                let expected_txt = &challenge.dns_value;
                 let dns_resolver = resolvers
                     .get(&challenge.acme_domain)
                     .context("no resolver for challenge domain")?;
-                let settled = match dns_resolver.txt_lookup(&challenge.acme_domain).await {
-                    Ok(record) => record.answers().iter().any(|answer| {
-                        let RData::TXT(txt) = &answer.data else {
-                            return false;
-                        };
-                        let actual_txt = txt.to_string();
-                        debug!("Expected challenge: {expected_txt}, actual: {actual_txt}");
-                        actual_txt == *expected_txt
-                    }),
-                    Err(err) if err.is_no_records_found() => false,
+                let published = match dns_resolver.txt_lookup(&challenge.acme_domain).await {
+                    Ok(records) => records
+                        .answers()
+                        .iter()
+                        .filter_map(|answer| match &answer.data {
+                            RData::TXT(txt) => Some(txt.to_string()),
+                            _ => None,
+                        })
+                        .collect::<Vec<_>>(),
+                    Err(err) if err.is_no_records_found() => Vec::new(),
                     Err(err) if !fell_back.contains(&challenge.acme_domain) => {
                         // Transport failures land here rather than in the arm
                         // above: `is_no_records_found` covers only
@@ -548,10 +833,14 @@ impl AcmeClient {
                             domain = &challenge.acme_domain,
                             "dns lookup failed: {err:#}"
                         );
-                        false
+                        Vec::new()
                     }
                 };
-                if !settled {
+                debug!(
+                    "Expected challenge: {}, actual: {published:?}",
+                    challenge.expected
+                );
+                if !challenge.expected.satisfied_by(&published, now_secs()) {
                     delay = Duration::from_secs(32).min(delay * 2);
                     tries += 1;
                     debug!(
@@ -570,13 +859,14 @@ impl AcmeClient {
         Ok(())
     }
 
-    /// Tell the ACME server every pending dns-01 challenge is answerable.
+    /// Tell the ACME server every pending challenge is answerable.
     ///
-    /// The TXT records are published first and verified for propagation, so this
-    /// is a second pass over the same authorizations: 0.8 dropped
-    /// `Order::set_challenge_ready(url)`, and `ChallengeHandle::set_ready()`
-    /// borrows the order, so the handle cannot be held across the DNS wait.
+    /// The DNS records are published and checked first, so this is a second pass
+    /// over the same authorizations: 0.8 dropped `Order::set_challenge_ready(url)`,
+    /// and `ChallengeHandle::set_ready()` borrows the order, so the handle cannot
+    /// be held across the DNS wait.
     async fn set_challenges_ready(&self, order: &mut Order) -> Result<()> {
+        let challenge_type = self.validation.challenge_type();
         let mut authorizations = order.authorizations();
         while let Some(authz) = authorizations.next().await {
             let mut authz = authz.context("failed to get authorizations")?;
@@ -584,8 +874,8 @@ impl AcmeClient {
                 continue;
             }
             let mut challenge = authz
-                .challenge(ChallengeType::Dns01)
-                .context("no dns01 challenge found")?;
+                .challenge(challenge_type.clone())
+                .with_context(|| format!("no {challenge_type:?} challenge found"))?;
             debug!("setting challenge ready for {}", challenge.url);
             challenge
                 .set_ready()
@@ -695,17 +985,36 @@ fn needs_purge(published: &[Challenge], acme_domain: &str) -> bool {
         .any(|challenge| challenge.acme_domain == acme_domain)
 }
 
-/// The name of the TXT record that answers a dns-01 challenge for `identifier`.
+/// The name of the TXT record that answers `challenge`'s validation for `identifier`.
 ///
 /// The record always lives under the bare name: a wildcard authorization for
-/// `*.example.com` is answered at `_acme-challenge.example.com`. `AuthorizedIdentifier`
+/// `*.example.com` is answered at `_acme-challenge.example.com`, and at
+/// `_validation-persist.example.com` under dns-persist-01. `AuthorizedIdentifier`
 /// renders the wildcard prefix in its `Display`, so formatting it directly would publish
 /// the record at `_acme-challenge.*.example.com` and fail every wildcard issuance.
-fn challenge_domain(identifier: &AuthorizedIdentifier<'_>) -> Result<String> {
+fn challenge_domain(
+    challenge: ChallengeKind,
+    identifier: &AuthorizedIdentifier<'_>,
+) -> Result<String> {
     let Identifier::Dns(name) = identifier.identifier else {
         bail!("unsupported identifier type in authorization: {identifier}");
     };
-    Ok(format!("_acme-challenge.{name}"))
+    Ok(match challenge {
+        ChallengeKind::Dns01 => format!("_acme-challenge.{name}"),
+        ChallengeKind::DnsPersist01 => dns_persist::validation_domain(name),
+    })
+}
+
+/// Drop a leading underscore label, so the zone walk starts at a real name.
+///
+/// Challenge records live under a reserved label -- `_acme-challenge` for
+/// dns-01, `_validation-persist` for dns-persist-01 -- which never carries NS
+/// records, so querying it only costs a round trip.
+fn strip_challenge_label(domain: &str) -> &str {
+    match domain.starts_with('_') {
+        true => domain.split_once('.').map_or(domain, |(_, rest)| rest),
+        false => domain,
+    }
 }
 
 async fn find_error(order: &mut Order) -> Result<Option<Problem>> {
@@ -952,6 +1261,23 @@ mod challenge_parsing_tests {
 mod ns_discovery_tests {
     use super::parent_zone;
 
+    #[test]
+    fn a_challenge_label_is_dropped_before_the_walk_starts() {
+        use super::strip_challenge_label;
+
+        // Both challenge methods put their record under a reserved underscore
+        // label, and neither label can carry NS records.
+        assert_eq!(
+            strip_challenge_label("_acme-challenge.example.com"),
+            "example.com"
+        );
+        assert_eq!(
+            strip_challenge_label("_validation-persist.example.com"),
+            "example.com"
+        );
+        assert_eq!(strip_challenge_label("example.com"), "example.com");
+    }
+
     #[test]
     fn the_walk_climbs_to_a_name_that_can_carry_ns_records() {
         // A challenge name is not a zone cut, so the walk has to climb to the
@@ -974,43 +1300,156 @@ mod ns_discovery_tests {
 
 #[cfg(test)]
 mod challenge_domain_tests {
-    use super::challenge_domain;
+    use super::{challenge_domain, ChallengeKind};
     use instant_acme::Identifier;
 
     /// A wildcard order authorizes the bare name with `wildcard: true`, and the TXT record
     /// answering it must be published under that bare name. Formatting the identifier
     /// through its `Display` instead would ask for `_acme-challenge.*.example.com`.
+    /// dns-persist-01 answers the same authorization under its own label, and the
+    /// wildcard prefix must not leak into that name either.
     #[test]
     fn the_challenge_domain_never_carries_a_wildcard_prefix() {
         let dns = Identifier::Dns("example.com".to_string());
+        for wildcard in [false, true] {
+            assert_eq!(
+                challenge_domain(ChallengeKind::Dns01, &dns.authorized(wildcard)).unwrap(),
+                "_acme-challenge.example.com",
+                "wildcard={wildcard}"
+            );
+            assert_eq!(
+                challenge_domain(ChallengeKind::DnsPersist01, &dns.authorized(wildcard)).unwrap(),
+                "_validation-persist.example.com",
+                "wildcard={wildcard}"
+            );
+        }
+    }
+
+    /// Both methods are only defined for DNS identifiers; anything else is a bug in the
+    /// order we built, so it must be an error rather than a nonsensical record name.
+    #[test]
+    fn a_non_dns_identifier_is_rejected() {
+        let ip = Identifier::Ip("192.0.2.1".parse().unwrap());
+        assert!(challenge_domain(ChallengeKind::Dns01, &ip.authorized(false)).is_err());
+        assert!(challenge_domain(ChallengeKind::DnsPersist01, &ip.authorized(false)).is_err());
+    }
+}
+
+#[cfg(test)]
+mod required_record_tests {
+    use super::*;
+
+    const ACCOUNT: &str = "https://acme-v02.api.letsencrypt.org/acme/acct/1234567890";
+
+    fn records(challenge: ChallengeKind, domains: &[&str]) -> Vec<String> {
+        required_dns_records(
+            challenge,
+            dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+            ACCOUNT,
+            &domains.iter().map(|d| d.to_string()).collect::<Vec<_>>(),
+        )
+        .iter()
+        .map(ToString::to_string)
+        .collect()
+    }
+
+    #[test]
+    fn dns01_caa_content_is_byte_identical_to_the_pinned_format() {
+        // CAA values published by earlier releases have to keep matching, or
+        // issuance stops the moment this string drifts.
         assert_eq!(
-            challenge_domain(&dns.authorized(false)).unwrap(),
-            "_acme-challenge.example.com"
+            caa_content(
+                ChallengeKind::Dns01,
+                dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+                ACCOUNT
+            ),
+            format!("letsencrypt.org;validationmethods=dns-01;accounturi={ACCOUNT}")
         );
+    }
+
+    #[test]
+    fn caa_content_names_the_challenge_in_use() {
+        // A CAA record still pinned to dns-01 refuses every dns-persist-01 order.
+        assert!(caa_content(
+            ChallengeKind::DnsPersist01,
+            dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+            ACCOUNT
+        )
+        .contains("validationmethods=dns-persist-01"));
+    }
+
+    #[test]
+    fn dns01_needs_no_validation_record() {
+        // certbot writes the `_acme-challenge` record itself, per order.
         assert_eq!(
-            challenge_domain(&dns.authorized(true)).unwrap(),
-            "_acme-challenge.example.com"
+            records(ChallengeKind::Dns01, &["*.example.com"]),
+            vec![
+                format!(
+                    "example.com. IN CAA 0 issue \"letsencrypt.org;validationmethods=dns-01;accounturi={ACCOUNT}\""
+                ),
+                format!(
+                    "example.com. IN CAA 0 issuewild \"letsencrypt.org;validationmethods=dns-01;accounturi={ACCOUNT}\""
+                ),
+            ]
         );
     }
 
-    /// dns-01 is only defined for DNS identifiers; anything else is a bug in the order we
-    /// built, so it must be an error rather than a nonsensical record name.
     #[test]
-    fn a_non_dns_identifier_is_rejected() {
-        let ip = Identifier::Ip("192.0.2.1".parse().unwrap());
-        assert!(challenge_domain(&ip.authorized(false)).is_err());
+    fn dns_persist_asks_for_the_wildcard_policy_only_when_a_wildcard_is_ordered() {
+        let plain = records(ChallengeKind::DnsPersist01, &["example.com"]);
+        assert_eq!(
+            plain[0],
+            format!(
+                "_validation-persist.example.com. IN TXT \"letsencrypt.org; accounturi={ACCOUNT}\""
+            )
+        );
+
+        let wildcard = records(ChallengeKind::DnsPersist01, &["*.example.com"]);
+        assert_eq!(
+            wildcard[0],
+            format!(
+                "_validation-persist.example.com. IN TXT \"letsencrypt.org; accounturi={ACCOUNT}; policy=wildcard\""
+            )
+        );
+    }
+
+    #[test]
+    fn a_name_and_its_wildcard_share_one_validation_record() {
+        // Both identifiers authorize from the same base name, so asking the
+        // operator for two records would be asking for one too many.
+        let records = records(
+            ChallengeKind::DnsPersist01,
+            &["example.com", "*.example.com"],
+        );
+        assert_eq!(
+            records.iter().filter(|r| r.contains(" TXT ")).count(),
+            1,
+            "{records:#?}"
+        );
+        assert!(records[0].contains("policy=wildcard"), "{records:#?}");
+    }
+
+    #[test]
+    fn each_base_name_gets_its_own_records() {
+        let records = records(
+            ChallengeKind::DnsPersist01,
+            &["*.a.example.com", "*.b.example.com"],
+        );
+        assert_eq!(records.len(), 6, "{records:#?}");
+        assert!(records[0].starts_with("_validation-persist.a.example.com."));
+        assert!(records[3].starts_with("_validation-persist.b.example.com."));
     }
 }
 
 #[cfg(test)]
 mod purge_tests {
-    use super::{needs_purge, Challenge};
+    use super::{needs_purge, Challenge, Expected};
 
     fn challenge(acme_domain: &str, dns_value: &str) -> Challenge {
         Challenge {
-            id: format!("rec-{dns_value}"),
+            id: Some(format!("rec-{dns_value}")),
             acme_domain: acme_domain.to_string(),
-            dns_value: dns_value.to_string(),
+            expected: Expected::KeyAuthorization(dns_value.to_string()),
         }
     }
 
diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs
index 0daf499e3..30718d02d 100644
--- a/dstack/certbot/src/bot.rs
+++ b/dstack/certbot/src/bot.rs
@@ -9,12 +9,13 @@ use std::{
     time::Duration,
 };
 
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 use fs_err as fs;
 use tokio::time::sleep;
-use tracing::{error, info};
+use tracing::{error, info, warn};
 
-use crate::acme_client::{acme_matches, read_pem};
+use crate::acme_client::{acme_matches, read_pem, ChallengeKind, RequiredRecord, ValidationMethod};
+use crate::dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME;
 
 use super::{AcmeClient, Dns01Client};
 
@@ -27,6 +28,15 @@ pub struct CertBotConfig {
     auto_set_caa: bool,
     credentials_file: PathBuf,
     auto_create_account: bool,
+    /// ACME challenge used to prove control of the domains.
+    #[builder(default)]
+    challenge: ChallengeKind,
+    /// Issuer Domain Name naming the CA in `dns-persist-01` and CAA records.
+    ///
+    /// Must be one of the `issuer-domain-names` the CA sends in the challenge.
+    #[builder(default = LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string())]
+    issuer_domain_name: String,
+    /// Cloudflare API token. Unused, and warned about, under `dns-persist-01`.
     cf_api_token: String,
     cf_api_url: Option<String>,
     cert_file: PathBuf,
@@ -57,17 +67,12 @@ pub struct CertBot {
 
 async fn create_new_account(
     config: &CertBotConfig,
-    dns01_client: Dns01Client,
+    validation: ValidationMethod,
 ) -> Result<AcmeClient> {
     info!("creating new ACME account");
-    let client = AcmeClient::new_account(
-        &config.acme_url,
-        dns01_client,
-        config.max_dns_wait,
-        config.dns_txt_ttl,
-    )
-    .await
-    .context("failed to create new account")?;
+    let client = AcmeClient::new_account(&config.acme_url, validation, config.max_dns_wait)
+        .await
+        .context("failed to create new account")?;
     let credentials = client
         .dump_credentials()
         .context("failed to dump credentials")?;
@@ -87,39 +92,20 @@ async fn create_new_account(
 impl CertBot {
     /// Build a new `CertBot` from a `CertBotConfig`.
     pub async fn build(config: CertBotConfig) -> Result<Self> {
-        let base_domain = config
-            .cert_subject_alt_names
-            .first()
-            .context("cert_subject_alt_names is empty")?
-            .trim()
-            .trim_start_matches("*.")
-            .trim_end_matches('.')
-            .to_string();
-        let dns01_client = Dns01Client::new_cloudflare(
-            base_domain,
-            config.cf_api_token.clone(),
-            config.cf_api_url.clone(),
-        )
-        .await?;
+        let validation = build_validation_method(&config).await?;
         let acme_client = match fs::read_to_string(&config.credentials_file) {
             Ok(credentials) => {
                 if acme_matches(&credentials, &config.acme_url) {
-                    AcmeClient::load(
-                        dns01_client,
-                        &credentials,
-                        config.max_dns_wait,
-                        config.dns_txt_ttl,
-                    )
-                    .await?
+                    AcmeClient::load(validation, &credentials, config.max_dns_wait).await?
                 } else {
-                    create_new_account(&config, dns01_client).await?
+                    create_new_account(&config, validation).await?
                 }
             }
             Err(e) if e.kind() == ErrorKind::NotFound => {
                 if !config.auto_create_account {
                     return Err(e).context("credentials file not found");
                 }
-                create_new_account(&config, dns01_client).await?
+                create_new_account(&config, validation).await?
             }
             Err(e) => {
                 return Err(e).context("failed to read credentials file");
@@ -262,6 +248,60 @@ impl CertBot {
             .set_caa_records(&self.config.cert_subject_alt_names)
             .await
     }
+
+    /// The DNS records that have to exist for the configured domains.
+    pub fn required_dns_records(&self) -> Vec<RequiredRecord> {
+        self.acme_client
+            .required_dns_records(&self.config.cert_subject_alt_names)
+    }
+}
+
+/// Resolve the configured challenge into a live validation method.
+///
+/// `dns-01` resolves the Cloudflare zone here, which is an authenticated call,
+/// so a bad credential fails at startup rather than at the first renewal.
+/// `dns-persist-01` talks to no provider at all.
+async fn build_validation_method(config: &CertBotConfig) -> Result<ValidationMethod> {
+    match config.challenge {
+        ChallengeKind::Dns01 => {
+            let base_domain = config
+                .cert_subject_alt_names
+                .first()
+                .context("cert_subject_alt_names is empty")?
+                .trim()
+                .trim_start_matches("*.")
+                .trim_end_matches('.')
+                .to_string();
+            let client = Dns01Client::new_cloudflare(
+                base_domain,
+                config.cf_api_token.clone(),
+                config.cf_api_url.clone(),
+            )
+            .await?;
+            Ok(ValidationMethod::Dns01 {
+                client,
+                txt_ttl: config.dns_txt_ttl,
+            })
+        }
+        ChallengeKind::DnsPersist01 => {
+            // Refuse rather than silently skip: `auto_set_caa` promises the CAA
+            // records are kept in sync, and without DNS write access nothing here
+            // can keep that promise. `certbot dns-records` prints what to publish.
+            if config.auto_set_caa {
+                bail!(
+                    "auto_set_caa is not supported with dns-persist-01, which has no DNS \
+                     write access; set auto_set_caa = false and publish the records from \
+                     `certbot dns-records` by hand"
+                );
+            }
+            if !config.cf_api_token.is_empty() {
+                warn!("ignoring cf_api_token: dns-persist-01 needs no DNS provider credential");
+            }
+            Ok(ValidationMethod::DnsPersist01 {
+                issuer_domain_name: config.issuer_domain_name.clone(),
+            })
+        }
+    }
 }
 
 pub fn read_pubkey(cert_pem: &str) -> Result<Vec<u8>> {
diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs
new file mode 100644
index 000000000..4712a4901
--- /dev/null
+++ b/dstack/certbot/src/dns_persist.rs
@@ -0,0 +1,399 @@
+// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! The persistent DNS authorization record used by the `dns-persist-01` challenge.
+//!
+//! `dns-persist-01` (draft-ietf-acme-dns-persist-01) replaces the per-order
+//! `_acme-challenge` TXT record of `dns-01` with a single record that stays in the
+//! zone: `_validation-persist.<name>`, naming the CA and the ACME account allowed
+//! to issue for that name. The account key proves who is asking; the record proves
+//! the zone owner agreed. Nothing about it changes between orders, so a client that
+//! uses this method never needs write access to the zone.
+//!
+//! This module owns the record's syntax: rendering the line an operator has to
+//! publish, and deciding whether what is currently published would satisfy the CA.
+//! The RDATA is an RFC 8659 `issue-value` — the same grammar as a CAA `issue`
+//! record — so the parser here mirrors the one CAs run (see `va/dns_persist.go` in
+//! Boulder), including the parts that reject rather than ignore: a trailing
+//! semicolon, a repeated tag, whitespace inside a value.
+
+use std::fmt;
+
+use anyhow::{bail, Context, Result};
+
+/// Label prepended to the name being validated to form the validation domain name.
+///
+/// draft-ietf-acme-dns-persist-01, section 4.
+const VALIDATION_LABEL: &str = "_validation-persist";
+
+/// Issuer Domain Name for Let's Encrypt, matching the `caaIdentities` it advertises.
+///
+/// A CA lists the names it answers to in the challenge object's
+/// `issuer-domain-names`; a record naming anything else is ignored by that CA.
+pub const LETS_ENCRYPT_ISSUER_DOMAIN_NAME: &str = "letsencrypt.org";
+
+/// The `policy` value that widens a record to cover wildcards.
+const POLICY_WILDCARD: &str = "wildcard";
+
+/// The validation domain name for `name`, where the CA looks for the TXT record.
+///
+/// A wildcard request is authorized by the record on its base name: ACME strips the
+/// `*.` before creating the authorization, so `*.example.com` and `example.com`
+/// share `_validation-persist.example.com` and are told apart by `policy=wildcard`.
+pub fn validation_domain(name: &str) -> String {
+    let base = name.strip_prefix("*.").unwrap_or(name);
+    format!("{VALIDATION_LABEL}.{base}")
+}
+
+/// The record an operator has to publish for one name.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AuthorizationRecord {
+    /// Issuer Domain Name of the CA this record authorizes.
+    pub issuer_domain_name: String,
+    /// URI of the ACME account allowed to issue, compared byte for byte by the CA.
+    pub account_uri: String,
+    /// Whether the record also covers `*.<name>`.
+    pub wildcard: bool,
+}
+
+impl AuthorizationRecord {
+    /// The TXT RDATA to publish, as a single string.
+    ///
+    /// Rendered without a trailing semicolon: CAs read the RDATA as an RFC 8659
+    /// `issue-value`, where a trailing semicolon is an empty parameter and makes
+    /// the whole record malformed.
+    pub fn rdata(&self) -> String {
+        let mut rdata = format!(
+            "{}; accounturi={}",
+            self.issuer_domain_name, self.account_uri
+        );
+        if self.wildcard {
+            rdata.push_str("; policy=wildcard");
+        }
+        rdata
+    }
+
+    /// Whether `rdata` currently published at the validation domain satisfies this
+    /// record's requirement, as of `now` (UNIX seconds).
+    ///
+    /// Follows the CA's own filter: a record naming a different issuer is not a
+    /// failure, it belongs to another CA and is skipped. Only a record that names
+    /// our issuer is held to the account, policy and lifetime checks.
+    fn satisfied_by(&self, rdata: &str, now: u64) -> bool {
+        let Ok(parsed) = IssueValue::parse(rdata) else {
+            return false;
+        };
+        if parsed.issuer_domain_name != self.issuer_domain_name {
+            return false;
+        }
+        if parsed.account_uri != self.account_uri {
+            return false;
+        }
+        if parsed.persist_until.is_some_and(|until| now > until) {
+            return false;
+        }
+        // A record without `policy=wildcard` authorizes the exact name only, so it
+        // cannot stand in for a wildcard request. The reverse is fine: a wildcard
+        // record also covers the name itself.
+        !self.wildcard || parsed.policy.as_deref().is_some_and(is_wildcard_policy)
+    }
+
+    /// Whether any of the TXT records at the validation domain satisfies this one.
+    ///
+    /// Several records may sit at the same label, one per CA or per account, and
+    /// the CA accepts the name if any single record passes.
+    pub fn satisfied_by_any(&self, published: &[String], now: u64) -> bool {
+        published.iter().any(|rdata| self.satisfied_by(rdata, now))
+    }
+}
+
+impl fmt::Display for AuthorizationRecord {
+    /// Renders the full zone-file line, ready to paste into a DNS provider.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "IN TXT \"{}\"", self.rdata())
+    }
+}
+
+/// A parsed RFC 8659 `issue-value`, the grammar shared with CAA `issue` records.
+#[derive(Debug, PartialEq, Eq)]
+struct IssueValue {
+    issuer_domain_name: String,
+    account_uri: String,
+    policy: Option<String>,
+    /// UNIX timestamp after which the CA stops accepting the record.
+    persist_until: Option<u64>,
+}
+
+impl IssueValue {
+    fn parse(rdata: &str) -> Result<Self> {
+        let mut parts = rdata.split(';');
+        let issuer_domain_name = trim_wsp(parts.next().unwrap_or_default());
+        if issuer_domain_name.is_empty() {
+            bail!("missing issuer domain name");
+        }
+
+        let mut account_uri = None;
+        let mut policy = None;
+        let mut persist_until = None;
+        let mut seen = Vec::new();
+        for part in parts {
+            let part = trim_wsp(part);
+            // An empty parameter means a doubled or trailing semicolon. CAs treat
+            // that as malformed rather than skipping it, so neither do we.
+            if part.is_empty() {
+                bail!("empty parameter or trailing semicolon");
+            }
+            let (tag, value) = part.split_once('=').context("parameter is not tag=value")?;
+            // RFC 8659 matches tags case-insensitively; values are not folded.
+            let tag = tag.to_lowercase();
+            if !value.bytes().all(is_value_byte) {
+                bail!("parameter {tag} has a value with a forbidden character");
+            }
+            if seen.contains(&tag) {
+                bail!("duplicate parameter {tag}");
+            }
+            seen.push(tag.clone());
+            match tag.as_str() {
+                "accounturi" => account_uri = Some(value.to_string()),
+                "policy" => policy = Some(value.to_string()),
+                "persistuntil" => {
+                    persist_until = Some(
+                        value
+                            .parse::<u64>()
+                            .context("persistUntil is not a base-10 timestamp")?,
+                    )
+                }
+                // The draft requires unrecognized tags to be ignored, so that
+                // later revisions can add parameters without invalidating records.
+                _ => {}
+            }
+        }
+
+        Ok(Self {
+            issuer_domain_name: issuer_domain_name.to_string(),
+            account_uri: account_uri.context("missing mandatory accounturi parameter")?,
+            policy,
+            persist_until,
+        })
+    }
+}
+
+fn is_wildcard_policy(policy: &str) -> bool {
+    policy.eq_ignore_ascii_case(POLICY_WILDCARD)
+}
+
+/// Trim the whitespace RFC 8659 allows around the issuer name and each parameter.
+fn trim_wsp(part: &str) -> &str {
+    part.trim_matches([' ', '\t'])
+}
+
+/// Whether a byte may appear in a parameter value.
+///
+/// RFC 8659 allows printable ASCII except `;`, which excludes whitespace: a value
+/// containing a space is a malformed record, not a value with a space in it.
+fn is_value_byte(byte: u8) -> bool {
+    matches!(byte, 0x21..=0x3a | 0x3c..=0x7e)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// An arbitrary "now" that precedes every `persistUntil` used below.
+    const NOW: u64 = 1_700_000_000;
+    const ACCOUNT: &str = "https://acme-v02.api.letsencrypt.org/acme/acct/1234567890";
+
+    fn record(wildcard: bool) -> AuthorizationRecord {
+        AuthorizationRecord {
+            issuer_domain_name: LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string(),
+            account_uri: ACCOUNT.to_string(),
+            wildcard,
+        }
+    }
+
+    /// Whether `published` satisfies a request for `name`, wildcard or not.
+    fn accepts(wildcard: bool, published: &str) -> bool {
+        record(wildcard).satisfied_by(published, NOW)
+    }
+
+    #[test]
+    fn validation_domain_prepends_the_label() {
+        assert_eq!(
+            validation_domain("example.com"),
+            "_validation-persist.example.com"
+        );
+    }
+
+    #[test]
+    fn validation_domain_strips_the_wildcard_prefix() {
+        // The CA looks up the base name for a wildcard authorization, so a record
+        // at `_validation-persist.*.example.com` would never be read.
+        assert_eq!(
+            validation_domain("*.example.com"),
+            "_validation-persist.example.com"
+        );
+    }
+
+    #[test]
+    fn rdata_has_no_trailing_semicolon() {
+        assert_eq!(
+            record(false).rdata(),
+            format!("letsencrypt.org; accounturi={ACCOUNT}")
+        );
+    }
+
+    #[test]
+    fn rdata_carries_the_wildcard_policy() {
+        assert_eq!(
+            record(true).rdata(),
+            format!("letsencrypt.org; accounturi={ACCOUNT}; policy=wildcard")
+        );
+    }
+
+    #[test]
+    fn display_renders_a_zone_file_line() {
+        assert_eq!(
+            record(false).to_string(),
+            format!("IN TXT \"letsencrypt.org; accounturi={ACCOUNT}\"")
+        );
+    }
+
+    #[test]
+    fn rendered_record_parses_back() {
+        for wildcard in [false, true] {
+            assert!(
+                accepts(wildcard, &record(wildcard).rdata()),
+                "wildcard={wildcard}"
+            );
+        }
+    }
+
+    #[test]
+    fn accepts_the_draft_example_layout() {
+        // draft-ietf-acme-dns-persist-01, figure 2, plus the whitespace RFC 8659
+        // allows around the issuer name and each parameter.
+        let record = AuthorizationRecord {
+            issuer_domain_name: "authority.example".to_string(),
+            account_uri: "https://ca.example/acct/123".to_string(),
+            wildcard: false,
+        };
+        for published in [
+            "authority.example; accounturi=https://ca.example/acct/123",
+            "authority.example;accounturi=https://ca.example/acct/123",
+            "\tauthority.example ;\taccounturi=https://ca.example/acct/123 ",
+        ] {
+            assert!(record.satisfied_by(published, NOW), "{published:?}");
+        }
+    }
+
+    #[test]
+    fn matches_tags_case_insensitively() {
+        assert!(accepts(
+            true,
+            &format!("letsencrypt.org; AccountURI={ACCOUNT}; Policy=WILDCARD")
+        ));
+    }
+
+    #[test]
+    fn rejects_a_different_account_uri() {
+        assert!(!accepts(
+            false,
+            "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/9"
+        ));
+    }
+
+    #[test]
+    fn compares_the_account_uri_without_case_folding() {
+        // The draft pins Simple String Comparison, so an upper-cased URI is a
+        // different account as far as the CA is concerned.
+        assert!(!accepts(
+            false,
+            &format!("letsencrypt.org; accounturi={}", ACCOUNT.to_uppercase())
+        ));
+    }
+
+    #[test]
+    fn ignores_a_record_naming_another_issuer() {
+        assert!(!accepts(
+            false,
+            &format!("otherca.example; accounturi={ACCOUNT}")
+        ));
+    }
+
+    #[test]
+    fn rejects_a_plain_record_for_a_wildcard_request() {
+        let published = record(false).rdata();
+        assert!(accepts(false, &published));
+        assert!(!accepts(true, &published));
+    }
+
+    #[test]
+    fn accepts_a_wildcard_record_for_a_plain_request() {
+        assert!(accepts(false, &record(true).rdata()));
+    }
+
+    #[test]
+    fn rejects_a_trailing_semicolon() {
+        // A CA reads the empty tail as an empty parameter and fails the whole
+        // record, so a record rendered with one would be silently unusable.
+        assert!(!accepts(
+            false,
+            &format!("letsencrypt.org; accounturi={ACCOUNT};")
+        ));
+    }
+
+    #[test]
+    fn rejects_a_duplicate_parameter() {
+        assert!(!accepts(
+            false,
+            &format!("letsencrypt.org; accounturi={ACCOUNT}; accounturi={ACCOUNT}")
+        ));
+    }
+
+    #[test]
+    fn rejects_whitespace_inside_a_value() {
+        assert!(!accepts(
+            false,
+            "letsencrypt.org; accounturi=https://ca.example/acct/1 2"
+        ));
+    }
+
+    #[test]
+    fn rejects_a_record_without_accounturi() {
+        assert!(!accepts(false, "letsencrypt.org; policy=wildcard"));
+    }
+
+    #[test]
+    fn ignores_unrecognized_parameters() {
+        assert!(accepts(
+            true,
+            &format!("letsencrypt.org; accounturi={ACCOUNT}; policy=wildcard; futuretag=whatever")
+        ));
+    }
+
+    #[test]
+    fn rejects_an_expired_persist_until() {
+        let published = format!("letsencrypt.org; accounturi={ACCOUNT}; persistUntil={NOW}");
+        assert!(record(false).satisfied_by(&published, NOW));
+        assert!(!record(false).satisfied_by(&published, NOW + 1));
+    }
+
+    #[test]
+    fn rejects_a_malformed_persist_until() {
+        assert!(!accepts(
+            false,
+            &format!("letsencrypt.org; accounturi={ACCOUNT}; persistUntil=tomorrow")
+        ));
+    }
+
+    #[test]
+    fn accepts_any_one_of_the_published_records() {
+        let published = vec![
+            "otherca.example; accounturi=https://other.example/acct/1".to_string(),
+            record(true).rdata(),
+        ];
+        assert!(record(true).satisfied_by_any(&published, NOW));
+        assert!(!record(true).satisfied_by_any(&published[..1], NOW));
+    }
+}
diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs
index df71b9935..7d70f1f07 100644
--- a/dstack/certbot/src/lib.rs
+++ b/dstack/certbot/src/lib.rs
@@ -11,18 +11,23 @@
 //!
 //! - Automatic certificate issuance and renewal
 //! - DNS-01 challenge support (currently implemented for Cloudflare)
+//! - DNS-PERSIST-01 challenge support, which needs no DNS provider credential
 //! - Easy integration with existing Rust applications
 //!
 //! For more detailed information on the available methods and their usage, please refer
 //! to the documentation of individual structs and functions.
 
-pub use acme_client::AcmeClient;
+pub use acme_client::{
+    required_dns_records, AcmeClient, ChallengeKind, RequiredRecord, ValidationMethod,
+};
 pub use bot::{read_pubkey, CertBot, CertBotConfig};
 pub use dns01_client::Dns01Client;
+pub use dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME;
 pub use workdir::WorkDir;
 
 mod acme_client;
 mod bot;
 mod dns01_client;
+mod dns_persist;
 mod http_client;
 mod workdir;
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 288b3403e..0b713c3fd 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -11,7 +11,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{bail, Context, Result};
-use certbot::{AcmeClient, Dns01Client};
+use certbot::{AcmeClient, Dns01Client, ValidationMethod};
 use dstack_guest_agent_rpc::v0::RawQuoteArgs;
 use ra_tls::attestation::QuoteContentType;
 use ra_tls::rcgen::KeyPair;
@@ -111,13 +111,17 @@ impl DistributedCertBot {
         Ok(())
     }
 
-    async fn dns_client(&self, domain: &str, dns_cred: &DnsCredential) -> Result<Dns01Client> {
-        match &dns_cred.provider {
+    async fn dns_client(&self, domain: &str, dns_cred: &DnsCredential) -> Result<ValidationMethod> {
+        let client = match &dns_cred.provider {
             DnsProvider::Cloudflare { api_token, api_url } => {
                 Dns01Client::new_cloudflare(domain.to_string(), api_token.clone(), api_url.clone())
-                    .await
+                    .await?
             }
-        }
+        };
+        Ok(ValidationMethod::Dns01 {
+            client,
+            txt_ttl: dns_cred.dns_txt_ttl,
+        })
     }
 
     /// Rotate the shared ACME account without interrupting certificate serving.
@@ -179,14 +183,9 @@ impl DistributedCertBot {
             bail!("no ZT-Domain configured for ACME credential rotation");
         };
 
-        let client = AcmeClient::new_account(
-            acme_url,
-            first_client,
-            first_cred.max_dns_wait,
-            first_cred.dns_txt_ttl,
-        )
-        .await
-        .context("failed to create replacement ACME account")?;
+        let client = AcmeClient::new_account(acme_url, first_client, first_cred.max_dns_wait)
+            .await
+            .context("failed to create replacement ACME account")?;
         let credentials = client
             .dump_credentials()
             .context("failed to encode replacement ACME credentials")?;
@@ -221,14 +220,9 @@ impl DistributedCertBot {
         );
         for (domain, dns_cred, dns_client) in prepared {
             let result = async {
-                let client = AcmeClient::load(
-                    dns_client,
-                    &credentials,
-                    dns_cred.max_dns_wait,
-                    dns_cred.dns_txt_ttl,
-                )
-                .await
-                .context("failed to prepare ACME client")?;
+                let client = AcmeClient::load(dns_client, &credentials, dns_cred.max_dns_wait)
+                    .await
+                    .context("failed to prepare ACME client")?;
                 client
                     .set_caa_records(std::slice::from_ref(domain))
                     .await
@@ -689,14 +683,9 @@ impl DistributedCertBot {
             );
         }
         let dns01_client = self.dns_client(domain, dns_cred).await?;
-        let client = AcmeClient::load(
-            dns01_client,
-            &creds.acme_credentials,
-            dns_cred.max_dns_wait,
-            dns_cred.dns_txt_ttl,
-        )
-        .await
-        .context("failed to load ACME client from KvStore credentials")?;
+        let client = AcmeClient::load(dns01_client, &creds.acme_credentials, dns_cred.max_dns_wait)
+            .await
+            .context("failed to load ACME client from KvStore credentials")?;
         Ok(Some(client))
     }
 
@@ -722,14 +711,9 @@ impl DistributedCertBot {
 
         info!("creating new global ACME account at {acme_url}");
         let dns01_client = self.dns_client(domain, dns_cred).await?;
-        let client = AcmeClient::new_account(
-            acme_url,
-            dns01_client,
-            dns_cred.max_dns_wait,
-            dns_cred.dns_txt_ttl,
-        )
-        .await
-        .context("failed to create new ACME account")?;
+        let client = AcmeClient::new_account(acme_url, dns01_client, dns_cred.max_dns_wait)
+            .await
+            .context("failed to create new ACME account")?;
 
         let creds_json = client
             .dump_credentials()

From 72bbcae411067920e3b366dbed08dc94353c95a5 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 02:50:00 -0700
Subject: [PATCH 02/37] feat(certbot): print the one-time dns-persist-01
 records from the CLI

Under dns-persist-01 certbot cannot write the records it needs, so the
records are the setup. `certbot dns-records` prints them as zone-file lines
for the configured domains, ready to paste into any provider, once
`certbot init` has registered the account they name.

`challenge` and `issuer_domain_name` join certbot.toml, and `cf_api_token`
becomes optional -- a token left configured alongside dns-persist-01 is
warned about rather than silently ignored. `auto_set_caa` is refused outright
with dns-persist-01: it promises certbot keeps CAA in sync, and without write
access nothing can keep that promise.
---
 dstack/certbot/cli/src/main.rs | 47 ++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/dstack/certbot/cli/src/main.rs b/dstack/certbot/cli/src/main.rs
index 68348b5f9..f72838466 100644
--- a/dstack/certbot/cli/src/main.rs
+++ b/dstack/certbot/cli/src/main.rs
@@ -6,7 +6,7 @@
 use std::{path::PathBuf, time::Duration};
 
 use anyhow::{Context, Result};
-use certbot::{CertBotConfig, WorkDir};
+use certbot::{CertBotConfig, ChallengeKind, WorkDir, LETS_ENCRYPT_ISSUER_DOMAIN_NAME};
 use clap::Parser;
 use documented::DocumentedFields;
 use fs_err as fs;
@@ -40,6 +40,15 @@ enum Command {
         #[arg(short, long, default_value = "certbot.toml")]
         config: PathBuf,
     },
+    /// Print the DNS records the configured domains need
+    ///
+    /// With `challenge = "dns-persist-01"` these are not written by certbot and
+    /// have to be published once, by hand, before the first issuance.
+    DnsRecords {
+        /// Path to the configuration file
+        #[arg(short, long, default_value = "certbot.toml")]
+        config: PathBuf,
+    },
     /// Generate configuration template
     Cfg {
         /// Write to file
@@ -60,7 +69,23 @@ struct Config {
     workdir: PathBuf,
     /// ACME server URL
     acme_url: String,
-    /// Cloudflare API token
+    /// ACME challenge used to prove control of the domains
+    ///
+    /// "dns-01" (default) writes a TXT record per order through the Cloudflare
+    /// API and needs cf_api_token.
+    ///
+    /// "dns-persist-01" proves control with a _validation-persist TXT record
+    /// published once, by hand: no API token, and the zone can be hosted
+    /// anywhere. Run `certbot init` then `certbot dns-records` to get the
+    /// records to publish. Experimental: the draft is still changing and
+    /// Let's Encrypt serves this challenge on staging only.
+    #[serde(default)]
+    challenge: ChallengeKind,
+    /// Issuer Domain Name naming the CA in dns-persist-01 and CAA records
+    #[serde(default = "default_issuer_domain_name")]
+    issuer_domain_name: String,
+    /// Cloudflare API token (unused with dns-persist-01)
+    #[serde(default)]
     cf_api_token: String,
     /// Optional Cloudflare-compatible API base URL
     #[serde(default)]
@@ -90,6 +115,8 @@ impl Default for Config {
         Self {
             workdir: ".".into(),
             acme_url: "https://acme-staging-v02.api.letsencrypt.org/directory".into(),
+            challenge: ChallengeKind::default(),
+            issuer_domain_name: default_issuer_domain_name(),
             cf_api_token: "".into(),
             cf_api_url: None,
             dns_txt_ttl: default_dns_txt_ttl(),
@@ -108,6 +135,10 @@ const fn default_dns_txt_ttl() -> u32 {
     60
 }
 
+fn default_issuer_domain_name() -> String {
+    LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string()
+}
+
 impl Config {
     fn to_commented_toml(&self) -> Result<String> {
         let mut doc = to_document(self)?;
@@ -152,6 +183,8 @@ fn load_config(config: &PathBuf) -> Result<CertBotConfig> {
         .key_file(workdir.key_path())
         .auto_create_account(true)
         .cert_subject_alt_names(config.domains)
+        .challenge(config.challenge)
+        .issuer_domain_name(config.issuer_domain_name)
         .cf_api_token(config.cf_api_token)
         .maybe_cf_api_url(config.cf_api_url)
         .dns_txt_ttl(config.dns_txt_ttl)
@@ -231,6 +264,16 @@ async fn main() -> Result<()> {
                 .context("Failed to build bot")?;
             bot.set_caa().await?;
         }
+        Command::DnsRecords { config } => {
+            let bot_config = load_config(&config).context("Failed to load configuration")?;
+            let bot = bot_config
+                .build_bot()
+                .await
+                .context("Failed to build bot")?;
+            for record in bot.required_dns_records() {
+                println!("{record}");
+            }
+        }
         Command::Cfg { write_to } => {
             let toml_str = Config::default().to_commented_toml()?;
             match write_to {

From b6834d401f1158cb08969c0532dffcd6bdd94048 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 02:50:46 -0700
Subject: [PATCH 03/37] feat(gateway): let a ZT domain validate with
 dns-persist-01

A gateway CVM running dns-01 holds a Cloudflare token with write access to
the operator's whole zone. Attestation covers what the CVM runs, not what
becomes of a secret it holds, so that token is the widest credential in the
deployment and it exists only to write one TXT record per order.
dns-persist-01 removes it: control comes from a `_validation-persist` record
the operator publishes once, and the CVM never gets DNS write access at all.

`ZtDomainConfig.challenge` picks the method per domain and defaults to
dns-01, so records written before the field existed decode as the method
those deployments were using -- pinned by a test over both the named and the
legacy positional msgpack encodings. Such a domain needs no DNS credential,
and `validation_for` never looks one up for it.

`GetZtDomain` and `ListZtDomains` return the records to publish in
`required_dns_records`, rendered from the stored account URI with no ACME
round trip so the listing endpoints stay cheap; it comes back empty rather
than failing when no account exists yet.

Two operations cannot be self-service for such a domain, and say so rather
than failing silently:

- `SetCaa` skips it and logs the records instead. There is nothing to
  reconcile without write access, and one such domain must not make the RPC
  unusable for the dns-01 domains beside it; the summary reports how many
  were left to the operator.
- `RotateAcmeCredentials` moves the cluster to a new account while every
  `_validation-persist` record still names the old one, so orders for those
  domains fail until the operator republishes. The response now carries the
  new records in `required_dns_records`, rendered after the switch so they
  name the account the cluster actually moved to, and `domains_updated`
  counts only the domains whose CAA was re-pinned.
---
 dstack/gateway/rpc/proto/gateway_rpc.proto |  16 ++
 dstack/gateway/src/admin_service.rs        |  43 +++-
 dstack/gateway/src/distributed_certbot.rs  | 278 +++++++++++++++++----
 dstack/gateway/src/kv/mod.rs               |  54 ++++
 dstack/gateway/src/main_service.rs         |   4 +-
 5 files changed, 334 insertions(+), 61 deletions(-)

diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto
index 4935974e5..7700e808b 100644
--- a/dstack/gateway/rpc/proto/gateway_rpc.proto
+++ b/dstack/gateway/rpc/proto/gateway_rpc.proto
@@ -210,6 +210,10 @@ message RotateAcmeCredentialsResponse {
   string account_uri = 1;
   // Number of ZT domains whose CAA records were updated for the new account.
   uint32 domains_updated = 2;
+  // Zone-file lines an operator must publish by hand, for domains the gateway
+  // cannot write (dns-persist-01). Non-empty means issuance for those domains
+  // stays broken until they name the new account.
+  repeated string required_dns_records = 3;
 }
 
 // Get HostInfo for associated instance id.
@@ -742,6 +746,14 @@ message ZtDomainConfig {
   optional uint32 node = 4;
   // Priority for default base_domain selection (higher = preferred)
   int32 priority = 5;
+  // ACME challenge proving control of this domain: "dns-01" (default) or
+  // "dns-persist-01". Empty means "dns-01".
+  //
+  // "dns-persist-01" needs no DNS credential: control comes from a
+  // _validation-persist TXT record the operator publishes once, so the gateway
+  // CVM never holds write access to the zone. Experimental — the draft is still
+  // changing and Let's Encrypt serves it on staging only.
+  string challenge = 6;
 }
 
 // ZT-Domain information (config + certificate status)
@@ -750,6 +762,10 @@ message ZtDomainInfo {
   ZtDomainConfig config = 1;
   // Certificate status
   ZtDomainCertStatus cert_status = 2;
+  // Zone-file lines the domain's DNS must contain. Under "dns-01" the gateway
+  // writes these itself and they are informational; under "dns-persist-01" they
+  // are the one-time setup an operator has to publish.
+  repeated string required_dns_records = 3;
 }
 
 // ZT-Domain certificate status
diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs
index b9f5e6955..87227fbfd 100644
--- a/dstack/gateway/src/admin_service.rs
+++ b/dstack/gateway/src/admin_service.rs
@@ -6,6 +6,7 @@ use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{bail, ensure, Context, Result};
+use certbot::ChallengeKind;
 use dstack_gateway_rpc::{
     admin_server::{AdminRpc, AdminServer},
     CertAttestationInfo, CertbotConfigResponse, ClearInstancePortPolicyRequest,
@@ -110,10 +111,11 @@ impl AdminRpc for AdminRpcHandler {
     }
 
     async fn rotate_acme_credentials(self) -> Result<RotateAcmeCredentialsResponse> {
-        let (account_uri, domains_updated) = self.state.rotate_acme_credentials().await?;
+        let outcome = self.state.rotate_acme_credentials().await?;
         Ok(RotateAcmeCredentialsResponse {
-            account_uri,
-            domains_updated: domains_updated.try_into().unwrap_or(u32::MAX),
+            account_uri: outcome.account_uri,
+            domains_updated: outcome.domains_updated.try_into().unwrap_or(u32::MAX),
+            required_dns_records: outcome.required_dns_records,
         })
     }
 
@@ -554,11 +556,15 @@ impl AdminRpc for AdminRpcHandler {
     async fn list_zt_domains(self) -> Result<ListZtDomainsResponse> {
         let kv_store = self.state.kv_store();
         let cert_resolver = &self.state.cert_resolver;
+        let certbot = &self.state.certbot;
 
         let domains = kv_store
             .list_zt_domain_configs()
             .into_iter()
-            .map(|config| zt_domain_to_proto(config, kv_store, cert_resolver))
+            .map(|config| {
+                let records = certbot.required_dns_records(&config);
+                zt_domain_to_proto(config, kv_store, cert_resolver, records)
+            })
             .collect();
 
         Ok(ListZtDomainsResponse { domains })
@@ -573,7 +579,8 @@ impl AdminRpc for AdminRpcHandler {
             .get_zt_domain_config(&domain)
             .context("ZT-Domain config not found")?;
 
-        Ok(zt_domain_to_proto(config, kv_store, cert_resolver))
+        let records = self.state.certbot.required_dns_records(&config);
+        Ok(zt_domain_to_proto(config, kv_store, cert_resolver, records))
     }
 
     async fn add_zt_domain(self, request: ProtoZtDomainConfig) -> Result<ZtDomainInfo> {
@@ -591,7 +598,8 @@ impl AdminRpc for AdminRpcHandler {
         kv_store.save_zt_domain_config(&config)?;
         info!("Added ZT-Domain config: {}", config.domain);
 
-        Ok(zt_domain_to_proto(config, kv_store, cert_resolver))
+        let records = self.state.certbot.required_dns_records(&config);
+        Ok(zt_domain_to_proto(config, kv_store, cert_resolver, records))
     }
 
     async fn update_zt_domain(self, request: ProtoZtDomainConfig) -> Result<ZtDomainInfo> {
@@ -608,7 +616,8 @@ impl AdminRpc for AdminRpcHandler {
         kv_store.save_zt_domain_config(&config)?;
         info!("Updated ZT-Domain config: {}", config.domain);
 
-        Ok(zt_domain_to_proto(config, kv_store, cert_resolver))
+        let records = self.state.certbot.required_dns_records(&config);
+        Ok(zt_domain_to_proto(config, kv_store, cert_resolver, records))
     }
 
     async fn delete_zt_domain(self, request: DeleteZtDomainRequest) -> Result<()> {
@@ -976,20 +985,33 @@ fn proto_to_zt_domain_config(
         bail!("port must be between 1 and 65535");
     }
 
+    // Empty means the historical default: every ZT domain predates the choice.
+    let challenge = match proto.challenge.as_str() {
+        "" | "dns-01" => ChallengeKind::Dns01,
+        "dns-persist-01" => ChallengeKind::DnsPersist01,
+        other => bail!("unsupported challenge {other:?}, expected dns-01 or dns-persist-01"),
+    };
+
     Ok(ZtDomainConfig {
         domain,
         dns_cred_id,
         port: proto.port.try_into().context("port out of range")?,
         node: proto.node,
         priority: proto.priority,
+        challenge,
     })
 }
 
 /// Convert internal ZtDomainConfig to proto ZtDomainInfo (with cert status)
+///
+/// `required_dns_records` is best effort: rendering it needs the ACME account
+/// URI, and a domain whose ACME client cannot be built yet still has to be
+/// listable. It comes back empty in that case rather than failing the call.
 fn zt_domain_to_proto(
     config: ZtDomainConfig,
     kv_store: &crate::kv::KvStore,
     cert_resolver: &crate::cert_store::CertResolver,
+    required_dns_records: Vec<String>,
 ) -> ZtDomainInfo {
     // Get certificate data for status
     let cert_data = kv_store.get_cert_data(&config.domain);
@@ -1003,6 +1025,11 @@ fn zt_domain_to_proto(
         loaded_in_memory,
     });
 
+    let challenge = match config.challenge {
+        ChallengeKind::Dns01 => "dns-01",
+        ChallengeKind::DnsPersist01 => "dns-persist-01",
+    };
+
     ZtDomainInfo {
         config: Some(ProtoZtDomainConfig {
             domain: config.domain,
@@ -1010,8 +1037,10 @@ fn zt_domain_to_proto(
             port: config.port.into(),
             node: config.node,
             priority: config.priority,
+            challenge: challenge.to_string(),
         }),
         cert_status,
+        required_dns_records,
     }
 }
 
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 0b713c3fd..af1d0d88a 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -11,7 +11,9 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{bail, Context, Result};
-use certbot::{AcmeClient, Dns01Client, ValidationMethod};
+use certbot::{
+    AcmeClient, ChallengeKind, Dns01Client, ValidationMethod, LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+};
 use dstack_guest_agent_rpc::v0::RawQuoteArgs;
 use ra_tls::attestation::QuoteContentType;
 use ra_tls::rcgen::KeyPair;
@@ -33,6 +35,42 @@ const ROTATION_LOCK_TIMEOUT_SECS: u64 = 600;
 /// Default ACME URL (Let's Encrypt production)
 const DEFAULT_ACME_URL: &str = "https://acme-v02.api.letsencrypt.org/directory";
 
+/// How long a `dns-persist-01` domain waits for its record to become visible.
+///
+/// A `dns-01` domain reads this from its DNS credential, and a `dns-persist-01`
+/// domain has none. It stays a constant rather than a config knob because the
+/// wait is advisory: certbot polls its own resolver and starts the order either
+/// way, so the value cannot decide whether issuance succeeds. Matches the
+/// default `max_dns_wait` of a DNS credential.
+const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(300);
+
+/// What an ACME credential rotation left behind.
+#[derive(Debug)]
+pub struct RotationOutcome {
+    /// URI of the new ACME account.
+    pub account_uri: String,
+    /// Domains whose CAA records were re-pinned to the new account.
+    pub domains_updated: usize,
+    /// Records an operator has to publish by hand, for domains the gateway
+    /// cannot write. Non-empty means issuance for those domains is broken until
+    /// they are published.
+    pub required_dns_records: Vec<String>,
+}
+
+/// How one ZT domain answers ACME challenges.
+struct DomainValidation {
+    method: ValidationMethod,
+    /// How long to wait for the challenge records to become visible.
+    max_dns_wait: Duration,
+}
+
+impl DomainValidation {
+    /// Whether the gateway can write this domain's DNS records itself.
+    fn writes_dns(&self) -> bool {
+        matches!(self.method, ValidationMethod::Dns01 { .. })
+    }
+}
+
 /// Multi-domain certificate manager
 pub struct DistributedCertBot {
     kv_store: Arc<KvStore>,
@@ -111,19 +149,101 @@ impl DistributedCertBot {
         Ok(())
     }
 
-    async fn dns_client(&self, domain: &str, dns_cred: &DnsCredential) -> Result<ValidationMethod> {
-        let client = match &dns_cred.provider {
+    async fn dns_client(&self, domain: &str, dns_cred: &DnsCredential) -> Result<Dns01Client> {
+        match &dns_cred.provider {
             DnsProvider::Cloudflare { api_token, api_url } => {
                 Dns01Client::new_cloudflare(domain.to_string(), api_token.clone(), api_url.clone())
-                    .await?
+                    .await
             }
-        };
-        Ok(ValidationMethod::Dns01 {
-            client,
-            txt_ttl: dns_cred.dns_txt_ttl,
+        }
+    }
+
+    /// Build the ACME validation method for one ZT domain.
+    ///
+    /// `dns-01` resolves the zone through the provider API, so a missing or
+    /// broken credential surfaces here rather than at the first order.
+    /// `dns-persist-01` reads no credential at all: the gateway CVM holds no
+    /// DNS write access for that domain, which is the point of the method.
+    async fn validation_for(
+        &self,
+        domain: &str,
+        config: &ZtDomainConfig,
+    ) -> Result<DomainValidation> {
+        match config.challenge {
+            ChallengeKind::Dns01 => {
+                let dns_cred = dns_credential_for(&self.kv_store, config)?;
+                let client = self.dns_client(domain, &dns_cred).await?;
+                Ok(DomainValidation {
+                    method: ValidationMethod::Dns01 {
+                        client,
+                        txt_ttl: dns_cred.dns_txt_ttl,
+                    },
+                    max_dns_wait: dns_cred.max_dns_wait,
+                })
+            }
+            ChallengeKind::DnsPersist01 => Ok(DomainValidation {
+                method: ValidationMethod::DnsPersist01 {
+                    issuer_domain_name: self.issuer_domain_name()?,
+                },
+                max_dns_wait: DNS_PERSIST_MAX_DNS_WAIT,
+            }),
+        }
+    }
+
+    /// Issuer Domain Name to name in `dns-persist-01` and CAA records.
+    fn issuer_domain_name(&self) -> Result<String> {
+        let configured = self.config()?.issuer_domain_name;
+        Ok(match configured.is_empty() {
+            true => LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string(),
+            false => configured,
         })
     }
 
+    /// The DNS records a ZT domain's zone has to contain, as zone-file lines.
+    ///
+    /// Rendered from the stored account URI with no ACME round trip, so the
+    /// admin listing endpoints stay cheap. Empty until an ACME account exists —
+    /// every record names one — and empty rather than an error when the stored
+    /// credentials cannot be read, since listing a domain must not depend on
+    /// them.
+    pub fn required_dns_records(&self, config: &ZtDomainConfig) -> Vec<String> {
+        let Ok(issuer_domain_name) = self.issuer_domain_name() else {
+            return Vec::new();
+        };
+        let Some(account_uri) = self
+            .kv_store
+            .get_acme_credentials()
+            .ok()
+            .flatten()
+            .and_then(|creds| extract_account_uri(&creds.acme_credentials))
+        else {
+            return Vec::new();
+        };
+        // The gateway only ever orders `*.{domain}`, so that is the whole set of
+        // identifiers the CA will look records up for.
+        certbot::required_dns_records(
+            config.challenge,
+            &issuer_domain_name,
+            &account_uri,
+            &[format!("*.{}", config.domain)],
+        )
+        .iter()
+        .map(ToString::to_string)
+        .collect()
+    }
+
+    /// Log the records an operator has to publish for a domain, and return them.
+    fn report_manual_records(&self, config: &ZtDomainConfig) -> Vec<String> {
+        let records = self.required_dns_records(config);
+        for record in &records {
+            warn!(
+                "cert[{}]: publish this record by hand: {record}",
+                config.domain
+            );
+        }
+        records
+    }
+
     /// Rotate the shared ACME account without interrupting certificate serving.
     ///
     /// The sequence is: validate every domain's DNS credential, create the
@@ -146,7 +266,7 @@ impl DistributedCertBot {
     ///
     /// Runs under the shared ACME lock, so a concurrent reconciliation or
     /// first-use registration on any node is refused for the duration.
-    pub async fn rotate_acme_credentials(&self) -> Result<(String, usize)> {
+    pub async fn rotate_acme_credentials(&self) -> Result<RotationOutcome> {
         let rotation_lock = self.acquire_acme_lock("rotate ACME credentials")?;
         let result = self.do_rotate_acme_credentials().await;
         if let Err(err) = self.release_rotation_lock(&rotation_lock) {
@@ -155,7 +275,7 @@ impl DistributedCertBot {
         result
     }
 
-    async fn do_rotate_acme_credentials(&self) -> Result<(String, usize)> {
+    async fn do_rotate_acme_credentials(&self) -> Result<RotationOutcome> {
         let configs = self.kv_store.list_zt_domain_configs();
         let certbot_config = self.config()?;
         let acme_url = if certbot_config.acme_url.is_empty() {
@@ -167,25 +287,30 @@ impl DistributedCertBot {
         // Validate every domain's DNS credential up front: constructing a DNS
         // client resolves the zone through an authenticated API call, so a
         // misconfigured domain aborts the rotation here with no side effects
-        // and no ACME account consumed.
+        // and no ACME account consumed. A dns-persist-01 domain has no
+        // credential to check and is prepared without touching any provider.
         let mut prepared = Vec::with_capacity(configs.len());
         for config in &configs {
-            let dns_cred = dns_credential_for(&self.kv_store, config)?;
-            let dns_client = self
-                .dns_client(&config.domain, &dns_cred)
+            let validation = self
+                .validation_for(&config.domain, config)
                 .await
                 .with_context(|| format!("DNS credential check failed for {}", config.domain))?;
-            prepared.push((&config.domain, dns_cred, dns_client));
+            prepared.push((config, validation));
         }
         let total = prepared.len();
         let mut prepared = prepared.into_iter();
-        let Some((first_domain, first_cred, first_client)) = prepared.next() else {
+        let Some((first_config, first_validation)) = prepared.next() else {
             bail!("no ZT-Domain configured for ACME credential rotation");
         };
+        let first_writes_dns = first_validation.writes_dns();
 
-        let client = AcmeClient::new_account(acme_url, first_client, first_cred.max_dns_wait)
-            .await
-            .context("failed to create replacement ACME account")?;
+        let client = AcmeClient::new_account(
+            acme_url,
+            first_validation.method,
+            first_validation.max_dns_wait,
+        )
+        .await
+        .context("failed to create replacement ACME account")?;
         let credentials = client
             .dump_credentials()
             .context("failed to encode replacement ACME credentials")?;
@@ -202,36 +327,59 @@ impl DistributedCertBot {
         // Re-pin every domain's CAA to the new account, best effort across all
         // domains: one failing domain must not block re-pinning the rest. The
         // first domain reuses the registration client, which is already bound
-        // to its DNS client and the new credentials.
+        // to its validation method and the new credentials.
+        //
+        // A dns-persist-01 domain has nothing to re-pin from here, and rotation
+        // is not transparent for it either: its `_validation-persist` record
+        // still names the *old* account, so orders for that domain fail until
+        // the operator republishes. Those records are reported below.
         let mut failed = Vec::new();
-        let mut record = |domain: &String, result: Result<()>| match result {
+        let mut manual_domains = 0usize;
+        let mut repin = |domain: &str, result: Result<()>| match result {
             Ok(()) => info!("cert[{domain}]: CAA re-pinned to {account_uri}"),
             Err(err) => {
                 error!("cert[{domain}]: failed to re-pin CAA: {err:?}");
-                failed.push(domain.clone());
+                failed.push(domain.to_string());
             }
         };
-        record(
-            first_domain,
-            client
-                .set_caa_records(std::slice::from_ref(first_domain))
-                .await
-                .context("failed to update CAA records"),
-        );
-        for (domain, dns_cred, dns_client) in prepared {
-            let result = async {
-                let client = AcmeClient::load(dns_client, &credentials, dns_cred.max_dns_wait)
+        if first_writes_dns {
+            repin(
+                &first_config.domain,
+                client
+                    .set_caa_records(std::slice::from_ref(&first_config.domain))
                     .await
-                    .context("failed to prepare ACME client")?;
+                    .context("failed to update CAA records"),
+            );
+        } else {
+            manual_domains += 1;
+        }
+        for (config, validation) in prepared {
+            if !validation.writes_dns() {
+                manual_domains += 1;
+                continue;
+            }
+            let result = async {
+                let client =
+                    AcmeClient::load(validation.method, &credentials, validation.max_dns_wait)
+                        .await
+                        .context("failed to prepare ACME client")?;
                 client
-                    .set_caa_records(std::slice::from_ref(domain))
+                    .set_caa_records(std::slice::from_ref(&config.domain))
                     .await
                     .context("failed to update CAA records")
             }
             .await;
-            record(domain, result);
+            repin(&config.domain, result);
         }
 
+        // Rendered after the new credentials are published, so the records name
+        // the account the cluster has actually switched to.
+        let manual = configs
+            .iter()
+            .filter(|config| config.challenge != ChallengeKind::Dns01)
+            .flat_map(|config| self.report_manual_records(config))
+            .collect::<Vec<_>>();
+
         // Attest the new account only after CAA re-pinning: attestation does
         // not gate issuance, so its agent round trips must not widen the
         // window where the published account and the CAA records disagree.
@@ -250,7 +398,17 @@ impl DistributedCertBot {
                 failed.join(", ")
             );
         }
-        Ok((account_uri, total))
+        if manual_domains > 0 {
+            warn!(
+                "{manual_domains}/{total} domains use dns-persist-01: issuance for them stays \
+                 broken until the records above name the new account {account_uri}"
+            );
+        }
+        Ok(RotationOutcome {
+            account_uri,
+            domains_updated: total - manual_domains,
+            required_dns_records: manual,
+        })
     }
 
     /// Get the current certbot configuration from KV store.
@@ -370,8 +528,17 @@ impl DistributedCertBot {
     async fn do_set_caa_all(&self, configs: Vec<ZtDomainConfig>) -> Result<()> {
         let total = configs.len();
         let mut failed = Vec::new();
+        let mut skipped = 0usize;
         for config in configs {
             let domain = config.domain.clone();
+            // A dns-persist-01 domain has no DNS write access to reconcile with.
+            // Log the records the operator owns instead of failing the call, so
+            // one such domain does not make SetCaa unusable for the rest.
+            if config.challenge != ChallengeKind::Dns01 {
+                self.report_manual_records(&config);
+                skipped += 1;
+                continue;
+            }
             match self.set_caa(&domain, &config).await {
                 Ok(()) => info!("cert[{domain}]: CAA records reconciled"),
                 Err(err) => {
@@ -389,7 +556,10 @@ impl DistributedCertBot {
                 failed.join(", ")
             );
         }
-        info!("CAA records reconciled for {total} domains");
+        info!(
+            "CAA records reconciled for {} domains ({skipped} on dns-persist-01 left to the operator)",
+            total - skipped
+        );
         Ok(())
     }
 
@@ -407,10 +577,9 @@ impl DistributedCertBot {
         // the lock may call [`Self::acquire_acme_lock`], and keeping the
         // registering variant out of this path is what makes that structural
         // rather than a rule to remember.
-        let dns_cred = dns_credential_for(&self.kv_store, config)?;
         let acme_url = self.acme_url()?;
         let acme_client = self
-            .load_stored_acme_client(domain, &dns_cred, &acme_url)
+            .load_stored_acme_client(domain, config, &acme_url)
             .await
             .context("failed to initialize ACME client")?
             .context("no shared ACME account is registered for this cluster")?;
@@ -621,12 +790,10 @@ impl DistributedCertBot {
         domain: &str,
         config: &ZtDomainConfig,
     ) -> Result<AcmeClient> {
-        // Get DNS credential (from config or default)
-        let dns_cred = dns_credential_for(&self.kv_store, config)?;
         let acme_url = self.acme_url()?;
 
         if let Some(client) = self
-            .load_stored_acme_client(domain, &dns_cred, &acme_url)
+            .load_stored_acme_client(domain, config, &acme_url)
             .await?
         {
             info!("loaded global ACME account credentials from KvStore");
@@ -644,7 +811,7 @@ impl DistributedCertBot {
         // lock and look again before spending a registration.
         let rotation_lock = self.acquire_acme_lock("register the shared ACME account")?;
         let client = self
-            .register_or_adopt_account(domain, &dns_cred, &acme_url)
+            .register_or_adopt_account(domain, config, &acme_url)
             .await;
         if let Err(err) = self.release_rotation_lock(&rotation_lock) {
             error!("failed to release ACME rotation lock: {err:?}");
@@ -664,7 +831,7 @@ impl DistributedCertBot {
     async fn load_stored_acme_client(
         &self,
         domain: &str,
-        dns_cred: &DnsCredential,
+        config: &ZtDomainConfig,
         acme_url: &str,
     ) -> Result<Option<AcmeClient>> {
         let Some(creds) = self
@@ -682,10 +849,14 @@ impl DistributedCertBot {
                  call RotateAcmeCredentials to switch directories"
             );
         }
-        let dns01_client = self.dns_client(domain, dns_cred).await?;
-        let client = AcmeClient::load(dns01_client, &creds.acme_credentials, dns_cred.max_dns_wait)
-            .await
-            .context("failed to load ACME client from KvStore credentials")?;
+        let validation = self.validation_for(domain, config).await?;
+        let client = AcmeClient::load(
+            validation.method,
+            &creds.acme_credentials,
+            validation.max_dns_wait,
+        )
+        .await
+        .context("failed to load ACME client from KvStore credentials")?;
         Ok(Some(client))
     }
 
@@ -698,11 +869,11 @@ impl DistributedCertBot {
     async fn register_or_adopt_account(
         &self,
         domain: &str,
-        dns_cred: &DnsCredential,
+        config: &ZtDomainConfig,
         acme_url: &str,
     ) -> Result<AcmeClient> {
         if let Some(client) = self
-            .load_stored_acme_client(domain, dns_cred, acme_url)
+            .load_stored_acme_client(domain, config, acme_url)
             .await?
         {
             info!("adopted the ACME account registered while this node waited for the lock");
@@ -710,8 +881,8 @@ impl DistributedCertBot {
         }
 
         info!("creating new global ACME account at {acme_url}");
-        let dns01_client = self.dns_client(domain, dns_cred).await?;
-        let client = AcmeClient::new_account(acme_url, dns01_client, dns_cred.max_dns_wait)
+        let validation = self.validation_for(domain, config).await?;
+        let client = AcmeClient::new_account(acme_url, validation.method, validation.max_dns_wait)
             .await
             .context("failed to create new ACME account")?;
 
@@ -951,6 +1122,7 @@ mod tests {
             port: 443,
             node: None,
             priority: 0,
+            challenge: ChallengeKind::Dns01,
         }
     }
 
@@ -1152,7 +1324,7 @@ mod tests {
         let err = match certbot
             .register_or_adopt_account(
                 "app.example.com",
-                &unreachable_dns_credential(),
+                &test_zt_domain_config(),
                 DEFAULT_ACME_URL,
             )
             .await
diff --git a/dstack/gateway/src/kv/mod.rs b/dstack/gateway/src/kv/mod.rs
index 62bdc3870..251927ab5 100644
--- a/dstack/gateway/src/kv/mod.rs
+++ b/dstack/gateway/src/kv/mod.rs
@@ -57,6 +57,7 @@ use std::{
 };
 
 use anyhow::{Context, Result};
+use certbot::ChallengeKind;
 
 use crate::models::InstanceInfo;
 use crate::time::{encode_ts, now_secs};
@@ -378,6 +379,12 @@ pub struct ZtDomainConfig {
     /// The domain with highest priority is returned as the default base_domain in APIs
     #[serde(default)]
     pub priority: i32,
+    /// ACME challenge used to prove control of this domain.
+    ///
+    /// Records written before dns-persist-01 support have no such field and
+    /// decode as `dns-01`, which is the only method those deployments had.
+    #[serde(default)]
+    pub challenge: ChallengeKind,
 }
 
 /// Global certbot configuration (stored in KV, synced across nodes)
@@ -394,6 +401,13 @@ pub struct GlobalCertbotConfig {
     pub renew_timeout: Duration,
     /// ACME server URL (None means use default Let's Encrypt production)
     pub acme_url: String,
+    /// Issuer Domain Name naming the CA in dns-persist-01 and CAA records.
+    ///
+    /// Empty means Let's Encrypt. Only read by domains using `dns-persist-01`:
+    /// the record has to name a CA the challenge lists in `issuer-domain-names`,
+    /// so a private or staging ACME server needs its own value here.
+    #[serde(default)]
+    pub issuer_domain_name: String,
 }
 
 impl Default for GlobalCertbotConfig {
@@ -403,6 +417,7 @@ impl Default for GlobalCertbotConfig {
             renew_before_expiration: Duration::from_secs(30 * 86400), // 30 days
             renew_timeout: Duration::from_secs(300),        // 5 minutes
             acme_url: Default::default(),                   // default Let's Encrypt
+            issuer_domain_name: Default::default(),         // default Let's Encrypt
         }
     }
 }
@@ -2524,6 +2539,43 @@ mod value_encoding_tests {
         }
     }
 
+    /// A ZT domain stored before `dns-persist-01` existed has no `challenge`
+    /// field, and must keep decoding as the method it was actually using.
+    #[test]
+    fn a_zt_domain_without_a_challenge_field_decodes_as_dns01() {
+        /// The shape `ZtDomainConfig` had before the field was added.
+        #[derive(Serialize)]
+        struct LegacyZtDomainConfig {
+            domain: String,
+            dns_cred_id: Option<String>,
+            port: u16,
+            node: Option<u32>,
+            priority: i32,
+        }
+
+        let legacy = LegacyZtDomainConfig {
+            domain: "app.example.com".to_string(),
+            dns_cred_id: Some("cred-1".to_string()),
+            port: 443,
+            node: None,
+            priority: 7,
+        };
+
+        for (label, encoded) in [
+            ("named", encode(&legacy).expect("named encode")),
+            (
+                "legacy positional",
+                rmp_serde::encode::to_vec(&legacy).expect("positional encode"),
+            ),
+        ] {
+            let decoded: ZtDomainConfig =
+                decode(&encoded).unwrap_or_else(|err| panic!("{label} decode failed: {err}"));
+            assert_eq!(decoded.domain, "app.example.com", "{label}");
+            assert_eq!(decoded.priority, 7, "{label}");
+            assert_eq!(decoded.challenge, ChallengeKind::Dns01, "{label}");
+        }
+    }
+
     /// The configured window has to reach the store, not just the config file.
     ///
     /// An fsync per write runs under the store lock, so it bounds how fast this
@@ -3413,6 +3465,7 @@ mod corruption_tests {
             port: 443,
             node: None,
             priority: 0,
+            challenge: Default::default(),
         })
         .expect("save should succeed");
         // Same record filed under another domain's key: honouring the value
@@ -3427,6 +3480,7 @@ mod corruption_tests {
                     port: 443,
                     node: None,
                     priority: 100,
+                    challenge: Default::default(),
                 },
                 true,
             )
diff --git a/dstack/gateway/src/main_service.rs b/dstack/gateway/src/main_service.rs
index 427405c59..678117b91 100644
--- a/dstack/gateway/src/main_service.rs
+++ b/dstack/gateway/src/main_service.rs
@@ -619,7 +619,9 @@ impl Proxy {
         }
     }
 
-    pub(crate) async fn rotate_acme_credentials(&self) -> Result<(String, usize)> {
+    pub(crate) async fn rotate_acme_credentials(
+        &self,
+    ) -> Result<crate::distributed_certbot::RotationOutcome> {
         self.certbot.rotate_acme_credentials().await
     }
 

From c883548ed812c074a70043f82c8030bb332a4073 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 02:50:46 -0700
Subject: [PATCH 04/37] docs: document dns-persist-01 certificate issuance

---
 CHANGELOG.md                   |   1 +
 docs/certbot-dns-persist-01.md | 166 +++++++++++++++++++++++++++++++++
 2 files changed, 167 insertions(+)
 create mode 100644 docs/certbot-dns-persist-01.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4636d26da..042d32100 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - guest-agent: v1 `Attest` accepts `include_boottime_gpu_evidence` and returns the boot-time GPU attestation evidence in `AttestResponse.boottime_gpu_evidence`, so a verifier fetches the attestation and the GPU evidence in one round trip. It arrives as a `GpuEvidenceBundle` list -- the same shape `AttestGpu` returns, so a consumer writes one bundle parser and dispatches on `format`: `nvidia-nvattest-boottime-json-v1` is the record written at boot, `nvidia-nvattest-collect-evidence-json-v1` is collected on demand against a caller's nonce, and a verifier for one does not appraise the other. Absence is the empty list. The bundle's `evidence` is the nvattest output byte for byte as read from disk, because the only thing binding it to the boot is sha256 over precisely those bytes against the measured `gpu-attestation` event. `Attest` is also v1's sole CVM attestation entry point: the `VersionedAttestation` it returns already carries the TDX quote and event log, and unlike `GetQuote` it answers on every supported platform
 - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs
 - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796)
+- certbot/gateway: opt-in `dns-persist-01` certificate validation ([draft-ietf-acme-dns-persist-01](https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01)), which issues without a DNS provider credential at all. `dns-01` needs write access to the zone on every order, so a gateway CVM holds a Cloudflare API token for the life of the deployment -- a token that rewrites the whole zone, not just `_acme-challenge`. Under `dns-persist-01` the zone owner publishes one `_validation-persist.<name>` TXT record naming the CA and the ACME account; nothing about it changes between orders, so certbot only ever reads DNS and the zone can be hosted anywhere, with no provider integration. Set `challenge = "dns-persist-01"` in `certbot.toml`, or `challenge` on a gateway ZT domain; the default stays `dns-01` and existing deployments are untouched. `certbot dns-records` prints the records to publish, `GetZtDomain`/`ListZtDomains` return them in `required_dns_records`, and the gateway logs them wherever it would otherwise have written DNS. CAA records carry `validationmethods=dns-persist-01` to match, so switching methods means republishing both. Two gateway operations change shape for such a domain: `SetCaa` skips it, having nothing to reconcile without write access, and `RotateAcmeCredentials` leaves it broken until the operator republishes -- the record still names the old account -- so the response now returns the new records in `required_dns_records`. **Experimental**: the draft is still changing, and Let's Encrypt serves the challenge on staging only pending an open working-group issue. Documented in `docs/certbot-dns-persist-01.md`
 - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer
 - gateway: `Admin.SetInstanceReady` takes a CVM instance out of its app's load-balancing rotation without stopping it; instance-id routing stays open so the instance can still be investigated, and the setting survives re-registration
 - gateway: operator-set per-instance overrides now live under their own KV keys — `admin/<instance_id>/ready` and `admin/<instance_id>/port_policy` — instead of inside the instance record, so a CVM re-registration can no longer drop them and setting one cannot discard a peer's unsynced change to the other. An override left in an instance record by an earlier build is moved across on load
diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
new file mode 100644
index 000000000..371ac1b2b
--- /dev/null
+++ b/docs/certbot-dns-persist-01.md
@@ -0,0 +1,166 @@
+# Certificate issuance without a DNS credential (`dns-persist-01`)
+
+`dns-01` asks certbot to write a fresh `_acme-challenge` TXT record for every
+order, so whatever runs certbot holds a DNS API token with write access to the
+zone, forever. In dstack that token lives inside the gateway CVM. Attestation
+covers what the CVM is running, but a token is a token: anything that gets hold
+of it can rewrite the zone, including records that have nothing to do with
+certificates.
+
+`dns-persist-01` moves the proof out of the issuance loop. The zone owner
+publishes one record naming the CA and the ACME account allowed to issue:
+
+```dns
+_validation-persist.example.com. IN TXT "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1234567890"
+```
+
+The account key proves who is asking, the record proves the zone owner agreed,
+and neither changes between orders. certbot reads DNS and never writes it, so
+the CVM holds no DNS credential and the zone can be hosted anywhere — no
+Cloudflare account, no API token, no provider integration.
+
+> **Experimental.** `dns-persist-01` is specified in
+> [draft-ietf-acme-dns-persist-01][draft], which is still changing: an open
+> working-group issue may add a client-key-derived value to the record, and
+> Let's Encrypt has said it will not deploy the challenge to production until
+> that is resolved. It is live on Let's Encrypt **staging** and in
+> [Pebble][pebble]. Treat the record format as unstable, and expect to
+> republish when the draft settles.
+
+[draft]: https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01
+[pebble]: https://github.com/letsencrypt/pebble
+
+## What the record means
+
+| Part | Effect |
+| --- | --- |
+| `letsencrypt.org` | Issuer Domain Name. A CA ignores records naming a different issuer, so one label can hold records for several CAs. |
+| `accounturi=` | The ACME account authorized to issue. Compared byte for byte — no case folding, no URI normalization. |
+| `policy=wildcard` | Extends the record to `*.example.com`. Without it the CA authorizes `example.com` alone and refuses wildcard orders. |
+| `persistUntil=` | Optional UNIX timestamp after which the CA stops accepting the record. |
+
+A wildcard order authorizes from its base name — `*.example.com` is validated
+against `_validation-persist.example.com`, not
+`_validation-persist.*.example.com` — so one record covers a name and its
+wildcard.
+
+Two things about the syntax bite in practice, because a CA rejects the whole
+record rather than ignoring the offending part: **no trailing semicolon**, and
+**no whitespace inside a value**. certbot renders records that satisfy both;
+copy them verbatim rather than retyping.
+
+The scope stops at the names above. Let's Encrypt does not walk up the tree, so
+a record on `example.com` does not authorize `sub.example.com` — give each base
+name its own record.
+
+## Standalone certbot
+
+`certbot` never writes DNS in this mode, so setup is: create the account, read
+the records off it, publish them, then issue.
+
+```toml
+# certbot.toml
+workdir = "/var/lib/certbot"
+acme_url = "https://acme-staging-v02.api.letsencrypt.org/directory"
+challenge = "dns-persist-01"
+issuer_domain_name = "letsencrypt.org"
+# auto_set_caa promises certbot keeps CAA in sync, which it cannot do without
+# write access. Leave it off and publish the CAA records below by hand.
+auto_set_caa = false
+domains = ["example.com", "*.example.com"]
+renew_interval = 3600
+renew_days_before = 10
+renew_timeout = 120
+max_dns_wait = 300
+```
+
+`cf_api_token` is unused and can be left out; certbot warns if one is set.
+
+```console
+$ certbot init -c certbot.toml
+INFO certbot::bot: creating new ACME account
+INFO certbot::bot: created new ACME account: https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890
+
+$ certbot dns-records -c certbot.toml
+_validation-persist.example.com. IN TXT "letsencrypt.org; accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890; policy=wildcard"
+example.com. IN CAA 0 issue "letsencrypt.org;validationmethods=dns-persist-01;accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890"
+example.com. IN CAA 0 issuewild "letsencrypt.org;validationmethods=dns-persist-01;accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890"
+```
+
+Publish all three, wait for them to propagate, then issue:
+
+```console
+$ certbot renew --once -c certbot.toml
+INFO certbot::acme_client: requesting new certificates for example.com, *.example.com
+INFO certbot::bot: created new certificate
+```
+
+Renewals need nothing further. The record stays, and `certbot renew` reuses it
+for every order.
+
+The CAA records are optional but recommended — they stop any other account, at
+Let's Encrypt or elsewhere, from being issued for your name. Note the
+`validationmethods=dns-persist-01` in them: a CAA record left pinned to
+`dns-01` refuses every `dns-persist-01` order, so switching methods means
+updating CAA and the validation record together.
+
+### When issuance fails
+
+certbot checks its own resolver before starting an order, and says exactly what
+it expected to find:
+
+```
+WARN certbot::acme_client: no TXT record at _validation-persist.example.com matches the expected value: letsencrypt.org; accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890; policy=wildcard
+Error: order is invalid: API error: No valid TXT record found for DNS-PERSIST-01 challenge (urn:ietf:params:acme:error:unauthorized)
+```
+
+The check is advisory and never blocks an order: certbot's resolver is not the
+CA's, and its expectation can be stricter than what the CA would accept. A
+warning with a successful issuance underneath it is a resolver difference, not
+a problem.
+
+If the CA rejects the order, compare the published record against
+`certbot dns-records` character by character. The usual causes are a stale
+`accounturi` after the account was recreated, a missing `policy=wildcard` on a
+wildcard order, and an issuer domain name that does not match the CA.
+
+## dstack-gateway
+
+A ZT domain picks its method with the `challenge` field, which defaults to
+`dns-01` — existing domains are unaffected:
+
+```json
+{ "domain": "app.example.com", "port": 443, "challenge": "dns-persist-01" }
+```
+
+Such a domain needs no `dns_cred_id`, and the gateway CVM never receives a DNS
+credential for it. `GetZtDomain` and `ListZtDomains` return the records to
+publish in `required_dns_records`, and the gateway logs them whenever it cannot
+write DNS itself:
+
+```
+WARN cert[app.example.com]: publish this record by hand: _validation-persist.app.example.com. IN TXT "letsencrypt.org; accounturi=...; policy=wildcard"
+```
+
+The gateway issues for `*.{domain}` only, so each domain needs one validation
+record with `policy=wildcard`.
+
+For a non-production ACME server, set `issuer_domain_name` in the global
+certbot config to whatever that server puts in `issuer-domain-names` —
+`pebble.letsencrypt.org` for Pebble. Empty means `letsencrypt.org`.
+
+Two operations behave differently on these domains:
+
+- **`SetCaa`** skips them. There is nothing to reconcile without write access;
+  the records are logged instead, and the summary reports how many were left to
+  the operator.
+- **`RotateAcmeCredentials`** is not self-service. Rotation moves the cluster
+  to a new ACME account, and every `_validation-persist` record still names the
+  old one, so orders for those domains fail until the operator republishes. The
+  response returns the new records in `required_dns_records` and the gateway
+  logs them; publish before the next renewal comes due.
+
+## Related
+
+- [dstack-gateway](dstack-gateway.md) — gateway architecture and TLS termination
+- [deployment.md](deployment.md#4-zero-trust-https-optional) — the `dns-01` setup this replaces

From 23f2fe318e856857cb7af7a0f028a9e5b426657c Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:20:54 -0700
Subject: [PATCH 05/37] docs: quote the real dns-persist-01 rejection from
 staging

---
 docs/certbot-dns-persist-01.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index 371ac1b2b..c6102b8a7 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -111,13 +111,14 @@ it expected to find:
 
 ```
 WARN certbot::acme_client: no TXT record at _validation-persist.example.com matches the expected value: letsencrypt.org; accounturi=https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890; policy=wildcard
-Error: order is invalid: API error: No valid TXT record found for DNS-PERSIST-01 challenge (urn:ietf:params:acme:error:unauthorized)
+Error: order is invalid: API error: Checking DNS-PERSIST-01 challenge TXT record with issuer-domain-name "letsencrypt.org": accounturi mismatch: expected "https://acme-staging-v02.api.letsencrypt.org/acme/acct/1234567890", got "https://acme-staging-v02.api.letsencrypt.org/acme/acct/9876543210" (urn:ietf:params:acme:error:unauthorized)
 ```
 
 The check is advisory and never blocks an order: certbot's resolver is not the
 CA's, and its expectation can be stricter than what the CA would accept. A
 warning with a successful issuance underneath it is a resolver difference, not
-a problem.
+a problem. A record that genuinely does not match costs the full `max_dns_wait`
+before the order is sent, because the check waits out its budget first.
 
 If the CA rejects the order, compare the published record against
 `certbot dns-records` character by character. The usual causes are a stale

From 87486307907fd9be24f1bbdf9415e07f59c02747 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:35:22 -0700
Subject: [PATCH 06/37] fix(certbot): read a dns-persist-01 record the way the
 CA reads it

---
 dstack/certbot/src/dns_persist.rs | 104 +++++++++++++++++++++++++++---
 1 file changed, 95 insertions(+), 9 deletions(-)

diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs
index 4712a4901..987548220 100644
--- a/dstack/certbot/src/dns_persist.rs
+++ b/dstack/certbot/src/dns_persist.rs
@@ -84,7 +84,9 @@ impl AuthorizationRecord {
         let Ok(parsed) = IssueValue::parse(rdata) else {
             return false;
         };
-        if parsed.issuer_domain_name != self.issuer_domain_name {
+        if normalized_issuer(&parsed.issuer_domain_name)
+            != normalized_issuer(&self.issuer_domain_name)
+        {
             return false;
         }
         if parsed.account_uri != self.account_uri {
@@ -145,17 +147,40 @@ impl IssueValue {
                 bail!("empty parameter or trailing semicolon");
             }
             let (tag, value) = part.split_once('=').context("parameter is not tag=value")?;
-            // RFC 8659 matches tags case-insensitively; values are not folded.
-            let tag = tag.to_lowercase();
+            // RFC 8659's grammar is `parameter = tag *WSP "=" *WSP value`, so the
+            // separator may be padded on either side and each half is trimmed on
+            // its own. Trimming only the whole parameter would read the tag of
+            // `accounturi = <uri>` as `"accounturi "` and report the mandatory
+            // parameter missing on a record the CA accepts.
+            let tag = trim_wsp(tag).to_lowercase();
+            let value = trim_wsp(value);
+            if tag.is_empty() {
+                bail!("parameter has an empty tag");
+            }
             if !value.bytes().all(is_value_byte) {
                 bail!("parameter {tag} has a value with a forbidden character");
             }
-            if seen.contains(&tag) {
-                bail!("duplicate parameter {tag}");
+            match tag.as_str() {
+                "accounturi" | "policy" | "persistuntil" => {
+                    // Only recognized tags are held to uniqueness: the draft has
+                    // the CA ignore unknown tags outright, so a record repeating
+                    // one is still a record the CA issues from.
+                    if seen.contains(&tag) {
+                        bail!("duplicate parameter {tag}");
+                    }
+                    seen.push(tag.clone());
+                }
+                // The draft requires unrecognized tags to be ignored, so that
+                // later revisions can add parameters without invalidating records.
+                _ => continue,
             }
-            seen.push(tag.clone());
             match tag.as_str() {
-                "accounturi" => account_uri = Some(value.to_string()),
+                "accounturi" => {
+                    if value.is_empty() {
+                        bail!("empty value for the mandatory accounturi parameter");
+                    }
+                    account_uri = Some(value.to_string());
+                }
                 "policy" => policy = Some(value.to_string()),
                 "persistuntil" => {
                     persist_until = Some(
@@ -164,8 +189,6 @@ impl IssueValue {
                             .context("persistUntil is not a base-10 timestamp")?,
                     )
                 }
-                // The draft requires unrecognized tags to be ignored, so that
-                // later revisions can add parameters without invalidating records.
                 _ => {}
             }
         }
@@ -179,6 +202,19 @@ impl IssueValue {
     }
 }
 
+/// An Issuer Domain Name folded the way the CA folds it before comparing.
+///
+/// Boulder normalizes both sides (lowercase, then drop the root dot) before
+/// deciding whether a record is one of its own, so `LetsEncrypt.ORG.` names the
+/// same CA as `letsencrypt.org`. Comparing raw bytes here would treat a record
+/// the CA honours as belonging to someone else and warn about a missing record
+/// through every issuance. IDNA folding is left out: this compares against a
+/// name dstack configures, and an operator writing a non-ASCII issuer name has
+/// a mismatch a self-check cannot paper over.
+fn normalized_issuer(name: &str) -> String {
+    name.trim_end_matches('.').to_lowercase()
+}
+
 fn is_wildcard_policy(policy: &str) -> bool {
     policy.eq_ignore_ascii_case(POLICY_WILDCARD)
 }
@@ -217,6 +253,56 @@ mod tests {
         record(wildcard).satisfied_by(published, NOW)
     }
 
+    /// RFC 8659 spells the parameter grammar `tag *WSP "=" *WSP value`, and
+    /// Boulder trims each half separately. A record padded around the separator
+    /// is one the CA issues from, so the self-check has to read it the same way.
+    #[test]
+    fn whitespace_around_the_separator_is_not_part_of_the_tag_or_value() {
+        assert!(accepts(
+            false,
+            &format!("letsencrypt.org; accounturi = {ACCOUNT}")
+        ));
+        assert!(accepts(
+            true,
+            &format!("letsencrypt.org;\taccounturi\t=\t{ACCOUNT}; policy\t=\twildcard")
+        ));
+    }
+
+    /// The draft has the CA ignore unrecognized tags outright, so repeating one
+    /// cannot invalidate a record. Uniqueness is only enforced where the CA
+    /// enforces it.
+    #[test]
+    fn a_repeated_unknown_tag_is_ignored_rather_than_fatal() {
+        assert!(accepts(
+            false,
+            &format!("letsencrypt.org; accounturi={ACCOUNT}; futuretag=a; futuretag=b")
+        ));
+        assert!(!accepts(
+            false,
+            &format!("letsencrypt.org; accounturi={ACCOUNT}; accounturi={ACCOUNT}")
+        ));
+    }
+
+    /// Boulder folds case and drops the root dot before deciding whether a
+    /// record names it, so the same record must not read as another CA's here.
+    #[test]
+    fn the_issuer_name_is_compared_the_way_the_ca_compares_it() {
+        assert!(accepts(
+            false,
+            &format!("LetsEncrypt.ORG.; accounturi={ACCOUNT}")
+        ));
+        assert!(!accepts(
+            false,
+            &format!("other-ca.example; accounturi={ACCOUNT}")
+        ));
+    }
+
+    /// `accounturi` is mandatory, and an empty value does not supply it.
+    #[test]
+    fn an_empty_account_uri_is_not_an_account_uri() {
+        assert!(!accepts(false, "letsencrypt.org; accounturi="));
+    }
+
     #[test]
     fn validation_domain_prepends_the_label() {
         assert_eq!(

From 108f500b2c848af5bc25318c3349b838e0bba743 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:35:26 -0700
Subject: [PATCH 07/37] fix(certbot): name the configured CA in dns-01 CAA
 records too

---
 dstack/certbot/src/acme_client.rs         | 15 ++++++++++++---
 dstack/certbot/src/bot.rs                 |  1 +
 dstack/gateway/src/distributed_certbot.rs |  4 +++-
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index 3cc73ab3d..17bed930b 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -62,6 +62,8 @@ pub enum ValidationMethod {
         client: Dns01Client,
         /// TTL of the published records, in seconds (1 = auto, min 60 on Cloudflare).
         txt_ttl: u32,
+        /// Issuer Domain Name to name in the CAA records certbot publishes.
+        issuer_domain_name: String,
     },
     /// draft-ietf-acme-dns-persist-01 `dns-persist-01`: control is proven by a
     /// `_validation-persist` TXT record naming the CA and this ACME account,
@@ -98,11 +100,18 @@ impl ValidationMethod {
     }
 
     /// Issuer Domain Name to write into CAA records.
+    ///
+    /// Both methods read it from configuration, whose default is
+    /// `letsencrypt.org` -- the name existing deployments already have published
+    /// -- so an untouched configuration writes what it wrote before. A CAA
+    /// record naming a CA other than the one at `acme_url` forbids the very
+    /// issuance it is published to enable, and that is not a dns-01/
+    /// dns-persist-01 distinction.
     fn issuer_domain_name(&self) -> &str {
         match self {
-            // Unconfigurable for dns-01, as it has always been: existing
-            // deployments have CAA records published under this name.
-            Self::Dns01 { .. } => dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+            Self::Dns01 {
+                issuer_domain_name, ..
+            } => issuer_domain_name,
             Self::DnsPersist01 { issuer_domain_name } => issuer_domain_name,
         }
     }
diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs
index 30718d02d..886cf926e 100644
--- a/dstack/certbot/src/bot.rs
+++ b/dstack/certbot/src/bot.rs
@@ -281,6 +281,7 @@ async fn build_validation_method(config: &CertBotConfig) -> Result<ValidationMet
             Ok(ValidationMethod::Dns01 {
                 client,
                 txt_ttl: config.dns_txt_ttl,
+                issuer_domain_name: config.issuer_domain_name.clone(),
             })
         }
         ChallengeKind::DnsPersist01 => {
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index af1d0d88a..7c252c4d5 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -177,6 +177,7 @@ impl DistributedCertBot {
                     method: ValidationMethod::Dns01 {
                         client,
                         txt_ttl: dns_cred.dns_txt_ttl,
+                        issuer_domain_name: self.issuer_domain_name()?,
                     },
                     max_dns_wait: dns_cred.max_dns_wait,
                 })
@@ -190,7 +191,8 @@ impl DistributedCertBot {
         }
     }
 
-    /// Issuer Domain Name to name in `dns-persist-01` and CAA records.
+    /// Issuer Domain Name to name in `dns-persist-01` records and in the CAA
+    /// records written for either challenge.
     fn issuer_domain_name(&self) -> Result<String> {
         let configured = self.config()?.issuer_domain_name;
         Ok(match configured.is_empty() {

From e49b1ba36bb79100ed1559c7c1ee120d0faae413 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:35:26 -0700
Subject: [PATCH 08/37] fix(gateway): fit the dns-persist-01 dns wait inside
 the order timeout

---
 dstack/gateway/src/distributed_certbot.rs | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 7c252c4d5..5cca88690 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -40,9 +40,16 @@ const DEFAULT_ACME_URL: &str = "https://acme-v02.api.letsencrypt.org/directory";
 /// A `dns-01` domain reads this from its DNS credential, and a `dns-persist-01`
 /// domain has none. It stays a constant rather than a config knob because the
 /// wait is advisory: certbot polls its own resolver and starts the order either
-/// way, so the value cannot decide whether issuance succeeds. Matches the
-/// default `max_dns_wait` of a DNS credential.
-const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(300);
+/// way, so the value cannot decide whether issuance succeeds.
+///
+/// It has to stay comfortably under `renew_timeout` (300s by default), which
+/// wraps the whole order, for that to be true. The wait is measured from after
+/// the order and its authorizations are fetched, so a value equal to the outer
+/// timeout means the outer one always fires first: the order is aborted with
+/// "certificate request timed out" instead of proceeding to the CA, and the
+/// warning naming the record it could not see -- the first thing an operator
+/// is told to look at -- is never logged.
+const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(120);
 
 /// What an ACME credential rotation left behind.
 #[derive(Debug)]

From da71eb224294b4785cf1c6c47b7edda8c86d1bcd Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:35:37 -0700
Subject: [PATCH 09/37] feat(gateway): make the issuer domain name settable

---
 docs/certbot-dns-persist-01.md             | 13 ++++++++---
 dstack/gateway/rpc/proto/gateway_rpc.proto |  7 ++++++
 dstack/gateway/src/admin_service.rs        | 25 ++++++++++++++++++++++
 dstack/gateway/templates/dashboard.html    | 10 +++++++++
 4 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index c6102b8a7..7ca6ab3a1 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -146,9 +146,16 @@ WARN cert[app.example.com]: publish this record by hand: _validation-persist.app
 The gateway issues for `*.{domain}` only, so each domain needs one validation
 record with `policy=wildcard`.
 
-For a non-production ACME server, set `issuer_domain_name` in the global
-certbot config to whatever that server puts in `issuer-domain-names` —
-`pebble.letsencrypt.org` for Pebble. Empty means `letsencrypt.org`.
+A ZT domain's challenge is chosen when it is added and carried forward by every
+edit, in the dashboard's ZT-Domain form as well as over the API.
+
+For a non-production ACME server, set `issuer_domain_name` in the global certbot
+config — `Admin.SetCertbotConfig`, or the field of that name in the dashboard's
+Certbot Configuration — to whatever that server puts in `issuer-domain-names`,
+`pebble.letsencrypt.org` for Pebble. Empty means `letsencrypt.org`. It also
+names the CA in the CAA records certbot writes for `dns-01` domains, so one
+setting covers both challenges rather than pinning CAA to Let's Encrypt while
+orders go elsewhere.
 
 Two operations behave differently on these domains:
 
diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto
index 7700e808b..eed45978d 100644
--- a/dstack/gateway/rpc/proto/gateway_rpc.proto
+++ b/dstack/gateway/rpc/proto/gateway_rpc.proto
@@ -860,6 +860,9 @@ message CertbotConfigResponse {
   uint64 renew_timeout_secs = 3;
   // ACME server URL (empty means default Let's Encrypt production)
   string acme_url = 4;
+  // Issuer Domain Name naming the CA in dns-persist-01 and CAA records
+  // (empty means letsencrypt.org)
+  string issuer_domain_name = 5;
 }
 
 // Set certbot configuration request
@@ -872,6 +875,10 @@ message SetCertbotConfigRequest {
   optional uint64 renew_timeout_secs = 3;
   // ACME server URL (empty means use default Let's Encrypt production)
   optional string acme_url = 4;
+  // Issuer Domain Name naming the CA in dns-persist-01 and CAA records
+  // (empty means letsencrypt.org). Must be one of the issuer-domain-names the
+  // ACME server at acme_url advertises, or every dns-persist-01 order fails.
+  optional string issuer_domain_name = 5;
 }
 
 // ==================== Tombstone GC Configuration Messages ====================
diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs
index 87227fbfd..2cb1cb36d 100644
--- a/dstack/gateway/src/admin_service.rs
+++ b/dstack/gateway/src/admin_service.rs
@@ -717,6 +717,7 @@ impl AdminRpc for AdminRpcHandler {
             renew_before_expiration_secs: config.renew_before_expiration.as_secs(),
             renew_timeout_secs: config.renew_timeout.as_secs(),
             acme_url: config.acme_url,
+            issuer_domain_name: config.issuer_domain_name,
         })
     }
 
@@ -1094,6 +1095,9 @@ fn merge_certbot_config(
     if let Some(url) = request.acme_url {
         config.acme_url = url;
     }
+    if let Some(name) = request.issuer_domain_name {
+        config.issuer_domain_name = name;
+    }
     Ok(config)
 }
 
@@ -1148,11 +1152,32 @@ mod certbot_config_tests {
                 renew_before_expiration_secs: Some(86400),
                 renew_timeout_secs: Some(30),
                 acme_url: Some("https://acme-staging.example/directory".to_string()),
+                issuer_domain_name: None,
             },
         )
         .expect("a complete request replaces the record");
         assert_eq!(merged.renew_interval, Duration::from_secs(60));
         assert_eq!(merged.acme_url, "https://acme-staging.example/directory");
+        // Not required to repair the record: empty is a meaningful value that
+        // means Let's Encrypt, so it needs no operator decision.
+        assert_eq!(merged.issuer_domain_name, "");
+    }
+
+    /// The name the CA is known by has to be settable, or a deployment pointed
+    /// at a non-Let's-Encrypt ACME server publishes records naming the wrong CA
+    /// with no way to correct them.
+    #[test]
+    fn the_issuer_domain_name_round_trips() {
+        let merged = merge_certbot_config(
+            Ok(stored()),
+            SetCertbotConfigRequest {
+                issuer_domain_name: Some("pebble.letsencrypt.org".to_string()),
+                ..Default::default()
+            },
+        )
+        .expect("a partial update keeps the rest");
+        assert_eq!(merged.issuer_domain_name, "pebble.letsencrypt.org");
+        assert_eq!(merged.acme_url, stored().acme_url);
     }
 }
 
diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html
index 7d62d69e0..bc40c17f3 100644
--- a/dstack/gateway/templates/dashboard.html
+++ b/dstack/gateway/templates/dashboard.html
@@ -549,6 +549,13 @@ <h2>Certbot Configuration</h2>
             </td>
             <td>ACME server URL (empty = Let's Encrypt production)</td>
         </tr>
+        <tr>
+            <td>Issuer Domain Name</td>
+            <td>
+                <input type="text" id="certbot-issuer-domain-name" placeholder="letsencrypt.org (default)">
+            </td>
+            <td>Name of the CA in dns-persist-01 and CAA records; must be one the ACME server advertises</td>
+        </tr>
         <tr>
             <td>Renewal Interval</td>
             <td>
@@ -952,6 +959,7 @@ <h3>Add ZT-Domain</h3>
                 });
                 const data = await response.json();
                 document.getElementById('certbot-acme-url').value = data.acme_url || '';
+                document.getElementById('certbot-issuer-domain-name').value = data.issuer_domain_name || '';
                 document.getElementById('certbot-renew-interval').value = data.renew_interval_secs || 43200;
                 document.getElementById('certbot-renew-before').value = data.renew_before_expiration_secs || 2592000;
                 document.getElementById('certbot-renew-timeout').value = data.renew_timeout_secs || 300;
@@ -962,6 +970,7 @@ <h3>Add ZT-Domain</h3>
 
         async function saveCertbotConfig() {
             const acmeUrl = document.getElementById('certbot-acme-url').value.trim();
+            const issuerDomainName = document.getElementById('certbot-issuer-domain-name').value.trim();
             const renewInterval = parseInt(document.getElementById('certbot-renew-interval').value) || undefined;
             const renewBefore = parseInt(document.getElementById('certbot-renew-before').value) || undefined;
             const renewTimeout = parseInt(document.getElementById('certbot-renew-timeout').value) || undefined;
@@ -969,6 +978,7 @@ <h3>Add ZT-Domain</h3>
             try {
                 const body = {};
                 if (acmeUrl !== undefined) body.acme_url = acmeUrl;
+                if (issuerDomainName !== undefined) body.issuer_domain_name = issuerDomainName;
                 if (renewInterval !== undefined) body.renew_interval_secs = renewInterval;
                 if (renewBefore !== undefined) body.renew_before_expiration_secs = renewBefore;
                 if (renewTimeout !== undefined) body.renew_timeout_secs = renewTimeout;

From 45cd81b69b4e86b201bf5dbb5b3ad6c435ae104e Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:35:37 -0700
Subject: [PATCH 10/37] fix(gateway): keep a ZT domain's challenge across a
 dashboard edit

---
 dstack/gateway/templates/dashboard.html | 34 ++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html
index bc40c17f3..605f4455d 100644
--- a/dstack/gateway/templates/dashboard.html
+++ b/dstack/gateway/templates/dashboard.html
@@ -767,6 +767,13 @@ <h3>Edit ZT-Domain</h3>
                     <option value="">Use Default</option>
                 </select>
             </div>
+            <div class="modal-field">
+                <label>Validation Challenge</label>
+                <select id="edit-domain-challenge">
+                    <option value="dns-01">dns-01 (certbot writes DNS; needs a credential)</option>
+                    <option value="dns-persist-01">dns-persist-01 (one record published by hand; experimental)</option>
+                </select>
+            </div>
             <div class="modal-field">
                 <label>Port (0 = default)</label>
                 <input type="number" id="edit-domain-port" placeholder="443" min="0" max="65535">
@@ -800,6 +807,13 @@ <h3>Add ZT-Domain</h3>
                     <option value="">Use Default</option>
                 </select>
             </div>
+            <div class="modal-field">
+                <label>Validation Challenge</label>
+                <select id="domain-challenge">
+                    <option value="dns-01">dns-01 (certbot writes DNS; needs a credential)</option>
+                    <option value="dns-persist-01">dns-persist-01 (one record published by hand; experimental)</option>
+                </select>
+            </div>
             <div class="modal-field">
                 <label>Port</label>
                 <input type="number" id="domain-port" placeholder="443" min="0" max="65535" value="443">
@@ -1237,6 +1251,7 @@ <h3>Add ZT-Domain</h3>
                         <th>Port</th>
                         <th>Node</th>
                         <th>Priority</th>
+                        <th>Challenge</th>
                         <th>DNS Credential</th>
                         <th>Status</th>
                         <th>Expires</th>
@@ -1249,7 +1264,12 @@ <h3>Add ZT-Domain</h3>
                 const certStatus = item.cert_status || {};
                 const hasCert = certStatus.has_cert;
                 const expiresAt = certStatus.not_after ? new Date(certStatus.not_after * 1000).toLocaleDateString() : 'N/A';
-                const dnsCredName = getDnsCredName(config.dns_cred_id);
+                const challenge = config.challenge || 'dns-01';
+                // A dns-persist-01 domain holds no credential by design, so
+                // naming one here would read as a misconfiguration.
+                const dnsCredName = challenge === 'dns-persist-01'
+                    ? 'None (published by hand)'
+                    : getDnsCredName(config.dns_cred_id);
                 const portDisplay = config.port ? config.port : '-';
                 const nodeDisplay = (config.node !== undefined && config.node !== null) ? `Node ${config.node}` : 'Any';
                 const priorityDisplay = config.priority || 0;
@@ -1260,6 +1280,7 @@ <h3>Add ZT-Domain</h3>
                         <td>${portDisplay}</td>
                         <td>${nodeDisplay}</td>
                         <td>${priorityDisplay}</td>
+                        <td>${escapeHtml(challenge)}</td>
                         <td>${escapeHtml(dnsCredName)}</td>
                         <td>
                             <div class="cert-status">
@@ -1332,6 +1353,7 @@ <h3>Add ZT-Domain</h3>
             document.getElementById('edit-domain-port').value = config.port || 0;
             document.getElementById('edit-domain-node').value = (config.node !== undefined && config.node !== null) ? config.node : 0;
             document.getElementById('edit-domain-priority').value = config.priority || 0;
+            document.getElementById('edit-domain-challenge').value = config.challenge || 'dns-01';
 
             updateEditDnsCredSelect();
             document.getElementById('edit-domain-dns-cred').value = config.dns_cred_id || '';
@@ -1355,7 +1377,11 @@ <h3>Add ZT-Domain</h3>
             const body = {
                 domain: domain,
                 port: port,
-                priority: priority
+                priority: priority,
+                // UpdateZtDomain replaces the whole record and an omitted
+                // challenge decodes as dns-01, so editing anything else would
+                // silently downgrade a dns-persist-01 domain.
+                challenge: document.getElementById('edit-domain-challenge').value
             };
             // Only include dns_cred_id if not empty (empty means use default)
             if (dnsCredId) {
@@ -1399,7 +1425,9 @@ <h3>Add ZT-Domain</h3>
             const body = {
                 domain: domain,
                 port: port,
-                priority: priority
+                priority: priority,
+                // Chosen once, at creation; the edit form carries it forward.
+                challenge: document.getElementById('domain-challenge').value
             };
             // Only include dns_cred_id if not empty (empty means use default)
             if (dnsCredId) {

From 1ce656787a1e11284eb8f90224af8c8d54b5a172 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:47:01 -0700
Subject: [PATCH 11/37] docs: spell out the gateway dns-persist-01 setup order

---
 docs/certbot-dns-persist-01.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index 7ca6ab3a1..723ccf696 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -146,6 +146,32 @@ WARN cert[app.example.com]: publish this record by hand: _validation-persist.app
 The gateway issues for `*.{domain}` only, so each domain needs one validation
 record with `policy=wildcard`.
 
+### Setup order
+
+The records name the ACME account, so there is nothing to publish until the
+cluster has one. A gateway registers its account lazily, on the first issuance
+it attempts, which means `required_dns_records` is empty on a fresh cluster and
+the sequence is:
+
+1. `SetCertbotConfig` — `acme_url`, and `issuer_domain_name` if the CA is not
+   Let's Encrypt.
+2. `AddZtDomain` with `challenge: "dns-persist-01"`. No `dns_cred_id`: the
+   domain needs none, and the gateway CVM never receives one for it.
+3. Register the account. `RotateAcmeCredentials` does it directly and reports
+   the records in `required_dns_records` (it needs at least one ZT domain, which
+   step 2 provided). Triggering `RenewZtDomainCert` works too: the first attempt
+   registers the account and then fails validation, since the record it names
+   does not exist yet.
+4. Read the records — `GetZtDomain`, `ListZtDomains`, or the gateway log, which
+   prints them whenever it cannot write DNS itself — and publish the
+   `_validation-persist` TXT record, plus the CAA records if you want issuance
+   pinned to this account.
+5. `RenewZtDomainCert` again. Renewals from here need nothing further.
+
+Adding a second dns-persist-01 domain later starts at step 2 and skips step 3:
+the account already exists, so its records are available as soon as the domain
+is.
+
 A ZT domain's challenge is chosen when it is added and carried forward by every
 edit, in the dashboard's ZT-Domain form as well as over the API.
 

From a2747b0017aec4353dbd06cf2b16f469fc177220 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 07:53:23 -0700
Subject: [PATCH 12/37] feat(gateway): register an ACME account before any ZT
 domain exists

---
 docs/certbot-dns-persist-01.md            | 36 ++++--------
 dstack/certbot/src/acme_client.rs         | 48 ++++++++++++----
 dstack/certbot/src/lib.rs                 |  2 +-
 dstack/gateway/src/distributed_certbot.rs | 67 ++++++++++++-----------
 4 files changed, 82 insertions(+), 71 deletions(-)

diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index 723ccf696..f265346ca 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -148,40 +148,24 @@ record with `policy=wildcard`.
 
 ### Setup order
 
-The records name the ACME account, so there is nothing to publish until the
-cluster has one. A gateway registers its account lazily, on the first issuance
-it attempts, which means `required_dns_records` is empty on a fresh cluster and
-the sequence is:
+The records name the ACME account, so the account comes first:
 
 1. `SetCertbotConfig` — `acme_url`, and `issuer_domain_name` if the CA is not
    Let's Encrypt.
-2. `AddZtDomain` with `challenge: "dns-persist-01"`. No `dns_cred_id`: the
+2. `RotateAcmeCredentials` — registers the account and returns its `account_uri`.
+   It needs no ZT domain: registration proves nothing about a domain. Every call
+   registers a *new* account, so call it once here; running it again later is a
+   rotation, with the consequences described below.
+3. `AddZtDomain` with `challenge: "dns-persist-01"`. No `dns_cred_id`: the
    domain needs none, and the gateway CVM never receives one for it.
-3. Register the account. `RotateAcmeCredentials` does it directly and reports
-   the records in `required_dns_records` (it needs at least one ZT domain, which
-   step 2 provided). Triggering `RenewZtDomainCert` works too: the first attempt
-   registers the account and then fails validation, since the record it names
-   does not exist yet.
 4. Read the records — `GetZtDomain`, `ListZtDomains`, or the gateway log, which
    prints them whenever it cannot write DNS itself — and publish the
    `_validation-persist` TXT record, plus the CAA records if you want issuance
    pinned to this account.
-5. `RenewZtDomainCert` again. Renewals from here need nothing further.
-
-Adding a second dns-persist-01 domain later starts at step 2 and skips step 3:
-the account already exists, so its records are available as soon as the domain
-is.
-
-A ZT domain's challenge is chosen when it is added and carried forward by every
-edit, in the dashboard's ZT-Domain form as well as over the API.
-
-For a non-production ACME server, set `issuer_domain_name` in the global certbot
-config — `Admin.SetCertbotConfig`, or the field of that name in the dashboard's
-Certbot Configuration — to whatever that server puts in `issuer-domain-names`,
-`pebble.letsencrypt.org` for Pebble. Empty means `letsencrypt.org`. It also
-names the CA in the CAA records certbot writes for `dns-01` domains, so one
-setting covers both challenges rather than pinning CAA to Let's Encrypt while
-orders go elsewhere.
+5. `RenewZtDomainCert`. Renewals from here need nothing further.
+
+Adding another dns-persist-01 domain later starts at step 3: the account already
+exists, so the new domain's records are available as soon as it is.
 
 Two operations behave differently on these domains:
 
diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index 17bed930b..c40e20547 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -209,6 +209,19 @@ pub fn required_dns_records(
     records
 }
 
+/// An ACME account, as registered, before any validation method is chosen.
+///
+/// Returned by [`AcmeClient::register_account`] so a caller can store the
+/// credentials and report the URI -- the two things a `dns-persist-01` record
+/// and an account-pinned CAA record are written from -- without holding a
+/// client it has no use for yet.
+pub struct AcmeAccount {
+    /// Encoded credentials, in the form [`AcmeClient::load`] takes.
+    pub credentials: String,
+    /// URI the CA identifies this account by.
+    pub account_uri: String,
+}
+
 /// A AcmeClient instance.
 pub struct AcmeClient {
     account: Account,
@@ -326,12 +339,15 @@ impl AcmeClient {
         })
     }
 
-    /// Create a new account.
-    pub async fn new_account(
-        acme_url: &str,
-        validation: ValidationMethod,
-        max_dns_wait: Duration,
-    ) -> Result<Self> {
+    /// Register a new ACME account, with no validation method attached.
+    ///
+    /// Registration is a POST to the directory's `newAccount`. It asserts
+    /// nothing about any domain and touches no DNS, so it needs neither a
+    /// provider credential nor a challenge choice -- and a deployment can
+    /// therefore have an account before it has anything to validate. That order
+    /// is forced under `dns-persist-01`: the record an operator publishes names
+    /// the account, so the account has to exist first.
+    pub async fn register_account(acme_url: &str) -> Result<AcmeAccount> {
         let http_client = Box::new(ReqwestHttpClient::new()?);
         let (account, credentials) = Account::builder_with_http(http_client)
             .create(
@@ -350,14 +366,24 @@ impl AcmeClient {
             account_id: account.id().to_string(),
             credentials,
         };
-        Ok(Self {
-            account,
-            validation,
-            credentials,
-            max_dns_wait,
+        Ok(AcmeAccount {
+            account_uri: credentials.account_id.clone(),
+            credentials: serde_json::to_string(&credentials)?,
         })
     }
 
+    /// Create a new account and a client bound to `validation`.
+    pub async fn new_account(
+        acme_url: &str,
+        validation: ValidationMethod,
+        max_dns_wait: Duration,
+    ) -> Result<Self> {
+        let account = Self::register_account(acme_url).await?;
+        Self::load(validation, &account.credentials, max_dns_wait)
+            .await
+            .context("failed to load the account just registered")
+    }
+
     /// Dump the account credentials to a JSON string.
     pub fn dump_credentials(&self) -> Result<String> {
         Ok(serde_json::to_string(&self.credentials)?)
diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs
index 7d70f1f07..543edef2e 100644
--- a/dstack/certbot/src/lib.rs
+++ b/dstack/certbot/src/lib.rs
@@ -18,7 +18,7 @@
 //! to the documentation of individual structs and functions.
 
 pub use acme_client::{
-    required_dns_records, AcmeClient, ChallengeKind, RequiredRecord, ValidationMethod,
+    required_dns_records, AcmeAccount, AcmeClient, ChallengeKind, RequiredRecord, ValidationMethod,
 };
 pub use bot::{read_pubkey, CertBot, CertBotConfig};
 pub use dns01_client::Dns01Client;
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 5cca88690..bba08d4ec 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -12,7 +12,8 @@ use std::time::Duration;
 
 use anyhow::{bail, Context, Result};
 use certbot::{
-    AcmeClient, ChallengeKind, Dns01Client, ValidationMethod, LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+    AcmeAccount, AcmeClient, ChallengeKind, Dns01Client, ValidationMethod,
+    LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
 };
 use dstack_guest_agent_rpc::v0::RawQuoteArgs;
 use ra_tls::attestation::QuoteContentType;
@@ -255,9 +256,14 @@ impl DistributedCertBot {
 
     /// Rotate the shared ACME account without interrupting certificate serving.
     ///
-    /// The sequence is: validate every domain's DNS credential, create the
+    /// The sequence is: validate every domain's DNS credential, register the
     /// replacement account, publish the new credentials, then re-pin every
-    /// domain's CAA record to the new account. Publishing before re-pinning
+    /// domain's CAA record to the new account.
+    ///
+    /// A cluster with no ZT domain registers an account and stops there, which
+    /// is how a `dns-persist-01` deployment bootstraps: the record an operator
+    /// publishes names the account, so there is nothing to publish -- and
+    /// therefore nothing that can be issued -- until one exists. Publishing before re-pinning
     /// makes the failure mode convergent: if some domains fail to re-pin, the
     /// cluster is already on the new account and rerunning `SetCaa` finishes
     /// the switch without registering yet another account (Let's Encrypt caps
@@ -307,23 +313,18 @@ impl DistributedCertBot {
             prepared.push((config, validation));
         }
         let total = prepared.len();
-        let mut prepared = prepared.into_iter();
-        let Some((first_config, first_validation)) = prepared.next() else {
-            bail!("no ZT-Domain configured for ACME credential rotation");
-        };
-        let first_writes_dns = first_validation.writes_dns();
 
-        let client = AcmeClient::new_account(
-            acme_url,
-            first_validation.method,
-            first_validation.max_dns_wait,
-        )
-        .await
-        .context("failed to create replacement ACME account")?;
-        let credentials = client
-            .dump_credentials()
-            .context("failed to encode replacement ACME credentials")?;
-        let account_uri = client.account_id().to_string();
+        // Registration needs no domain: it proves nothing about one. A cluster
+        // with no ZT domain yet registers here and gets an account URI, which is
+        // what a `dns-persist-01` record has to name before the domain it
+        // authorizes can ever be issued.
+        let account = AcmeClient::register_account(acme_url)
+            .await
+            .context("failed to create replacement ACME account")?;
+        let AcmeAccount {
+            credentials,
+            account_uri,
+        } = account;
 
         // Publish immediately. From here the cluster converges on the new
         // account, and recovering from a partial re-pin below never needs to
@@ -351,17 +352,6 @@ impl DistributedCertBot {
                 failed.push(domain.to_string());
             }
         };
-        if first_writes_dns {
-            repin(
-                &first_config.domain,
-                client
-                    .set_caa_records(std::slice::from_ref(&first_config.domain))
-                    .await
-                    .context("failed to update CAA records"),
-            );
-        } else {
-            manual_domains += 1;
-        }
         for (config, validation) in prepared {
             if !validation.writes_dns() {
                 manual_domains += 1;
@@ -1385,16 +1375,27 @@ mod tests {
         );
     }
 
+    /// Registering without a domain is the `dns-persist-01` bootstrap: the
+    /// record names the account, so the account comes first. The run here gets
+    /// as far as the ACME server -- an unroutable one, so the test stays
+    /// offline -- which is proof enough that no domain was demanded before it.
     #[tokio::test]
-    async fn rotate_acme_credentials_requires_a_configured_domain() {
+    async fn rotate_acme_credentials_registers_without_any_domain() {
         let data_dir = tempfile::tempdir().expect("failed to create temp dir");
         let certbot = test_certbot(data_dir.path());
+        certbot
+            .kv_store
+            .set_certbot_config(&crate::kv::GlobalCertbotConfig {
+                acme_url: "http://127.0.0.1:1/directory".to_string(),
+                ..Default::default()
+            })
+            .expect("failed to store certbot config");
         let err = certbot
             .rotate_acme_credentials()
             .await
-            .expect_err("rotation without domains should fail");
+            .expect_err("an unroutable ACME server cannot register an account");
         assert!(
-            err.to_string().contains("no ZT-Domain configured"),
+            err.to_string().contains("replacement ACME account"),
             "unexpected error: {err}"
         );
         // The failed rotation must release the KV lock so a later run can proceed.

From 3fb2a48bfb6151a6be0c2ffcf2dfe477e74d9620 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 08:32:53 -0700
Subject: [PATCH 13/37] fix(certbot): check the configured issuer domain name
 before it reaches a zone

---
 dstack/certbot/src/bot.rs                 | 17 ++++-
 dstack/certbot/src/dns_persist.rs         | 76 +++++++++++++++++++++++
 dstack/certbot/src/lib.rs                 |  2 +-
 dstack/gateway/src/admin_service.rs       | 59 +++++++++++++++---
 dstack/gateway/src/distributed_certbot.rs | 11 ++--
 dstack/gateway/src/kv/mod.rs              |  8 ++-
 6 files changed, 151 insertions(+), 22 deletions(-)

diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs
index 886cf926e..2ed424cf5 100644
--- a/dstack/certbot/src/bot.rs
+++ b/dstack/certbot/src/bot.rs
@@ -15,7 +15,7 @@ use tokio::time::sleep;
 use tracing::{error, info, warn};
 
 use crate::acme_client::{acme_matches, read_pem, ChallengeKind, RequiredRecord, ValidationMethod};
-use crate::dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME;
+use crate::dns_persist::{resolve_issuer_domain_name, LETS_ENCRYPT_ISSUER_DOMAIN_NAME};
 
 use super::{AcmeClient, Dns01Client};
 
@@ -256,6 +256,17 @@ impl CertBot {
     }
 }
 
+/// The Issuer Domain Name this configuration names, checked before it is used.
+///
+/// Both challenges read the same setting, so both get the same treatment: empty
+/// means the default, and a value that would not survive being written into a
+/// CAA or validation record is refused here rather than at the point it would
+/// corrupt a zone.
+fn issuer_domain_name(config: &CertBotConfig) -> Result<String> {
+    resolve_issuer_domain_name(&config.issuer_domain_name)
+        .context("invalid issuer_domain_name in the certbot configuration")
+}
+
 /// Resolve the configured challenge into a live validation method.
 ///
 /// `dns-01` resolves the Cloudflare zone here, which is an authenticated call,
@@ -281,7 +292,7 @@ async fn build_validation_method(config: &CertBotConfig) -> Result<ValidationMet
             Ok(ValidationMethod::Dns01 {
                 client,
                 txt_ttl: config.dns_txt_ttl,
-                issuer_domain_name: config.issuer_domain_name.clone(),
+                issuer_domain_name: issuer_domain_name(config)?,
             })
         }
         ChallengeKind::DnsPersist01 => {
@@ -299,7 +310,7 @@ async fn build_validation_method(config: &CertBotConfig) -> Result<ValidationMet
                 warn!("ignoring cf_api_token: dns-persist-01 needs no DNS provider credential");
             }
             Ok(ValidationMethod::DnsPersist01 {
-                issuer_domain_name: config.issuer_domain_name.clone(),
+                issuer_domain_name: issuer_domain_name(config)?,
             })
         }
     }
diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs
index 987548220..60f1b8ee5 100644
--- a/dstack/certbot/src/dns_persist.rs
+++ b/dstack/certbot/src/dns_persist.rs
@@ -33,6 +33,44 @@ const VALIDATION_LABEL: &str = "_validation-persist";
 /// `issuer-domain-names`; a record naming anything else is ignored by that CA.
 pub const LETS_ENCRYPT_ISSUER_DOMAIN_NAME: &str = "letsencrypt.org";
 
+/// The Issuer Domain Name to use, given what an operator configured.
+///
+/// Empty means the default, which is what an untouched configuration carries.
+/// Anything else is checked before it is believed: the name is interpolated
+/// straight into an RFC 8659 `issue-value`, both in a `_validation-persist`
+/// record and in the CAA records certbot publishes, and `set_caa_records`
+/// installs the new value *after* deleting the records it replaces. A value
+/// carrying a `;`, a space, or a `#` does not merely fail to authorize -- it
+/// leaves the zone holding a malformed `issue` property with no valid record
+/// behind it, which is CAA that forbids every issuer. Rejecting it at the point
+/// it is configured keeps that out of the zone entirely.
+pub fn resolve_issuer_domain_name(configured: &str) -> Result<String> {
+    let name = configured.trim();
+    if name.is_empty() {
+        return Ok(LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string());
+    }
+    let name = name.trim_end_matches('.');
+    if name.is_empty() || name.len() > 253 {
+        bail!("issuer domain name must be between 1 and 253 characters: {configured:?}");
+    }
+    for label in name.split('.') {
+        if label.is_empty() {
+            bail!("issuer domain name has an empty label: {configured:?}");
+        }
+        if !label
+            .bytes()
+            .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
+        {
+            bail!(
+                "issuer domain name must be a DNS name -- it is written verbatim into CAA \
+                 and validation records, where a space or a `;` makes the whole record \
+                 malformed: {configured:?}"
+            );
+        }
+    }
+    Ok(name.to_string())
+}
+
 /// The `policy` value that widens a record to cover wildcards.
 const POLICY_WILDCARD: &str = "wildcard";
 
@@ -303,6 +341,44 @@ mod tests {
         assert!(!accepts(false, "letsencrypt.org; accounturi="));
     }
 
+    /// Empty is what an untouched configuration carries, and it means the
+    /// default rather than a name of zero length -- which as an `issue-value`
+    /// would be CAA denying every issuer.
+    #[test]
+    fn an_unset_issuer_name_is_the_default_one() {
+        assert_eq!(
+            resolve_issuer_domain_name("").unwrap(),
+            LETS_ENCRYPT_ISSUER_DOMAIN_NAME
+        );
+        assert_eq!(
+            resolve_issuer_domain_name("   ").unwrap(),
+            LETS_ENCRYPT_ISSUER_DOMAIN_NAME
+        );
+        assert_eq!(
+            resolve_issuer_domain_name("pebble.letsencrypt.org.").unwrap(),
+            "pebble.letsencrypt.org"
+        );
+    }
+
+    /// The name is interpolated into records that a `;` or a space makes
+    /// malformed, and CAA is rewritten by deleting the old records first, so a
+    /// typo accepted here is a zone that forbids all issuance.
+    #[test]
+    fn a_name_that_would_corrupt_a_record_is_refused() {
+        for bad in [
+            "lets encrypt.org",
+            "letsencrypt.org;policy=wildcard",
+            "#letsencrypt.org",
+            "letsencrypt..org",
+            "\"letsencrypt.org\"",
+        ] {
+            assert!(
+                resolve_issuer_domain_name(bad).is_err(),
+                "{bad:?} should be refused"
+            );
+        }
+    }
+
     #[test]
     fn validation_domain_prepends_the_label() {
         assert_eq!(
diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs
index 543edef2e..22a1562e8 100644
--- a/dstack/certbot/src/lib.rs
+++ b/dstack/certbot/src/lib.rs
@@ -22,7 +22,7 @@ pub use acme_client::{
 };
 pub use bot::{read_pubkey, CertBot, CertBotConfig};
 pub use dns01_client::Dns01Client;
-pub use dns_persist::LETS_ENCRYPT_ISSUER_DOMAIN_NAME;
+pub use dns_persist::{resolve_issuer_domain_name, LETS_ENCRYPT_ISSUER_DOMAIN_NAME};
 pub use workdir::WorkDir;
 
 mod acme_client;
diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs
index 2cb1cb36d..f46a9fff2 100644
--- a/dstack/gateway/src/admin_service.rs
+++ b/dstack/gateway/src/admin_service.rs
@@ -726,11 +726,12 @@ impl AdminRpc for AdminRpcHandler {
         let config = merge_certbot_config(kv_store.get_certbot_config(), request)?;
         kv_store.set_certbot_config(&config)?;
         info!(
-            "Updated certbot config: renew_interval={:?}, renew_before_expiration={:?}, renew_timeout={:?}, acme_url={:?}",
+            "Updated certbot config: renew_interval={:?}, renew_before_expiration={:?}, renew_timeout={:?}, acme_url={:?}, issuer_domain_name={:?}",
             config.renew_interval,
             config.renew_before_expiration,
             config.renew_timeout,
-            config.acme_url
+            config.acme_url,
+            config.issuer_domain_name
         );
         Ok(())
     }
@@ -1072,10 +1073,12 @@ fn merge_certbot_config(
                 request.renew_interval_secs.is_some()
                     && request.renew_before_expiration_secs.is_some()
                     && request.renew_timeout_secs.is_some()
-                    && request.acme_url.is_some(),
+                    && request.acme_url.is_some()
+                    && request.issuer_domain_name.is_some(),
                 "the stored certbot config is unreadable ({err:#}), so it can only be \
                  replaced as a whole: resend with renew_interval_secs, \
-                 renew_before_expiration_secs, renew_timeout_secs and acme_url all set"
+                 renew_before_expiration_secs, renew_timeout_secs, acme_url and \
+                 issuer_domain_name all set"
             );
             warn!("certbot config is unreadable ({err:#}); replacing it wholesale");
             GlobalCertbotConfig::default()
@@ -1096,6 +1099,12 @@ fn merge_certbot_config(
         config.acme_url = url;
     }
     if let Some(name) = request.issuer_domain_name {
+        // Checked here rather than where it is used: it is written verbatim into
+        // CAA records, and those are rewritten by deleting the old ones first,
+        // so a value that cannot be a DNS name takes the zone's issuance
+        // permission down with it. Stored as configured -- empty keeps meaning
+        // the default -- but only once it is known to be usable.
+        certbot::resolve_issuer_domain_name(&name).context("invalid issuer_domain_name")?;
         config.issuer_domain_name = name;
     }
     Ok(config)
@@ -1152,15 +1161,49 @@ mod certbot_config_tests {
                 renew_before_expiration_secs: Some(86400),
                 renew_timeout_secs: Some(30),
                 acme_url: Some("https://acme-staging.example/directory".to_string()),
-                issuer_domain_name: None,
+                issuer_domain_name: Some("pebble.letsencrypt.org".to_string()),
             },
         )
         .expect("a complete request replaces the record");
         assert_eq!(merged.renew_interval, Duration::from_secs(60));
         assert_eq!(merged.acme_url, "https://acme-staging.example/directory");
-        // Not required to repair the record: empty is a meaningful value that
-        // means Let's Encrypt, so it needs no operator decision.
-        assert_eq!(merged.issuer_domain_name, "");
+        assert_eq!(merged.issuer_domain_name, "pebble.letsencrypt.org");
+    }
+
+    /// Repairing from the defaults would reset the issuer name to empty --
+    /// Let's Encrypt -- for a deployment pointed at another CA, which then
+    /// publishes CAA and validation records naming a CA its orders never reach.
+    /// That is the same hazard `acme_url` is in the required set for.
+    #[test]
+    fn repairing_an_unreadable_record_cannot_forget_the_issuer_name() {
+        let err = merge_certbot_config(
+            Err(anyhow::anyhow!("corrupt record")),
+            SetCertbotConfigRequest {
+                renew_interval_secs: Some(60),
+                renew_before_expiration_secs: Some(86400),
+                renew_timeout_secs: Some(30),
+                acme_url: Some("https://pebble.example/dir".to_string()),
+                issuer_domain_name: None,
+            },
+        )
+        .expect_err("a repair that omits the issuer name must be refused");
+        assert!(err.to_string().contains("issuer_domain_name"), "{err:#}");
+    }
+
+    /// The name is written verbatim into CAA records, and writing them deletes
+    /// the records they replace, so a value that cannot be a DNS name is
+    /// refused where it is set rather than where it would corrupt a zone.
+    #[test]
+    fn a_malformed_issuer_name_is_refused() {
+        let err = merge_certbot_config(
+            Ok(stored()),
+            SetCertbotConfigRequest {
+                issuer_domain_name: Some("lets encrypt.org".to_string()),
+                ..Default::default()
+            },
+        )
+        .expect_err("a name with a space must be refused");
+        assert!(err.to_string().contains("issuer_domain_name"), "{err:#}");
     }
 
     /// The name the CA is known by has to be settable, or a deployment pointed
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index bba08d4ec..6b11bec05 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -12,8 +12,8 @@ use std::time::Duration;
 
 use anyhow::{bail, Context, Result};
 use certbot::{
-    AcmeAccount, AcmeClient, ChallengeKind, Dns01Client, ValidationMethod,
-    LETS_ENCRYPT_ISSUER_DOMAIN_NAME,
+    resolve_issuer_domain_name, AcmeAccount, AcmeClient, ChallengeKind, Dns01Client,
+    ValidationMethod,
 };
 use dstack_guest_agent_rpc::v0::RawQuoteArgs;
 use ra_tls::attestation::QuoteContentType;
@@ -202,11 +202,8 @@ impl DistributedCertBot {
     /// Issuer Domain Name to name in `dns-persist-01` records and in the CAA
     /// records written for either challenge.
     fn issuer_domain_name(&self) -> Result<String> {
-        let configured = self.config()?.issuer_domain_name;
-        Ok(match configured.is_empty() {
-            true => LETS_ENCRYPT_ISSUER_DOMAIN_NAME.to_string(),
-            false => configured,
-        })
+        resolve_issuer_domain_name(&self.config()?.issuer_domain_name)
+            .context("invalid issuer_domain_name in the certbot config")
     }
 
     /// The DNS records a ZT domain's zone has to contain, as zone-file lines.
diff --git a/dstack/gateway/src/kv/mod.rs b/dstack/gateway/src/kv/mod.rs
index 251927ab5..7896d029a 100644
--- a/dstack/gateway/src/kv/mod.rs
+++ b/dstack/gateway/src/kv/mod.rs
@@ -403,9 +403,11 @@ pub struct GlobalCertbotConfig {
     pub acme_url: String,
     /// Issuer Domain Name naming the CA in dns-persist-01 and CAA records.
     ///
-    /// Empty means Let's Encrypt. Only read by domains using `dns-persist-01`:
-    /// the record has to name a CA the challenge lists in `issuer-domain-names`,
-    /// so a private or staging ACME server needs its own value here.
+    /// Empty means Let's Encrypt. A `dns-persist-01` record has to name a CA
+    /// the challenge lists in `issuer-domain-names`, and the CAA records written
+    /// for either challenge name the same CA, so a private or staging ACME
+    /// server needs its own value here -- one setting for both, since `acme_url`
+    /// is one setting for both.
     #[serde(default)]
     pub issuer_domain_name: String,
 }

From 0664f8f442fb9dada3bc4371591e432d63be84f7 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 08:32:58 -0700
Subject: [PATCH 14/37] fix(certbot): keep the advisory DNS wait inside the
 order timeout

---
 dstack/certbot/src/acme_client.rs         | 50 +++++++++++++++++++
 dstack/certbot/src/bot.rs                 | 16 +++++-
 dstack/certbot/src/lib.rs                 |  3 +-
 dstack/gateway/src/admin_service.rs       | 18 +++++++
 dstack/gateway/src/distributed_certbot.rs | 59 ++++++++++++++++-------
 5 files changed, 126 insertions(+), 20 deletions(-)

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index c40e20547..2b7066aae 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -222,6 +222,32 @@ pub struct AcmeAccount {
     pub account_uri: String,
 }
 
+/// The share of `renew_timeout` the pre-order DNS check may spend.
+///
+/// The rest pays for the order itself: `new_order`, the authorizations, the
+/// provider writes under `dns-01`, then finalize and the certificate fetch.
+const DNS_WAIT_SHARE_OF_RENEW_TIMEOUT: u32 = 2;
+
+/// The DNS wait to actually use, given what is configured and how long the whole
+/// order is allowed to take.
+///
+/// The wait is advisory by design: certbot polls its own resolver, logs what it
+/// could not see, and starts the order regardless, because the CA's DNS view is
+/// not this node's. That only holds if the wait ends before the timeout wrapping
+/// the order does. It does not by default -- a DNS credential's `max_dns_wait`
+/// defaults to 300s and `renew_timeout` defaults to 300s, and the wait is
+/// measured from after the order and its authorizations are fetched, so the
+/// outer timeout always fires first. The renewal then dies with "certificate
+/// request timed out", and the warning naming the record that was missing --
+/// the first thing an operator is told to look for -- is never logged.
+///
+/// Clamping here rather than picking a smaller constant covers both challenges
+/// with one rule, and keeps holding when an operator lowers `renew_timeout`
+/// from the dashboard, which no constant can.
+pub fn advisory_dns_wait(configured: Duration, renew_timeout: Duration) -> Duration {
+    configured.min(renew_timeout / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT)
+}
+
 /// A AcmeClient instance.
 pub struct AcmeClient {
     account: Account,
@@ -1602,3 +1628,27 @@ mod reissue_reason_tests {
         assert!(reason.contains("cannot read"), "{reason}");
     }
 }
+
+#[cfg(test)]
+mod dns_wait_tests {
+    use super::advisory_dns_wait;
+    use std::time::Duration;
+
+    /// The CLI's own defaults are the failing case: a 300s wait inside a 120s
+    /// renewal budget never reaches its "proceed anyway" exit, so an unanswered
+    /// check ends as a timeout rather than as the warning naming the record.
+    #[test]
+    fn the_wait_ends_before_the_renewal_budget_does() {
+        let wait = advisory_dns_wait(Duration::from_secs(300), Duration::from_secs(120));
+        assert!(wait < Duration::from_secs(120), "{wait:?}");
+    }
+
+    /// A wait already inside the budget is left where the operator put it.
+    #[test]
+    fn a_wait_that_already_fits_is_unchanged() {
+        assert_eq!(
+            advisory_dns_wait(Duration::from_secs(30), Duration::from_secs(600)),
+            Duration::from_secs(30)
+        );
+    }
+}
diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs
index 2ed424cf5..ba53122e4 100644
--- a/dstack/certbot/src/bot.rs
+++ b/dstack/certbot/src/bot.rs
@@ -18,6 +18,7 @@ use crate::acme_client::{acme_matches, read_pem, ChallengeKind, RequiredRecord,
 use crate::dns_persist::{resolve_issuer_domain_name, LETS_ENCRYPT_ISSUER_DOMAIN_NAME};
 
 use super::{AcmeClient, Dns01Client};
+use crate::acme_client::advisory_dns_wait;
 
 #[allow(clippy::duplicated_attributes)]
 #[derive(Clone, Debug, bon::Builder)]
@@ -70,7 +71,7 @@ async fn create_new_account(
     validation: ValidationMethod,
 ) -> Result<AcmeClient> {
     info!("creating new ACME account");
-    let client = AcmeClient::new_account(&config.acme_url, validation, config.max_dns_wait)
+    let client = AcmeClient::new_account(&config.acme_url, validation, dns_wait(config))
         .await
         .context("failed to create new account")?;
     let credentials = client
@@ -96,7 +97,7 @@ impl CertBot {
         let acme_client = match fs::read_to_string(&config.credentials_file) {
             Ok(credentials) => {
                 if acme_matches(&credentials, &config.acme_url) {
-                    AcmeClient::load(validation, &credentials, config.max_dns_wait).await?
+                    AcmeClient::load(validation, &credentials, dns_wait(&config)).await?
                 } else {
                     create_new_account(&config, validation).await?
                 }
@@ -256,6 +257,17 @@ impl CertBot {
     }
 }
 
+/// The DNS wait this configuration should actually use.
+///
+/// `renew_timeout` wraps the whole renewal here exactly as it does in the
+/// gateway, and the defaults are skewed the same way -- further, in fact:
+/// `max_dns_wait` defaults to 300s against a 120s renewal budget, so an
+/// unanswered check runs the renewal into its timeout every time instead of
+/// reporting the record it could not see.
+fn dns_wait(config: &CertBotConfig) -> Duration {
+    advisory_dns_wait(config.max_dns_wait, config.renew_timeout)
+}
+
 /// The Issuer Domain Name this configuration names, checked before it is used.
 ///
 /// Both challenges read the same setting, so both get the same treatment: empty
diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs
index 22a1562e8..e2a0314c9 100644
--- a/dstack/certbot/src/lib.rs
+++ b/dstack/certbot/src/lib.rs
@@ -18,7 +18,8 @@
 //! to the documentation of individual structs and functions.
 
 pub use acme_client::{
-    required_dns_records, AcmeAccount, AcmeClient, ChallengeKind, RequiredRecord, ValidationMethod,
+    advisory_dns_wait, required_dns_records, AcmeAccount, AcmeClient, ChallengeKind,
+    RequiredRecord, ValidationMethod,
 };
 pub use bot::{read_pubkey, CertBot, CertBotConfig};
 pub use dns01_client::Dns01Client;
diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs
index f46a9fff2..2305c0cf1 100644
--- a/dstack/gateway/src/admin_service.rs
+++ b/dstack/gateway/src/admin_service.rs
@@ -1093,6 +1093,9 @@ fn merge_certbot_config(
         config.renew_before_expiration = Duration::from_secs(secs);
     }
     if let Some(secs) = request.renew_timeout_secs {
+        // Zero would abort every order before it starts, and the DNS wait is
+        // derived from this value.
+        ensure!(secs > 0, "renew_timeout_secs must be greater than zero");
         config.renew_timeout = Duration::from_secs(secs);
     }
     if let Some(url) = request.acme_url {
@@ -1206,6 +1209,21 @@ mod certbot_config_tests {
         assert!(err.to_string().contains("issuer_domain_name"), "{err:#}");
     }
 
+    /// Zero would abort every order before it began, and the DNS wait is
+    /// derived from this value.
+    #[test]
+    fn a_zero_renew_timeout_is_refused() {
+        let err = merge_certbot_config(
+            Ok(stored()),
+            SetCertbotConfigRequest {
+                renew_timeout_secs: Some(0),
+                ..Default::default()
+            },
+        )
+        .expect_err("a zero renew timeout must be refused");
+        assert!(err.to_string().contains("renew_timeout_secs"), "{err:#}");
+    }
+
     /// The name the CA is known by has to be settable, or a deployment pointed
     /// at a non-Let's-Encrypt ACME server publishes records naming the wrong CA
     /// with no way to correct them.
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 6b11bec05..5b01825f6 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -12,8 +12,8 @@ use std::time::Duration;
 
 use anyhow::{bail, Context, Result};
 use certbot::{
-    resolve_issuer_domain_name, AcmeAccount, AcmeClient, ChallengeKind, Dns01Client,
-    ValidationMethod,
+    advisory_dns_wait, resolve_issuer_domain_name, AcmeAccount, AcmeClient, ChallengeKind,
+    Dns01Client, ValidationMethod,
 };
 use dstack_guest_agent_rpc::v0::RawQuoteArgs;
 use ra_tls::attestation::QuoteContentType;
@@ -38,19 +38,10 @@ const DEFAULT_ACME_URL: &str = "https://acme-v02.api.letsencrypt.org/directory";
 
 /// How long a `dns-persist-01` domain waits for its record to become visible.
 ///
-/// A `dns-01` domain reads this from its DNS credential, and a `dns-persist-01`
-/// domain has none. It stays a constant rather than a config knob because the
-/// wait is advisory: certbot polls its own resolver and starts the order either
-/// way, so the value cannot decide whether issuance succeeds.
-///
-/// It has to stay comfortably under `renew_timeout` (300s by default), which
-/// wraps the whole order, for that to be true. The wait is measured from after
-/// the order and its authorizations are fetched, so a value equal to the outer
-/// timeout means the outer one always fires first: the order is aborted with
-/// "certificate request timed out" instead of proceeding to the CA, and the
-/// warning naming the record it could not see -- the first thing an operator
-/// is told to look at -- is never logged.
-const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(120);
+/// A `dns-01` domain reads this from its DNS credential; a `dns-persist-01`
+/// domain has none, so it starts from the same 300s a credential defaults to
+/// and is clamped by [`advisory_dns_wait`] like every other domain.
+const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(300);
 
 /// What an ACME credential rotation left behind.
 #[derive(Debug)]
@@ -177,6 +168,7 @@ impl DistributedCertBot {
         domain: &str,
         config: &ZtDomainConfig,
     ) -> Result<DomainValidation> {
+        let renew_timeout = self.config()?.renew_timeout;
         match config.challenge {
             ChallengeKind::Dns01 => {
                 let dns_cred = dns_credential_for(&self.kv_store, config)?;
@@ -187,14 +179,14 @@ impl DistributedCertBot {
                         txt_ttl: dns_cred.dns_txt_ttl,
                         issuer_domain_name: self.issuer_domain_name()?,
                     },
-                    max_dns_wait: dns_cred.max_dns_wait,
+                    max_dns_wait: advisory_dns_wait(dns_cred.max_dns_wait, renew_timeout),
                 })
             }
             ChallengeKind::DnsPersist01 => Ok(DomainValidation {
                 method: ValidationMethod::DnsPersist01 {
                     issuer_domain_name: self.issuer_domain_name()?,
                 },
-                max_dns_wait: DNS_PERSIST_MAX_DNS_WAIT,
+                max_dns_wait: advisory_dns_wait(DNS_PERSIST_MAX_DNS_WAIT, renew_timeout),
             }),
         }
     }
@@ -1337,6 +1329,39 @@ mod tests {
     /// Two rotations on this node are ordered by the same lock that orders two
     /// nodes: the lock lives in the KV store, so a second in-process run sees
     /// the first one's record.
+    /// The check has to finish inside the timeout wrapping the order, or the
+    /// "proceed anyway" exit it is designed around is never reached and the
+    /// renewal dies with a timeout instead of a diagnosis. Both defaults are
+    /// 300s, which is exactly the case that fails.
+    #[test]
+    fn the_dns_wait_ends_before_the_order_times_out() {
+        let default_credential = Duration::from_secs(300);
+        let default_renew_timeout = Duration::from_secs(300);
+        assert!(
+            advisory_dns_wait(default_credential, default_renew_timeout) < default_renew_timeout
+        );
+
+        // Lowering renew_timeout from the dashboard has to keep the invariant,
+        // which is why this is derived rather than a constant.
+        for secs in [30, 60, 120, 600] {
+            let renew_timeout = Duration::from_secs(secs);
+            assert!(
+                advisory_dns_wait(DNS_PERSIST_MAX_DNS_WAIT, renew_timeout) < renew_timeout,
+                "renew_timeout={secs}s leaves no room for the order"
+            );
+        }
+    }
+
+    /// A credential asking for less than its share keeps asking for less: the
+    /// clamp is a ceiling, not a target.
+    #[test]
+    fn a_shorter_configured_wait_is_left_alone() {
+        assert_eq!(
+            advisory_dns_wait(Duration::from_secs(20), Duration::from_secs(300)),
+            Duration::from_secs(20)
+        );
+    }
+
     #[tokio::test]
     async fn rotate_acme_credentials_rejects_concurrent_runs() {
         let data_dir = tempfile::tempdir().expect("failed to create temp dir");

From 0e59643cd19db00e3c0e2bbdb0bb38b8aa1bc983 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 08:32:58 -0700
Subject: [PATCH 15/37] fix(gateway): stop the dashboard hiding dns-persist-01
 setup

---
 dstack/gateway/templates/dashboard.html | 57 +++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html
index 605f4455d..6ced374f0 100644
--- a/dstack/gateway/templates/dashboard.html
+++ b/dstack/gateway/templates/dashboard.html
@@ -793,6 +793,23 @@ <h3>Edit ZT-Domain</h3>
         </div>
     </div>
 
+    <!-- DNS Records To Publish Modal -->
+    <div id="dns-records-modal" class="modal-overlay">
+        <div class="modal">
+            <h3>Publish these DNS records</h3>
+            <p style="color: #666;">
+                This domain validates with <code>dns-persist-01</code>, so the gateway holds no
+                DNS credential for it and cannot write these itself. Certificates are only
+                issued once they are published.
+            </p>
+            <pre id="dns-records-list" style="white-space: pre-wrap; word-break: break-all;"></pre>
+            <div class="modal-actions">
+                <button class="action-btn secondary" onclick="copyDnsRecords()">Copy</button>
+                <button class="action-btn primary" onclick="hideModal('dns-records-modal')">Done</button>
+            </div>
+        </div>
+    </div>
+
     <!-- Add ZT-Domain Modal -->
     <div id="add-zt-domain-modal" class="modal-overlay">
         <div class="modal">
@@ -1293,6 +1310,7 @@ <h3>Add ZT-Domain</h3>
                         <td class="timestamp" data-original="${expiresAt}">${certStatus.not_after || 0}</td>
                         <td>
                             <button class="action-btn secondary" onclick="showEditZtDomainModal('${escapeHtml(config.domain)}')">Edit</button>
+                            ${challenge === 'dns-persist-01' ? `<button class="action-btn secondary" onclick="showZtDomainRecords('${escapeHtml(config.domain)}')">Records</button>` : ''}
                             <button class="action-btn warning" onclick="renewZtDomainCert('${escapeHtml(config.domain)}', false)">Renew</button>
                             <button class="action-btn secondary" onclick="renewZtDomainCert('${escapeHtml(config.domain)}', true)">Force Renew</button>
                             <button class="action-btn danger" onclick="deleteZtDomain('${escapeHtml(config.domain)}')">Delete</button>
@@ -1328,6 +1346,10 @@ <h3>Add ZT-Domain</h3>
             document.getElementById('domain-port').value = '443';
             document.getElementById('domain-node').value = '0';
             document.getElementById('domain-priority').value = '0';
+            // Reset with the rest: a form whose every visible field was blanked
+            // reads as fresh, and a challenge left over from the last add is
+            // sent unconditionally.
+            document.getElementById('domain-challenge').value = 'dns-01';
             showModal('add-zt-domain-modal');
         }
 
@@ -1447,9 +1469,14 @@ <h3>Add ZT-Domain</h3>
                     const err = await response.text();
                     throw new Error(err);
                 }
+                const added = await response.json();
                 hideModal('add-zt-domain-modal');
                 showToast('ZT-Domain added', 'success');
                 await loadZtDomains();
+                // A dns-persist-01 domain is not finished when the API says
+                // added: nothing can be issued for it until these records exist,
+                // and this response is the only place they are reported.
+                showRequiredDnsRecords(added.required_dns_records);
             } catch (e) {
                 showToast('Failed to add ZT-Domain: ' + e.message, 'error');
             }
@@ -1507,6 +1534,36 @@ <h3>Add ZT-Domain</h3>
 
         // ==================== UI Helpers ====================
 
+        function showZtDomainRecords(domain) {
+            const item = ztDomains.find(d => d.config && d.config.domain === domain);
+            const records = item && item.required_dns_records;
+            if (!records || records.length === 0) {
+                // Empty until the cluster has an ACME account: every record
+                // names it. RotateAcmeCredentials registers one.
+                showToast('No records yet — register an ACME account first', 'error');
+                return;
+            }
+            showRequiredDnsRecords(records);
+        }
+
+        function showRequiredDnsRecords(records) {
+            if (!records || records.length === 0) {
+                return;
+            }
+            document.getElementById('dns-records-list').textContent = records.join('\n');
+            showModal('dns-records-modal');
+        }
+
+        async function copyDnsRecords() {
+            const text = document.getElementById('dns-records-list').textContent;
+            try {
+                await navigator.clipboard.writeText(text);
+                showToast('Records copied', 'success');
+            } catch (e) {
+                showToast('Copy failed: ' + e.message, 'error');
+            }
+        }
+
         function showModal(id) {
             document.getElementById(id).classList.add('active');
         }

From db74dfa41c26bcbc810eac33258422f9aa7cd216 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 08:34:31 -0700
Subject: [PATCH 16/37] docs: restore the issuer-name guidance and record the
 new rules

---
 CHANGELOG.md                   |  4 +++-
 docs/certbot-dns-persist-01.md | 41 +++++++++++++++++++++++++++++-----
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 042d32100..5437d11af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,7 +25,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - guest-agent: v1 `Attest` accepts `include_boottime_gpu_evidence` and returns the boot-time GPU attestation evidence in `AttestResponse.boottime_gpu_evidence`, so a verifier fetches the attestation and the GPU evidence in one round trip. It arrives as a `GpuEvidenceBundle` list -- the same shape `AttestGpu` returns, so a consumer writes one bundle parser and dispatches on `format`: `nvidia-nvattest-boottime-json-v1` is the record written at boot, `nvidia-nvattest-collect-evidence-json-v1` is collected on demand against a caller's nonce, and a verifier for one does not appraise the other. Absence is the empty list. The bundle's `evidence` is the nvattest output byte for byte as read from disk, because the only thing binding it to the boot is sha256 over precisely those bytes against the measured `gpu-attestation` event. `Attest` is also v1's sole CVM attestation entry point: the `VersionedAttestation` it returns already carries the TDX quote and event log, and unlike `GetQuote` it answers on every supported platform
 - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs
 - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796)
-- certbot/gateway: opt-in `dns-persist-01` certificate validation ([draft-ietf-acme-dns-persist-01](https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01)), which issues without a DNS provider credential at all. `dns-01` needs write access to the zone on every order, so a gateway CVM holds a Cloudflare API token for the life of the deployment -- a token that rewrites the whole zone, not just `_acme-challenge`. Under `dns-persist-01` the zone owner publishes one `_validation-persist.<name>` TXT record naming the CA and the ACME account; nothing about it changes between orders, so certbot only ever reads DNS and the zone can be hosted anywhere, with no provider integration. Set `challenge = "dns-persist-01"` in `certbot.toml`, or `challenge` on a gateway ZT domain; the default stays `dns-01` and existing deployments are untouched. `certbot dns-records` prints the records to publish, `GetZtDomain`/`ListZtDomains` return them in `required_dns_records`, and the gateway logs them wherever it would otherwise have written DNS. CAA records carry `validationmethods=dns-persist-01` to match, so switching methods means republishing both. Two gateway operations change shape for such a domain: `SetCaa` skips it, having nothing to reconcile without write access, and `RotateAcmeCredentials` leaves it broken until the operator republishes -- the record still names the old account -- so the response now returns the new records in `required_dns_records`. **Experimental**: the draft is still changing, and Let's Encrypt serves the challenge on staging only pending an open working-group issue. Documented in `docs/certbot-dns-persist-01.md`
+- certbot/gateway: opt-in `dns-persist-01` certificate validation ([draft-ietf-acme-dns-persist-01](https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01)), which issues without a DNS provider credential at all. `dns-01` needs write access to the zone on every order, so a gateway CVM holds a Cloudflare API token for the life of the deployment -- a token that rewrites the whole zone, not just `_acme-challenge`. Under `dns-persist-01` the zone owner publishes one `_validation-persist.<name>` TXT record naming the CA and the ACME account; nothing about it changes between orders, so certbot only ever reads DNS and the zone can be hosted anywhere, with no provider integration. Set `challenge = "dns-persist-01"` in `certbot.toml`, or `challenge` on a gateway ZT domain; the default stays `dns-01` and existing deployments are untouched. `certbot dns-records` prints the records to publish, `GetZtDomain`/`ListZtDomains` return them in `required_dns_records`, and the gateway logs them wherever it would otherwise have written DNS. CAA records carry `validationmethods=dns-persist-01` to match, so switching methods means republishing both. Two gateway operations change shape for such a domain: `SetCaa` skips it, having nothing to reconcile without write access, and `RotateAcmeCredentials` leaves it broken until the operator republishes -- the record still names the old account -- so the response now returns the new records in `required_dns_records`. **Experimental**: the draft is still changing, and Let's Encrypt serves the challenge on staging only pending an open working-group issue. Documented in `docs/certbot-dns-persist-01.md`.
+
+  Two settings changed shape for every deployment, not only `dns-persist-01` ones. `issuer_domain_name` now names the CA in the CAA records written for `dns-01` too -- its default is `letsencrypt.org`, so an untouched configuration writes exactly what it wrote before -- and it is validated where it is set, since it lands verbatim in an RFC 8659 `issue-value` and CAA is republished by deleting the old records first. The pre-order DNS self-check is now capped at half of `renew_timeout` rather than at its own configured value: both defaulted to 300s in the gateway, and the CLI wrapped a 300s wait in a 120s budget, so the timeout around the order always fired first and a missing record was reported as `certificate request timed out` instead of by name
 - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer
 - gateway: `Admin.SetInstanceReady` takes a CVM instance out of its app's load-balancing rotation without stopping it; instance-id routing stays open so the instance can still be investigated, and the setting survives re-registration
 - gateway: operator-set per-instance overrides now live under their own KV keys — `admin/<instance_id>/ready` and `admin/<instance_id>/port_policy` — instead of inside the instance record, so a CVM re-registration can no longer drop them and setting one cannot discard a peer's unsynced change to the other. An override left in an instance record by an earlier build is moved across on load
diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index f265346ca..915aa3be6 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -158,14 +158,45 @@ The records name the ACME account, so the account comes first:
    rotation, with the consequences described below.
 3. `AddZtDomain` with `challenge: "dns-persist-01"`. No `dns_cred_id`: the
    domain needs none, and the gateway CVM never receives one for it.
-4. Read the records — `GetZtDomain`, `ListZtDomains`, or the gateway log, which
-   prints them whenever it cannot write DNS itself — and publish the
-   `_validation-persist` TXT record, plus the CAA records if you want issuance
-   pinned to this account.
+4. Read the records — the dashboard shows them when the domain is added and
+   again under its **Records** button, and `GetZtDomain`, `ListZtDomains` and
+   the gateway log all carry them — then publish the `_validation-persist` TXT
+   record, plus the CAA records if you want issuance pinned to this account.
 5. `RenewZtDomainCert`. Renewals from here need nothing further.
 
 Adding another dns-persist-01 domain later starts at step 3: the account already
-exists, so the new domain's records are available as soon as it is.
+exists, so the new domain's records are available as soon as it is. A domain's
+challenge is chosen when it is added — in the dashboard's ZT-Domain form as well
+as over the API — and carried forward by every edit.
+
+### Naming the CA
+
+For a non-production ACME server, set `issuer_domain_name` in the global certbot
+config — `Admin.SetCertbotConfig`, or the field of that name in the dashboard's
+Certbot Configuration — to whatever that server puts in `issuer-domain-names`,
+`pebble.letsencrypt.org` for Pebble. Empty means `letsencrypt.org`.
+
+It also names the CA in the CAA records certbot writes for `dns-01` domains, so
+one setting covers both challenges rather than pinning CAA to Let's Encrypt
+while orders go somewhere else. `acme_url` is one setting for the whole
+deployment; this follows it.
+
+The value is checked when it is set. It ends up verbatim inside an RFC 8659
+`issue-value`, and publishing CAA deletes the records it replaces before writing
+the new ones, so a name carrying a space or a `;` would leave the zone holding a
+malformed `issue` property with nothing valid behind it — CAA that forbids every
+issuer. Anything that is not a DNS name is refused by `SetCertbotConfig`, and by
+`certbot` at startup.
+
+### How long the check waits
+
+The self-check before each order waits for the record to become visible, but
+never longer than half of `renew_timeout`, the budget for the whole renewal.
+Past that the outer timeout fires first and the renewal ends as `certificate
+request timed out` with no mention of the record it could not find — the
+opposite of what the check is for. Both values defaulted to 300s in the gateway
+(and the CLI wrapped a 300s wait in a 120s budget), so this was the default
+behaviour rather than an edge case.
 
 Two operations behave differently on these domains:
 

From 98f93d89d49bea0a2226f21066ddd23ba598c6a2 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:29:24 -0700
Subject: [PATCH 17/37] fix(certbot): keep the account a failed client build
 would have dropped

---
 dstack/certbot/src/acme_client.rs | 60 ++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index 2b7066aae..3c4126c96 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -374,24 +374,7 @@ impl AcmeClient {
     /// is forced under `dns-persist-01`: the record an operator publishes names
     /// the account, so the account has to exist first.
     pub async fn register_account(acme_url: &str) -> Result<AcmeAccount> {
-        let http_client = Box::new(ReqwestHttpClient::new()?);
-        let (account, credentials) = Account::builder_with_http(http_client)
-            .create(
-                &NewAccount {
-                    contact: &[],
-                    terms_of_service_agreed: true,
-                    only_return_existing: false,
-                },
-                acme_url.to_string(),
-                None,
-            )
-            .await
-            .with_context(|| format!("failed to create ACME account for {acme_url}"))?;
-        let credentials = Credentials {
-            acme_url: acme_url.to_string(),
-            account_id: account.id().to_string(),
-            credentials,
-        };
+        let (_, credentials) = Self::register(acme_url).await?;
         Ok(AcmeAccount {
             account_uri: credentials.account_id.clone(),
             credentials: serde_json::to_string(&credentials)?,
@@ -404,10 +387,45 @@ impl AcmeClient {
         validation: ValidationMethod,
         max_dns_wait: Duration,
     ) -> Result<Self> {
-        let account = Self::register_account(acme_url).await?;
-        Self::load(validation, &account.credentials, max_dns_wait)
+        // Built from the account `register` already holds rather than by
+        // reloading the credentials: `load` fetches the directory, and a
+        // transient failure there would drop an account that exists at the CA
+        // and counts against its registration limit, on a path whose caller has
+        // not persisted anything yet.
+        let (account, credentials) = Self::register(acme_url).await?;
+        Ok(Self {
+            account,
+            validation,
+            credentials,
+            max_dns_wait,
+        })
+    }
+
+    /// Register an account and keep both halves: the live account and the
+    /// credentials that reconstruct it.
+    async fn register(acme_url: &str) -> Result<(Account, Credentials)> {
+        let http_client = Box::new(ReqwestHttpClient::new()?);
+        let (account, credentials) = Account::builder_with_http(http_client)
+            .create(
+                &NewAccount {
+                    contact: &[],
+                    terms_of_service_agreed: true,
+                    only_return_existing: false,
+                },
+                acme_url.to_string(),
+                None,
+            )
             .await
-            .context("failed to load the account just registered")
+            .with_context(|| format!("failed to create ACME account for {acme_url}"))?;
+        let account_id = account.id().to_string();
+        Ok((
+            account,
+            Credentials {
+                acme_url: acme_url.to_string(),
+                account_id,
+                credentials,
+            },
+        ))
     }
 
     /// Dump the account credentials to a JSON string.

From 2c0ab2a82c418183a8d17b763812eef5626d3146 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:29:24 -0700
Subject: [PATCH 18/37] fix(certbot): stop the DNS wait overshooting the budget
 it was clamped to

---
 dstack/certbot/src/acme_client.rs | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index 3c4126c96..a8d2d1613 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -245,7 +245,16 @@ const DNS_WAIT_SHARE_OF_RENEW_TIMEOUT: u32 = 2;
 /// with one rule, and keeps holding when an operator lowers `renew_timeout`
 /// from the dashboard, which no constant can.
 pub fn advisory_dns_wait(configured: Duration, renew_timeout: Duration) -> Duration {
-    configured.min(renew_timeout / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT)
+    let capped = renew_timeout / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT;
+    if configured > capped {
+        // Silently ignoring a configured value leaves an operator raising it
+        // with no idea why nothing changes.
+        info!(
+            "waiting up to {capped:?} for DNS rather than the configured {configured:?}: \
+             the renewal timeout is {renew_timeout:?}, and the check has to finish inside it"
+        );
+    }
+    configured.min(capped)
 }
 
 /// A AcmeClient instance.
@@ -854,7 +863,17 @@ impl AcmeClient {
         }
 
         'outer: loop {
-            sleep(delay).await;
+            // The budget is checked before sleeping, and the sleep is cut to
+            // what is left of it. Sleeping first and checking after overshoots
+            // by a whole backoff step -- up to 32s -- which for a short
+            // `renew_timeout` is enough to hand the deadline to the timeout
+            // wrapping the order, so the graceful exit below is missed and the
+            // renewal ends as a timeout with nothing named.
+            let elapsed = start_time.elapsed();
+            let remaining = self.max_dns_wait.saturating_sub(elapsed);
+            if !remaining.is_zero() {
+                sleep(delay.min(remaining)).await;
+            }
 
             let elapsed = start_time.elapsed();
             if elapsed >= self.max_dns_wait {

From a0ae4f95e2e174ee76f4ac60cdb4c7e81fb9d7ae Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:29:24 -0700
Subject: [PATCH 19/37] fix(certbot): hold the issuer domain name to the RFC
 8659 label grammar

---
 dstack/certbot/src/dns_persist.rs | 41 ++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs
index 60f1b8ee5..ba663bd8c 100644
--- a/dstack/certbot/src/dns_persist.rs
+++ b/dstack/certbot/src/dns_persist.rs
@@ -54,17 +54,25 @@ pub fn resolve_issuer_domain_name(configured: &str) -> Result<String> {
         bail!("issuer domain name must be between 1 and 253 characters: {configured:?}");
     }
     for label in name.split('.') {
-        if label.is_empty() {
-            bail!("issuer domain name has an empty label: {configured:?}");
-        }
-        if !label
-            .bytes()
-            .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
-        {
+        // RFC 8659 spells a label `(ALPHA / DIGIT) *( *("-") (ALPHA / DIGIT) )`:
+        // alphanumeric at both ends, hyphens only between them, and nothing
+        // else -- no underscore, which is a DNS convention for names that are
+        // never issuer names. A CA holding the record to that grammar reads
+        // anything looser as a malformed `issue` property, which is the state
+        // this check exists to keep out of a zone.
+        let bytes = label.as_bytes();
+        let well_formed = !bytes.is_empty()
+            && bytes.len() <= 63
+            && bytes[0].is_ascii_alphanumeric()
+            && bytes[bytes.len() - 1].is_ascii_alphanumeric()
+            && bytes
+                .iter()
+                .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'-');
+        if !well_formed {
             bail!(
                 "issuer domain name must be a DNS name -- it is written verbatim into CAA \
-                 and validation records, where a space or a `;` makes the whole record \
-                 malformed: {configured:?}"
+                 and validation records, where a malformed label makes the whole record \
+                 unparseable: {configured:?}"
             );
         }
     }
@@ -363,6 +371,15 @@ mod tests {
     /// The name is interpolated into records that a `;` or a space makes
     /// malformed, and CAA is rewritten by deleting the old records first, so a
     /// typo accepted here is a zone that forbids all issuance.
+    /// A label longer than DNS allows is refused, and one right at the limit is
+    /// not: the bound is the protocol's, not a guess.
+    #[test]
+    fn a_label_is_held_to_its_dns_length() {
+        let sixty_three = "a".repeat(63);
+        assert!(resolve_issuer_domain_name(&format!("{sixty_three}.org")).is_ok());
+        assert!(resolve_issuer_domain_name(&format!("{sixty_three}a.org")).is_err());
+    }
+
     #[test]
     fn a_name_that_would_corrupt_a_record_is_refused() {
         for bad in [
@@ -371,6 +388,12 @@ mod tests {
             "#letsencrypt.org",
             "letsencrypt..org",
             "\"letsencrypt.org\"",
+            // RFC 8659's label grammar bounds both ends of a label, so these
+            // are as unparseable to a strict CA as a space is.
+            "_letsencrypt.org",
+            "-letsencrypt.org",
+            "letsencrypt-.org",
+            "lé.org",
         ] {
             assert!(
                 resolve_issuer_domain_name(bad).is_err(),

From fb85f275066e59b8011acb1ef212cdc0efd688ca Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:29:24 -0700
Subject: [PATCH 20/37] fix(gateway): show the records dialog only where it
 applies

---
 dstack/gateway/templates/dashboard.html | 45 ++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html
index 6ced374f0..52c80ba2d 100644
--- a/dstack/gateway/templates/dashboard.html
+++ b/dstack/gateway/templates/dashboard.html
@@ -549,6 +549,17 @@ <h2>Certbot Configuration</h2>
             </td>
             <td>ACME server URL (empty = Let's Encrypt production)</td>
         </tr>
+        <tr>
+            <td>ACME Account</td>
+            <td>
+                <button class="action-btn warning" onclick="rotateAcmeCredentials()">Register / Rotate</button>
+            </td>
+            <td>
+                Registers the shared ACME account. Every run registers a <em>new</em> one, so
+                <code>dns-persist-01</code> validation records and account-pinned CAA have to be
+                republished afterwards.
+            </td>
+        </tr>
         <tr>
             <td>Issuer Domain Name</td>
             <td>
@@ -1473,10 +1484,12 @@ <h3>Add ZT-Domain</h3>
                 hideModal('add-zt-domain-modal');
                 showToast('ZT-Domain added', 'success');
                 await loadZtDomains();
-                // A dns-persist-01 domain is not finished when the API says
-                // added: nothing can be issued for it until these records exist,
-                // and this response is the only place they are reported.
-                showRequiredDnsRecords(added.required_dns_records);
+                // Only for dns-persist-01: `required_dns_records` is non-empty
+                // for dns-01 too -- the CAA lines the gateway writes itself --
+                // and this dialog says the operator has to publish them.
+                if (body.challenge === 'dns-persist-01') {
+                    showRequiredDnsRecords(added.required_dns_records);
+                }
             } catch (e) {
                 showToast('Failed to add ZT-Domain: ' + e.message, 'error');
             }
@@ -1534,6 +1547,30 @@ <h3>Add ZT-Domain</h3>
 
         // ==================== UI Helpers ====================
 
+        async function rotateAcmeCredentials() {
+            if (!confirm('Register a new ACME account?\n\nEvery dns-persist-01 validation record and every account-pinned CAA record names the account, so they must be republished afterwards.')) {
+                return;
+            }
+            try {
+                const response = await fetch('/prpc/Admin.RotateAcmeCredentials', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: '{}'
+                });
+                if (!response.ok) {
+                    throw new Error(await response.text());
+                }
+                const result = await response.json();
+                showToast('ACME account: ' + result.account_uri, 'success');
+                await loadZtDomains();
+                // What the operator has to publish before those domains issue
+                // again -- the response is the only place it is reported.
+                showRequiredDnsRecords(result.required_dns_records);
+            } catch (e) {
+                showToast('Failed to register ACME account: ' + e.message, 'error');
+            }
+        }
+
         function showZtDomainRecords(domain) {
             const item = ztDomains.find(d => d.config && d.config.domain === domain);
             const records = item && item.required_dns_records;

From e82a5783fe70d5c4686683da615e76960b293b8d Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:55:22 -0700
Subject: [PATCH 21/37] fix(gateway): report a partial ACME rotation instead of
 raising it

---
 CHANGELOG.md                               |  2 +-
 docs/certbot-dns-persist-01.md             |  6 ++++++
 dstack/gateway/rpc/proto/gateway_rpc.proto |  5 +++++
 dstack/gateway/src/admin_service.rs        |  1 +
 dstack/gateway/src/distributed_certbot.rs  | 12 +++++++++--
 dstack/gateway/templates/dashboard.html    | 24 +++++++++++++++++++---
 6 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5437d11af..f4d6c5756 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - guest-agent: v1 `Attest` accepts `include_boottime_gpu_evidence` and returns the boot-time GPU attestation evidence in `AttestResponse.boottime_gpu_evidence`, so a verifier fetches the attestation and the GPU evidence in one round trip. It arrives as a `GpuEvidenceBundle` list -- the same shape `AttestGpu` returns, so a consumer writes one bundle parser and dispatches on `format`: `nvidia-nvattest-boottime-json-v1` is the record written at boot, `nvidia-nvattest-collect-evidence-json-v1` is collected on demand against a caller's nonce, and a verifier for one does not appraise the other. Absence is the empty list. The bundle's `evidence` is the nvattest output byte for byte as read from disk, because the only thing binding it to the boot is sha256 over precisely those bytes against the measured `gpu-attestation` event. `Attest` is also v1's sole CVM attestation entry point: the `VersionedAttestation` it returns already carries the TDX quote and event log, and unlike `GetQuote` it answers on every supported platform
 - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs
 - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796)
-- certbot/gateway: opt-in `dns-persist-01` certificate validation ([draft-ietf-acme-dns-persist-01](https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01)), which issues without a DNS provider credential at all. `dns-01` needs write access to the zone on every order, so a gateway CVM holds a Cloudflare API token for the life of the deployment -- a token that rewrites the whole zone, not just `_acme-challenge`. Under `dns-persist-01` the zone owner publishes one `_validation-persist.<name>` TXT record naming the CA and the ACME account; nothing about it changes between orders, so certbot only ever reads DNS and the zone can be hosted anywhere, with no provider integration. Set `challenge = "dns-persist-01"` in `certbot.toml`, or `challenge` on a gateway ZT domain; the default stays `dns-01` and existing deployments are untouched. `certbot dns-records` prints the records to publish, `GetZtDomain`/`ListZtDomains` return them in `required_dns_records`, and the gateway logs them wherever it would otherwise have written DNS. CAA records carry `validationmethods=dns-persist-01` to match, so switching methods means republishing both. Two gateway operations change shape for such a domain: `SetCaa` skips it, having nothing to reconcile without write access, and `RotateAcmeCredentials` leaves it broken until the operator republishes -- the record still names the old account -- so the response now returns the new records in `required_dns_records`. **Experimental**: the draft is still changing, and Let's Encrypt serves the challenge on staging only pending an open working-group issue. Documented in `docs/certbot-dns-persist-01.md`.
+- certbot/gateway: opt-in `dns-persist-01` certificate validation ([draft-ietf-acme-dns-persist-01](https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01)), which issues without a DNS provider credential at all. `dns-01` needs write access to the zone on every order, so a gateway CVM holds a Cloudflare API token for the life of the deployment -- a token that rewrites the whole zone, not just `_acme-challenge`. Under `dns-persist-01` the zone owner publishes one `_validation-persist.<name>` TXT record naming the CA and the ACME account; nothing about it changes between orders, so certbot only ever reads DNS and the zone can be hosted anywhere, with no provider integration. Set `challenge = "dns-persist-01"` in `certbot.toml`, or `challenge` on a gateway ZT domain; the default stays `dns-01` and existing deployments are untouched. `certbot dns-records` prints the records to publish, `GetZtDomain`/`ListZtDomains` return them in `required_dns_records`, and the gateway logs them wherever it would otherwise have written DNS. CAA records carry `validationmethods=dns-persist-01` to match, so switching methods means republishing both. Two gateway operations change shape for such a domain: `SetCaa` skips it, having nothing to reconcile without write access, and `RotateAcmeCredentials` leaves it broken until the operator republishes -- the record still names the old account -- so the response now returns the new records in `required_dns_records`. A rotation that registers the account but fails to re-pin some domain's CAA now reports those domains in `repin_failed_domains` instead of failing the call: the account exists and the cluster is using it, so raising an error there reads as "nothing happened" and invites a retry that registers another account. **Experimental**: the draft is still changing, and Let's Encrypt serves the challenge on staging only pending an open working-group issue. Documented in `docs/certbot-dns-persist-01.md`.
 
   Two settings changed shape for every deployment, not only `dns-persist-01` ones. `issuer_domain_name` now names the CA in the CAA records written for `dns-01` too -- its default is `letsencrypt.org`, so an untouched configuration writes exactly what it wrote before -- and it is validated where it is set, since it lands verbatim in an RFC 8659 `issue-value` and CAA is republished by deleting the old records first. The pre-order DNS self-check is now capped at half of `renew_timeout` rather than at its own configured value: both defaulted to 300s in the gateway, and the CLI wrapped a 300s wait in a 120s budget, so the timeout around the order always fired first and a missing record was reported as `certificate request timed out` instead of by name
 - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer
diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index 915aa3be6..f34b73713 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -209,6 +209,12 @@ Two operations behave differently on these domains:
   response returns the new records in `required_dns_records` and the gateway
   logs them; publish before the next renewal comes due.
 
+  A rotation that registers the account but cannot re-pin every `dns-01`
+  domain's CAA still succeeds — the account exists and the cluster is on it —
+  and names those domains in `repin_failed_domains`. Finish with `SetCaa`
+  rather than another rotation, which would register yet another account
+  against a rate-limited quota.
+
 ## Related
 
 - [dstack-gateway](dstack-gateway.md) — gateway architecture and TLS termination
diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto
index eed45978d..ef1a29267 100644
--- a/dstack/gateway/rpc/proto/gateway_rpc.proto
+++ b/dstack/gateway/rpc/proto/gateway_rpc.proto
@@ -214,6 +214,11 @@ message RotateAcmeCredentialsResponse {
   // cannot write (dns-persist-01). Non-empty means issuance for those domains
   // stays broken until they name the new account.
   repeated string required_dns_records = 3;
+  // Domains whose CAA records could not be re-pinned to the new account. The
+  // rotation itself succeeded -- the account exists and the cluster is using
+  // it -- so this is reported rather than raised: repeating the rotation would
+  // register yet another account. Rerun SetCaa until it succeeds.
+  repeated string repin_failed_domains = 4;
 }
 
 // Get HostInfo for associated instance id.
diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs
index 2305c0cf1..77b401fba 100644
--- a/dstack/gateway/src/admin_service.rs
+++ b/dstack/gateway/src/admin_service.rs
@@ -116,6 +116,7 @@ impl AdminRpc for AdminRpcHandler {
             account_uri: outcome.account_uri,
             domains_updated: outcome.domains_updated.try_into().unwrap_or(u32::MAX),
             required_dns_records: outcome.required_dns_records,
+            repin_failed_domains: outcome.repin_failed,
         })
     }
 
diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 5b01825f6..73f1cdbfe 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -48,6 +48,8 @@ const DNS_PERSIST_MAX_DNS_WAIT: Duration = Duration::from_secs(300);
 pub struct RotationOutcome {
     /// URI of the new ACME account.
     pub account_uri: String,
+    /// Domains whose CAA could not be re-pinned to the new account.
+    pub repin_failed: Vec<String>,
     /// Domains whose CAA records were re-pinned to the new account.
     pub domains_updated: usize,
     /// Records an operator has to publish by hand, for domains the gateway
@@ -378,7 +380,12 @@ impl DistributedCertBot {
         }
 
         if !failed.is_empty() {
-            bail!(
+            // Reported, not raised. The account is registered and the cluster is
+            // already using it, so an error here reads as "nothing happened" and
+            // invites a retry -- which would register yet another account
+            // against a rate-limited quota and rewrite CAA a second time. What
+            // is left is a `SetCaa` run, and the caller is told exactly that.
+            error!(
                 "rotated to {account_uri} and published the new credentials, but failed to \
                  re-pin CAA for {}/{total} domains: {}; rerun SetCaa until it succeeds — \
                  retrying the rotation would register yet another account",
@@ -394,8 +401,9 @@ impl DistributedCertBot {
         }
         Ok(RotationOutcome {
             account_uri,
-            domains_updated: total - manual_domains,
+            domains_updated: total - manual_domains - failed.len(),
             required_dns_records: manual,
+            repin_failed: failed,
         })
     }
 
diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html
index 52c80ba2d..73e224d69 100644
--- a/dstack/gateway/templates/dashboard.html
+++ b/dstack/gateway/templates/dashboard.html
@@ -522,7 +522,7 @@ <h2>Global Connections</h2>
         </tr>
         <tr>
             <td>Account URI</td>
-            <td>{{ acme_info.account_uri }}</td>
+            <td id="acme-account-uri">{{ acme_info.account_uri }}</td>
         </tr>
         <tr>
             <td>Historical Certificate Public Keys</td>
@@ -1548,7 +1548,14 @@ <h3>Add ZT-Domain</h3>
         // ==================== UI Helpers ====================
 
         async function rotateAcmeCredentials() {
-            if (!confirm('Register a new ACME account?\n\nEvery dns-persist-01 validation record and every account-pinned CAA record names the account, so they must be republished afterwards.')) {
+            const warning = [
+                'Register a new ACME account?',
+                '',
+                'This registers a new account at the CA, which is rate-limited, and rewrites the CAA records of every dns-01 domain to name it. Rewriting installs a temporary ";" guard and deletes the old records first, so a DNS provider failure part-way through can leave a domain with CAA that blocks all issuance until a later SetCaa succeeds.',
+                '',
+                'dns-persist-01 domains keep naming the old account until you republish their records by hand.'
+            ].join('\n');
+            if (!confirm(warning)) {
                 return;
             }
             try {
@@ -1561,7 +1568,18 @@ <h3>Add ZT-Domain</h3>
                     throw new Error(await response.text());
                 }
                 const result = await response.json();
-                showToast('ACME account: ' + result.account_uri, 'success');
+                // The account URI is server-rendered, so it would otherwise keep
+                // showing the account this rotation just replaced.
+                document.getElementById('acme-account-uri').textContent = result.account_uri;
+                const failed = result.repin_failed_domains || [];
+                if (failed.length > 0) {
+                    // The rotation itself succeeded. Saying "failed" here would
+                    // invite a retry, which registers another account.
+                    showToast('Rotated to ' + result.account_uri + ', but CAA re-pinning failed for: '
+                        + failed.join(', ') + '. Run SetCaa again — do not rotate again.', 'error');
+                } else {
+                    showToast('ACME account: ' + result.account_uri, 'success');
+                }
                 await loadZtDomains();
                 // What the operator has to publish before those domains issue
                 // again -- the response is the only place it is reported.

From 1e9571f23d2ef6ea4940e48d85653479d5e99f38 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:55:22 -0700
Subject: [PATCH 22/37] fix(certbot): say which character an issuer name was
 rejected for

---
 dstack/certbot/src/dns_persist.rs | 63 +++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 20 deletions(-)

diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs
index ba663bd8c..6c7fd977f 100644
--- a/dstack/certbot/src/dns_persist.rs
+++ b/dstack/certbot/src/dns_persist.rs
@@ -60,19 +60,33 @@ pub fn resolve_issuer_domain_name(configured: &str) -> Result<String> {
         // never issuer names. A CA holding the record to that grammar reads
         // anything looser as a malformed `issue` property, which is the state
         // this check exists to keep out of a zone.
+        // Each rejection names what it rejected: the operator has to fix the
+        // value, and "malformed" alone does not say which character to remove --
+        // an underscore, say, which is legal in plenty of DNS names and is the
+        // one most likely to be reached for here.
         let bytes = label.as_bytes();
-        let well_formed = !bytes.is_empty()
-            && bytes.len() <= 63
-            && bytes[0].is_ascii_alphanumeric()
-            && bytes[bytes.len() - 1].is_ascii_alphanumeric()
-            && bytes
-                .iter()
-                .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'-');
-        if !well_formed {
+        let context = "it is written verbatim into CAA and validation records, \
+                       where a malformed label makes the whole record unparseable";
+        if bytes.is_empty() {
+            bail!("issuer domain name {configured:?} has an empty label: {context}");
+        }
+        if bytes.len() > 63 {
+            bail!("issuer domain name {configured:?} has a label over 63 octets: {context}");
+        }
+        if let Some(byte) = bytes
+            .iter()
+            .find(|byte| !byte.is_ascii_alphanumeric() && **byte != b'-')
+        {
             bail!(
-                "issuer domain name must be a DNS name -- it is written verbatim into CAA \
-                 and validation records, where a malformed label makes the whole record \
-                 unparseable: {configured:?}"
+                "issuer domain name {configured:?} contains {:?}, which a DNS label cannot: \
+                 {context}",
+                *byte as char
+            );
+        }
+        if !bytes[0].is_ascii_alphanumeric() || !bytes[bytes.len() - 1].is_ascii_alphanumeric() {
+            bail!(
+                "issuer domain name {configured:?} has a label starting or ending with a \
+                 hyphen: {context}"
             );
         }
     }
@@ -349,6 +363,24 @@ mod tests {
         assert!(!accepts(false, "letsencrypt.org; accounturi="));
     }
 
+    /// A label longer than DNS allows is refused, and one right at the limit is
+    /// not: the bound is the protocol's, not a guess.
+    #[test]
+    fn a_label_is_held_to_its_dns_length() {
+        let sixty_three = "a".repeat(63);
+        assert!(resolve_issuer_domain_name(&format!("{sixty_three}.org")).is_ok());
+        assert!(resolve_issuer_domain_name(&format!("{sixty_three}a.org")).is_err());
+    }
+
+    /// A rejection has to say what to change: the character it objected to, not
+    /// just that the value was wrong.
+    #[test]
+    fn a_rejection_names_the_character_it_rejected() {
+        let err = resolve_issuer_domain_name("my_ca.example.com")
+            .expect_err("an underscore is not a DNS label character");
+        assert!(err.to_string().contains('_'), "{err:#}");
+    }
+
     /// Empty is what an untouched configuration carries, and it means the
     /// default rather than a name of zero length -- which as an `issue-value`
     /// would be CAA denying every issuer.
@@ -371,15 +403,6 @@ mod tests {
     /// The name is interpolated into records that a `;` or a space makes
     /// malformed, and CAA is rewritten by deleting the old records first, so a
     /// typo accepted here is a zone that forbids all issuance.
-    /// A label longer than DNS allows is refused, and one right at the limit is
-    /// not: the bound is the protocol's, not a guess.
-    #[test]
-    fn a_label_is_held_to_its_dns_length() {
-        let sixty_three = "a".repeat(63);
-        assert!(resolve_issuer_domain_name(&format!("{sixty_three}.org")).is_ok());
-        assert!(resolve_issuer_domain_name(&format!("{sixty_three}a.org")).is_err());
-    }
-
     #[test]
     fn a_name_that_would_corrupt_a_record_is_refused() {
         for bad in [

From 5c337e257ae4d6997607172ea58259b9482d1fd7 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 09:55:22 -0700
Subject: [PATCH 23/37] refactor(certbot): log the DNS budget at debug, not on
 every issuance

---
 dstack/certbot/src/acme_client.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index a8d2d1613..3c02cfabd 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -247,11 +247,15 @@ const DNS_WAIT_SHARE_OF_RENEW_TIMEOUT: u32 = 2;
 pub fn advisory_dns_wait(configured: Duration, renew_timeout: Duration) -> Duration {
     let capped = renew_timeout / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT;
     if configured > capped {
-        // Silently ignoring a configured value leaves an operator raising it
-        // with no idea why nothing changes.
-        info!(
-            "waiting up to {capped:?} for DNS rather than the configured {configured:?}: \
-             the renewal timeout is {renew_timeout:?}, and the check has to finish inside it"
+        // At `debug!` deliberately: the stock defaults have both values at 300s,
+        // so this fires on every issuance in every deployment. Phrased as the
+        // budget it is rather than as an override, so an operator who does raise
+        // `max_dns_wait` and finds nothing changed can see why here, without a
+        // permanent line in everyone else's log implying they configured
+        // something that was ignored.
+        debug!(
+            "DNS check budget is {capped:?}, half of the {renew_timeout:?} renewal timeout; \
+             the configured wait of {configured:?} does not fit inside it"
         );
     }
     configured.min(capped)

From 1ddcb3bc2c8e4e00f3e0c2a771227525c4da0444 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 17:40:47 -0700
Subject: [PATCH 24/37] fix(certbot): name the character an issuer name was
 rejected for, not its first byte
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The scan walked bytes and rendered the offender with `*byte as char`, which
reinterprets one byte of a multi-byte character as a Latin-1 codepoint. So
`lé.org` was rejected for a `Ã` -- a character that appears nowhere in the
value the same message quotes back through `{configured:?}`, leaving the
message contradicting itself about the one thing it exists to report.

Scanning `chars()` costs nothing here: the length bound stays on bytes, where
DNS's 63 is octets, and by the time the first/last check indexes the label the
scan has already established it is ASCII.
---
 dstack/certbot/src/dns_persist.rs | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/dstack/certbot/src/dns_persist.rs b/dstack/certbot/src/dns_persist.rs
index 6c7fd977f..ec21b0e98 100644
--- a/dstack/certbot/src/dns_persist.rs
+++ b/dstack/certbot/src/dns_persist.rs
@@ -73,14 +73,17 @@ pub fn resolve_issuer_domain_name(configured: &str) -> Result<String> {
         if bytes.len() > 63 {
             bail!("issuer domain name {configured:?} has a label over 63 octets: {context}");
         }
-        if let Some(byte) = bytes
-            .iter()
-            .find(|byte| !byte.is_ascii_alphanumeric() && **byte != b'-')
+        // Scanned as characters, not bytes: a byte cast to `char` renders a
+        // multi-byte character's first byte as some unrelated Latin-1 one, so
+        // `lé.org` would be rejected for a `Ã` that appears nowhere in the
+        // value the same message quotes back.
+        if let Some(ch) = label
+            .chars()
+            .find(|ch| !ch.is_ascii_alphanumeric() && *ch != '-')
         {
             bail!(
-                "issuer domain name {configured:?} contains {:?}, which a DNS label cannot: \
-                 {context}",
-                *byte as char
+                "issuer domain name {configured:?} contains {ch:?}, which a DNS label cannot: \
+                 {context}"
             );
         }
         if !bytes[0].is_ascii_alphanumeric() || !bytes[bytes.len() - 1].is_ascii_alphanumeric() {
@@ -379,6 +382,13 @@ mod tests {
         let err = resolve_issuer_domain_name("my_ca.example.com")
             .expect_err("an underscore is not a DNS label character");
         assert!(err.to_string().contains('_'), "{err:#}");
+
+        // A multi-byte character has to name itself. Scanning bytes and casting
+        // to `char` would report the first byte of `é` as `Ã`, which does not
+        // occur in the value the same message quotes back.
+        let err = resolve_issuer_domain_name("lé.org").expect_err("é is not a DNS label character");
+        assert!(err.to_string().contains('é'), "{err:#}");
+        assert!(!err.to_string().contains('Ã'), "{err:#}");
     }
 
     /// Empty is what an untouched configuration carries, and it means the

From 1ed5644a584965569b092401c46cecc83f619f13 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 17:40:54 -0700
Subject: [PATCH 25/37] fix(gateway): count a rotation's re-pin failures out of
 the domains it tried

The failure count was reported over `total`, which counts every domain the
rotation covered -- including the dns-persist-01 ones that `continue` before a
re-pin is ever attempted, having no CAA the gateway can write. Five domains,
three of them dns-persist-01 and the other two failing, logged "failed to
re-pin CAA for 2/5 domains", which says three were re-pinned when none were.
The response said `domains_updated: 0` for the same rotation, so the log and
the RPC contradicted each other.

The three numbers now derive from one `RepinTally` rather than from three
separate reads of the loop's leftovers, which is what let them disagree. That
also makes the arithmetic testable without an ACME server: reaching the
partial-failure path end to end needs registration to succeed and a CAA write
to fail, but the contract worth pinning is the tally, and it is now pure.

`saturating_sub` because this runs after the account is published -- an
underflow panic here would abandon a rotation the cluster has already
committed to.
---
 dstack/gateway/src/distributed_certbot.rs | 101 ++++++++++++++++++----
 1 file changed, 86 insertions(+), 15 deletions(-)

diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs
index 73f1cdbfe..05a59656a 100644
--- a/dstack/gateway/src/distributed_certbot.rs
+++ b/dstack/gateway/src/distributed_certbot.rs
@@ -58,6 +58,37 @@ pub struct RotationOutcome {
     pub required_dns_records: Vec<String>,
 }
 
+/// What the CAA re-pin pass over a rotation's domains produced.
+///
+/// The three numbers a rotation reports are all derived from one tally, so they
+/// cannot disagree: `total` counts every domain the rotation covered, `manual`
+/// the ones a re-pin was never attempted for -- a `dns-persist-01` domain has
+/// no CAA the gateway can write -- and `failed` the ones that were tried and
+/// did not take.
+#[derive(Debug, Default)]
+struct RepinTally {
+    /// Domains skipped because the gateway cannot write their DNS.
+    manual: usize,
+    /// Domains a re-pin was attempted for and failed.
+    failed: Vec<String>,
+}
+
+impl RepinTally {
+    /// Domains a re-pin was actually attempted for.
+    ///
+    /// The denominator a failure count belongs over. Counting the manual ones
+    /// in reports them as successes: "2/5 failed" says three domains were
+    /// re-pinned when the other three were never candidates.
+    fn attempted(&self, total: usize) -> usize {
+        total.saturating_sub(self.manual)
+    }
+
+    /// Domains now pinned to the new account.
+    fn updated(&self, total: usize) -> usize {
+        self.attempted(total).saturating_sub(self.failed.len())
+    }
+}
+
 /// How one ZT domain answers ACME challenges.
 struct DomainValidation {
     method: ValidationMethod,
@@ -334,18 +365,17 @@ impl DistributedCertBot {
         // is not transparent for it either: its `_validation-persist` record
         // still names the *old* account, so orders for that domain fail until
         // the operator republishes. Those records are reported below.
-        let mut failed = Vec::new();
-        let mut manual_domains = 0usize;
-        let mut repin = |domain: &str, result: Result<()>| match result {
+        let mut tally = RepinTally::default();
+        let repin = |tally: &mut RepinTally, domain: &str, result: Result<()>| match result {
             Ok(()) => info!("cert[{domain}]: CAA re-pinned to {account_uri}"),
             Err(err) => {
                 error!("cert[{domain}]: failed to re-pin CAA: {err:?}");
-                failed.push(domain.to_string());
+                tally.failed.push(domain.to_string());
             }
         };
         for (config, validation) in prepared {
             if !validation.writes_dns() {
-                manual_domains += 1;
+                tally.manual += 1;
                 continue;
             }
             let result = async {
@@ -359,7 +389,7 @@ impl DistributedCertBot {
                     .context("failed to update CAA records")
             }
             .await;
-            repin(&config.domain, result);
+            repin(&mut tally, &config.domain, result);
         }
 
         // Rendered after the new credentials are published, so the records name
@@ -379,31 +409,37 @@ impl DistributedCertBot {
             warn!("failed to attest rotated ACME account: {err:?}");
         }
 
-        if !failed.is_empty() {
+        if !tally.failed.is_empty() {
             // Reported, not raised. The account is registered and the cluster is
             // already using it, so an error here reads as "nothing happened" and
             // invites a retry -- which would register yet another account
             // against a rate-limited quota and rewrite CAA a second time. What
             // is left is a `SetCaa` run, and the caller is told exactly that.
+            //
+            // Out of the domains a re-pin was attempted for, not out of every
+            // domain: a dns-persist-01 domain was never a candidate, and
+            // counting it in the denominator reports it as re-pinned.
             error!(
                 "rotated to {account_uri} and published the new credentials, but failed to \
-                 re-pin CAA for {}/{total} domains: {}; rerun SetCaa until it succeeds — \
+                 re-pin CAA for {}/{} domains: {}; rerun SetCaa until it succeeds — \
                  retrying the rotation would register yet another account",
-                failed.len(),
-                failed.join(", ")
+                tally.failed.len(),
+                tally.attempted(total),
+                tally.failed.join(", ")
             );
         }
-        if manual_domains > 0 {
+        if tally.manual > 0 {
             warn!(
-                "{manual_domains}/{total} domains use dns-persist-01: issuance for them stays \
-                 broken until the records above name the new account {account_uri}"
+                "{}/{total} domains use dns-persist-01: issuance for them stays \
+                 broken until the records above name the new account {account_uri}",
+                tally.manual
             );
         }
         Ok(RotationOutcome {
             account_uri,
-            domains_updated: total - manual_domains - failed.len(),
+            domains_updated: tally.updated(total),
             required_dns_records: manual,
-            repin_failed: failed,
+            repin_failed: tally.failed,
         })
     }
 
@@ -1360,6 +1396,41 @@ mod tests {
         }
     }
 
+    /// A dns-persist-01 domain is never a re-pin candidate, so counting it into
+    /// the denominator reports it as re-pinned. Five domains, three of them
+    /// manual and the other two failing, is "2/2 failed" and zero updated --
+    /// not "2/5 failed", which claims three succeeded.
+    #[test]
+    fn the_failure_count_is_out_of_the_domains_actually_tried() {
+        let tally = RepinTally {
+            manual: 3,
+            failed: vec!["a.example".to_string(), "b.example".to_string()],
+        };
+        assert_eq!(tally.attempted(5), 2);
+        assert_eq!(tally.updated(5), 0);
+    }
+
+    /// Every domain manual: nothing was attempted and nothing was updated. The
+    /// arithmetic has to land on zero rather than underflow, which as a usize
+    /// subtraction would panic the rotation after the account was published.
+    #[test]
+    fn an_all_manual_rotation_updates_nothing() {
+        let tally = RepinTally {
+            manual: 4,
+            failed: vec![],
+        };
+        assert_eq!(tally.attempted(4), 0);
+        assert_eq!(tally.updated(4), 0);
+    }
+
+    /// The ordinary case: every domain tried and every one taken.
+    #[test]
+    fn a_clean_rotation_updates_every_domain() {
+        let tally = RepinTally::default();
+        assert_eq!(tally.attempted(3), 3);
+        assert_eq!(tally.updated(3), 3);
+    }
+
     /// A credential asking for less than its share keeps asking for less: the
     /// clamp is a ceiling, not a target.
     #[test]

From 3a7f40cfade50d75000aecbef4731bacd9274695 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 17:41:03 -0700
Subject: [PATCH 26/37] fix(gateway): keep a partial rotation on the page
 instead of in a toast

A rotation that could not re-pin every domain's CAA reported it through
`showToast`, which removes itself after three seconds and which the DNS-records
modal opened immediately afterwards covers. Everything else the handler does
reads as success -- the account URI cell flips to the new account, the domain
table reloads -- so once the toast expired the page held no trace that
re-pinning failed, for the operation whose whole point is that the operator
must not simply run it again.

Both lists are the same question, "what is left to do before this rotation is
finished", so they share one dialog. A dns-01-only cluster has the re-pin half
without the records half, so either alone opens it.

The re-pin section carries a Run SetCaa button, which is also the first time
that operation is reachable from the dashboard at all -- until now the docs
pointed at a remedy the UI could not perform. Rotation is reported rather than
raised because repeating it registers another account; SetCaa is the opposite,
idempotent and raising on partial failure, so it is safe to offer as a button
and reaching its success case means every domain took.
---
 docs/certbot-dns-persist-01.md          |  3 +-
 dstack/gateway/templates/dashboard.html | 85 ++++++++++++++++++-------
 2 files changed, 65 insertions(+), 23 deletions(-)

diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index f34b73713..2c93e1c96 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -213,7 +213,8 @@ Two operations behave differently on these domains:
   domain's CAA still succeeds — the account exists and the cluster is on it —
   and names those domains in `repin_failed_domains`. Finish with `SetCaa`
   rather than another rotation, which would register yet another account
-  against a rate-limited quota.
+  against a rate-limited quota. The dashboard reports both lists in one dialog
+  after a rotation, with a **Run SetCaa** button for the second.
 
 ## Related
 
diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html
index 73e224d69..d3854fafb 100644
--- a/dstack/gateway/templates/dashboard.html
+++ b/dstack/gateway/templates/dashboard.html
@@ -807,15 +807,31 @@ <h3>Edit ZT-Domain</h3>
     <!-- DNS Records To Publish Modal -->
     <div id="dns-records-modal" class="modal-overlay">
         <div class="modal">
-            <h3>Publish these DNS records</h3>
-            <p style="color: #666;">
-                This domain validates with <code>dns-persist-01</code>, so the gateway holds no
-                DNS credential for it and cannot write these itself. Certificates are only
-                issued once they are published.
-            </p>
-            <pre id="dns-records-list" style="white-space: pre-wrap; word-break: break-all;"></pre>
+            <h3>What is left to do</h3>
+            <div id="dns-records-repin" style="display: none;">
+                <p style="color: #b00;">
+                    The rotation succeeded and the cluster is on the new account, but these
+                    domains&#39; CAA records still name the old one, so nothing issues for them
+                    until a <code>SetCaa</code> run takes. Re-running the rotation would
+                    register yet another account against a rate-limited quota.
+                </p>
+                <pre id="dns-records-repin-list" style="white-space: pre-wrap; word-break: break-all;"></pre>
+                <div class="modal-actions">
+                    <button class="action-btn warning" onclick="retryCaa()">Run SetCaa</button>
+                </div>
+            </div>
+            <div id="dns-records-publish">
+                <p style="color: #666;">
+                    This domain validates with <code>dns-persist-01</code>, so the gateway holds no
+                    DNS credential for it and cannot write these itself. Certificates are only
+                    issued once they are published.
+                </p>
+                <pre id="dns-records-list" style="white-space: pre-wrap; word-break: break-all;"></pre>
+                <div class="modal-actions">
+                    <button class="action-btn secondary" onclick="copyDnsRecords()">Copy</button>
+                </div>
+            </div>
             <div class="modal-actions">
-                <button class="action-btn secondary" onclick="copyDnsRecords()">Copy</button>
                 <button class="action-btn primary" onclick="hideModal('dns-records-modal')">Done</button>
             </div>
         </div>
@@ -1571,19 +1587,13 @@ <h3>Add ZT-Domain</h3>
                 // The account URI is server-rendered, so it would otherwise keep
                 // showing the account this rotation just replaced.
                 document.getElementById('acme-account-uri').textContent = result.account_uri;
-                const failed = result.repin_failed_domains || [];
-                if (failed.length > 0) {
-                    // The rotation itself succeeded. Saying "failed" here would
-                    // invite a retry, which registers another account.
-                    showToast('Rotated to ' + result.account_uri + ', but CAA re-pinning failed for: '
-                        + failed.join(', ') + '. Run SetCaa again — do not rotate again.', 'error');
-                } else {
-                    showToast('ACME account: ' + result.account_uri, 'success');
-                }
+                showToast('ACME account: ' + result.account_uri, 'success');
                 await loadZtDomains();
-                // What the operator has to publish before those domains issue
-                // again -- the response is the only place it is reported.
-                showRequiredDnsRecords(result.required_dns_records);
+                // Everything still outstanding goes in the dialog, not a toast:
+                // a toast is gone in three seconds and the modal below would
+                // cover it, leaving a page that reads as a clean rotation. The
+                // response is the only place either list is reported.
+                showRequiredDnsRecords(result.required_dns_records, result.repin_failed_domains);
             } catch (e) {
                 showToast('Failed to register ACME account: ' + e.message, 'error');
             }
@@ -1601,14 +1611,45 @@ <h3>Add ZT-Domain</h3>
             showRequiredDnsRecords(records);
         }
 
-        function showRequiredDnsRecords(records) {
-            if (!records || records.length === 0) {
+        // Both halves of "this operation is not finished" in one dialog: the
+        // records only an operator can publish, and the domains whose CAA the
+        // gateway tried to re-pin and could not. A dns-01-only cluster has the
+        // second without the first, so either one alone still opens it.
+        function showRequiredDnsRecords(records, repinFailed) {
+            records = records || [];
+            repinFailed = repinFailed || [];
+            if (records.length === 0 && repinFailed.length === 0) {
                 return;
             }
+            document.getElementById('dns-records-publish').style.display =
+                records.length > 0 ? '' : 'none';
             document.getElementById('dns-records-list').textContent = records.join('\n');
+            document.getElementById('dns-records-repin').style.display =
+                repinFailed.length > 0 ? '' : 'none';
+            document.getElementById('dns-records-repin-list').textContent = repinFailed.join('\n');
             showModal('dns-records-modal');
         }
 
+        async function retryCaa() {
+            try {
+                const response = await fetch('/prpc/Admin.SetCaa', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: '{}'
+                });
+                if (!response.ok) {
+                    throw new Error(await response.text());
+                }
+                // SetCaa raises on partial failure, so reaching here means every
+                // domain took. Unlike rotation it is idempotent, which is why
+                // this is the operation to repeat and rotation is not.
+                document.getElementById('dns-records-repin').style.display = 'none';
+                showToast('CAA re-pinned', 'success');
+            } catch (e) {
+                showToast('SetCaa failed, run it again: ' + e.message, 'error');
+            }
+        }
+
         async function copyDnsRecords() {
             const text = document.getElementById('dns-records-list').textContent;
             try {

From 8eb783f81fc748ba3d40012d4ceeb16e7b53fd68 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 18:30:07 -0700
Subject: [PATCH 27/37] fix(gateway): let an update leave a ZT domain's
 challenge alone

`challenge` was a plain `string`, so the server could not tell "omitted" from
"dns-01". `UpdateZtDomain` replaces the whole record, so any caller that does
not know the field -- a cached dashboard bundle, an operator's script, an older
SDK -- downgraded a dns-persist-01 domain to dns-01 cluster-wide by editing
something unrelated. The domain's hand-published CAA names
`validationmethods=dns-persist-01`, so every order after that is refused by the
CA until someone rewrites the zone.

The compat layer cannot cover this: `carry_unknown_fields` restores fields the
writing *binary* does not declare, and this binary declares `challenge`.

Making it `optional` is what `issuer_domain_name` already does for the same
reason, and `merge_certbot_config` already assigns only on `Some`. An add has
nothing to preserve, so absence there is still the historical default; an
explicit empty string still reads as dns-01, which is what a proto3 zero value
carries.

Fixing it at the protocol rather than in the bundled dashboard also covers the
callers the dashboard fix could not reach -- it ships with the server, every
other client does not.
---
 dstack/gateway/rpc/proto/gateway_rpc.proto |   9 +-
 dstack/gateway/src/admin_service.rs        | 141 +++++++++++++++++++--
 2 files changed, 137 insertions(+), 13 deletions(-)

diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto
index ef1a29267..5a59732de 100644
--- a/dstack/gateway/rpc/proto/gateway_rpc.proto
+++ b/dstack/gateway/rpc/proto/gateway_rpc.proto
@@ -758,7 +758,14 @@ message ZtDomainConfig {
   // _validation-persist TXT record the operator publishes once, so the gateway
   // CVM never holds write access to the zone. Experimental — the draft is still
   // changing and Let's Encrypt serves it on staging only.
-  string challenge = 6;
+  //
+  // Optional so that omitting it means "leave it alone" rather than "dns-01".
+  // UpdateZtDomain replaces the whole record, so a caller that predates this
+  // field -- a cached dashboard bundle, a script, an older SDK -- would
+  // otherwise downgrade a dns-persist-01 domain to dns-01 cluster-wide on any
+  // unrelated edit, and its hand-published CAA naming dns-persist-01 then
+  // refuses every order.
+  optional string challenge = 6;
 }
 
 // ZT-Domain information (config + certificate status)
diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs
index 77b401fba..d840642e3 100644
--- a/dstack/gateway/src/admin_service.rs
+++ b/dstack/gateway/src/admin_service.rs
@@ -588,7 +588,8 @@ impl AdminRpc for AdminRpcHandler {
         let kv_store = self.state.kv_store();
         let cert_resolver = &self.state.cert_resolver;
 
-        let config = proto_to_zt_domain_config(&request, kv_store)?;
+        // Nothing stored to preserve: an omitted challenge is the default.
+        let config = proto_to_zt_domain_config(&request, kv_store, None)?;
 
         // Uniqueness is checked after normalization so wildcard, case, and a
         // trailing root dot cannot silently overwrite the same DNS name.
@@ -607,13 +608,16 @@ impl AdminRpc for AdminRpcHandler {
         let kv_store = self.state.kv_store();
         let cert_resolver = &self.state.cert_resolver;
 
-        let config = proto_to_zt_domain_config(&request, kv_store)?;
-
-        // Check the normalized key rather than the caller's presentation.
-        kv_store
-            .get_zt_domain_config(&config.domain)
+        // Read the stored record first: an omitted `challenge` means "leave it
+        // alone", and this is the only place its current value is known. Looked
+        // up on the normalized key rather than the caller's presentation.
+        let domain = normalize_zt_domain(&request.domain)?;
+        let stored = kv_store
+            .get_zt_domain_config(&domain)
             .context("ZT-Domain config not found")?;
 
+        let config = proto_to_zt_domain_config(&request, kv_store, Some(stored.challenge))?;
+
         kv_store.save_zt_domain_config(&config)?;
         info!("Updated ZT-Domain config: {}", config.domain);
 
@@ -965,9 +969,14 @@ fn validate_zt_domain(domain: &str) -> Result<()> {
 }
 
 /// Convert proto ZtDomainConfig to internal ZtDomainConfig
+///
+/// `current` is the challenge already stored for this domain, on an update.
+/// An omitted `challenge` resolves to it rather than to the default, so a caller
+/// that predates the field leaves it alone instead of downgrading the domain.
 fn proto_to_zt_domain_config(
     proto: &ProtoZtDomainConfig,
     kv_store: &crate::kv::KvStore,
+    current: Option<ChallengeKind>,
 ) -> Result<ZtDomainConfig> {
     // Normalize dns_cred_id: treat empty string as None (use default)
     let dns_cred_id = proto
@@ -988,11 +997,20 @@ fn proto_to_zt_domain_config(
         bail!("port must be between 1 and 65535");
     }
 
-    // Empty means the historical default: every ZT domain predates the choice.
-    let challenge = match proto.challenge.as_str() {
-        "" | "dns-01" => ChallengeKind::Dns01,
-        "dns-persist-01" => ChallengeKind::DnsPersist01,
-        other => bail!("unsupported challenge {other:?}, expected dns-01 or dns-persist-01"),
+    let challenge = match proto.challenge.as_deref() {
+        // Absent means "leave it as it is". UpdateZtDomain replaces the whole
+        // record, so reading absence as the default would let any edit from a
+        // caller that does not know the field -- a cached dashboard bundle, a
+        // script, an older SDK -- downgrade a dns-persist-01 domain to dns-01
+        // cluster-wide, after which its hand-published CAA naming
+        // dns-persist-01 refuses every order. On an add there is nothing to
+        // preserve, so it falls back to the historical default.
+        None => current.unwrap_or_default(),
+        // Present but empty is still the default: that is what an explicit
+        // proto3 zero value carries, and every ZT domain predates the choice.
+        Some("" | "dns-01") => ChallengeKind::Dns01,
+        Some("dns-persist-01") => ChallengeKind::DnsPersist01,
+        Some(other) => bail!("unsupported challenge {other:?}, expected dns-01 or dns-persist-01"),
     };
 
     Ok(ZtDomainConfig {
@@ -1040,7 +1058,7 @@ fn zt_domain_to_proto(
             port: config.port.into(),
             node: config.node,
             priority: config.priority,
-            challenge: challenge.to_string(),
+            challenge: Some(challenge.to_string()),
         }),
         cert_status,
         required_dns_records,
@@ -1243,6 +1261,105 @@ mod certbot_config_tests {
     }
 }
 
+/// An omitted `challenge` has to mean "leave it as it is", or every caller that
+/// predates the field silently downgrades a dns-persist-01 domain.
+#[cfg(test)]
+mod zt_domain_challenge_tests {
+    use super::*;
+
+    fn kv_store(dir: &std::path::Path) -> crate::kv::KvStore {
+        crate::kv::KvStore::new(1, vec![], dir, None).expect("failed to create kv store")
+    }
+
+    fn request(challenge: Option<&str>) -> ProtoZtDomainConfig {
+        ProtoZtDomainConfig {
+            domain: "example.com".to_string(),
+            dns_cred_id: None,
+            port: 443,
+            node: None,
+            priority: 7,
+            challenge: challenge.map(ToString::to_string),
+        }
+    }
+
+    /// The case the field is `optional` for: a cached dashboard bundle, a curl
+    /// script, or an older SDK edits an unrelated field and must not take the
+    /// domain's challenge down with it.
+    #[test]
+    fn an_omitted_challenge_keeps_the_stored_one() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let config = proto_to_zt_domain_config(
+            &request(None),
+            &kv_store(dir.path()),
+            Some(ChallengeKind::DnsPersist01),
+        )
+        .expect("an update without a challenge is valid");
+        assert_eq!(config.challenge, ChallengeKind::DnsPersist01);
+        assert_eq!(config.priority, 7);
+    }
+
+    /// On an add there is no stored value to preserve, so absence is the
+    /// historical default rather than an error.
+    #[test]
+    fn an_omitted_challenge_on_an_add_is_dns01() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let config = proto_to_zt_domain_config(&request(None), &kv_store(dir.path()), None)
+            .expect("an add without a challenge is valid");
+        assert_eq!(config.challenge, ChallengeKind::Dns01);
+    }
+
+    /// Preserving an omitted value must not make the field unsettable: a caller
+    /// that names dns-01 is asking to switch back, and gets it.
+    #[test]
+    fn an_explicit_challenge_overrides_the_stored_one() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let store = kv_store(dir.path());
+        let config = proto_to_zt_domain_config(
+            &request(Some("dns-01")),
+            &store,
+            Some(ChallengeKind::DnsPersist01),
+        )
+        .expect("an explicit challenge is valid");
+        assert_eq!(config.challenge, ChallengeKind::Dns01);
+
+        let config = proto_to_zt_domain_config(
+            &request(Some("dns-persist-01")),
+            &store,
+            Some(ChallengeKind::Dns01),
+        )
+        .expect("an explicit challenge is valid");
+        assert_eq!(config.challenge, ChallengeKind::DnsPersist01);
+    }
+
+    /// An explicit empty string is what a proto3 zero value carries, and every
+    /// ZT domain predates the choice, so it still reads as the default.
+    #[test]
+    fn an_explicitly_empty_challenge_is_the_default() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let config = proto_to_zt_domain_config(
+            &request(Some("")),
+            &kv_store(dir.path()),
+            Some(ChallengeKind::DnsPersist01),
+        )
+        .expect("an empty challenge is valid");
+        assert_eq!(config.challenge, ChallengeKind::Dns01);
+    }
+
+    /// An unrecognized value is refused rather than silently defaulted: a typo
+    /// that read as dns-01 would be the downgrade this field exists to prevent.
+    #[test]
+    fn an_unknown_challenge_is_refused() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let err = proto_to_zt_domain_config(
+            &request(Some("dns-persist-02")),
+            &kv_store(dir.path()),
+            None,
+        )
+        .expect_err("an unknown challenge must be refused");
+        assert!(err.to_string().contains("dns-persist-02"), "{err:#}");
+    }
+}
+
 #[cfg(test)]
 mod zt_domain_tests {
     use super::validate_zt_domain;

From e31ebf6b913a4a756efbc3dd9bca9d18707981f3 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 18:30:19 -0700
Subject: [PATCH 28/37] fix(certbot): name the missing setting when dns-01 has
 no cf_api_token

`cf_api_token` became `#[serde(default)]` so a dns-persist-01 config can omit
it, which it has no use for. That also stopped a dns-01 config from failing
deserialization when it forgets the token: it reached Cloudflare instead and
came back with

    failed to list zones: {"code":6111,"message":"Invalid format for
    Authorization header"}

naming a header the operator never wrote, where it used to say
`missing field 'cf_api_token'`. Still a startup failure either way, since
resolving the zone is authenticated -- but the diagnosis had to be recovered.

Checked in the dns-01 arm, symmetric with the `auto_set_caa` bail two arms
below, and it names the alternative: a deployment that does not want to hold a
provider credential has one.
---
 dstack/certbot/src/bot.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/dstack/certbot/src/bot.rs b/dstack/certbot/src/bot.rs
index ba53122e4..2ead321b6 100644
--- a/dstack/certbot/src/bot.rs
+++ b/dstack/certbot/src/bot.rs
@@ -287,6 +287,19 @@ fn issuer_domain_name(config: &CertBotConfig) -> Result<String> {
 async fn build_validation_method(config: &CertBotConfig) -> Result<ValidationMethod> {
     match config.challenge {
         ChallengeKind::Dns01 => {
+            // Named here rather than left to the provider. `cf_api_token` is
+            // `#[serde(default)]` so a dns-persist-01 config can omit it, which
+            // also means a dns-01 config that forgets it no longer fails
+            // deserialization -- it reaches Cloudflare and comes back as an
+            // "Invalid format for Authorization header", naming the header
+            // instead of the setting the operator has to add.
+            if config.cf_api_token.is_empty() {
+                bail!(
+                    "cf_api_token is required with dns-01, which proves control by writing a \
+                     TXT record through the DNS provider; set it, or switch to \
+                     challenge = \"dns-persist-01\", which needs no provider credential"
+                );
+            }
             let base_domain = config
                 .cert_subject_alt_names
                 .first()

From 5ebd99a296e522a005f3e514b8188b687068a853 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 18:30:19 -0700
Subject: [PATCH 29/37] fix(certbot): adopt stale CAA guards so a rerun can
 recover
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`set_caa_records` installs `;` guards, deletes the records they replace, writes
the new ones and drops the guards. A run interrupted anywhere in the middle
leaves the guards behind, and a `;` issue-value denies every issuer -- the
gateway's own error text tells the operator to rerun until it succeeds.

But the rerun's first call re-adds a byte-identical `0 issue ";"`. A provider
that rejects exact duplicates -- Cloudflare does for TXT -- fails the rerun at
step one, and the zone never recovers from the advice it was given.

So a rerun reuses the record it finds instead of adding a second one. The
obvious alternative, sweeping the stale guards first, fixes the same thing and
opens a worse hole: between the delete and the add the name has no issue or
issuewild record at all, and absent CAA is not "denied" but "any CA may issue"
(RFC 8659 §3). The trade is not symmetric -- a stranded guard is a renewal that
fails until someone reruns, loudly and recoverably, while a certificate
obtained during a fail-open window outlives the window and has to be found and
revoked. Adoption keeps the deny-all continuous, which is the only reason to
write a guard in the first place.

Guards beyond the two adopted are ordinary issue records as far as step 2 is
concerned, so they are removed there as before.
---
 dstack/certbot/src/acme_client.rs | 110 ++++++++++++++++++++++++++++--
 1 file changed, 104 insertions(+), 6 deletions(-)

diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs
index 3c02cfabd..56034b1d1 100644
--- a/dstack/certbot/src/acme_client.rs
+++ b/dstack/certbot/src/acme_client.rs
@@ -358,6 +358,19 @@ fn caa_tag(content: &str) -> Option<&str> {
     content.split_whitespace().nth(1)
 }
 
+/// Whether a rendered CAA record is one of the `;` guards [`AcmeClient::set_caa_records`]
+/// installs while it swaps the real records out.
+///
+/// A guard denies every issuer, which is the point while the old records are
+/// being replaced and the wrong thing to leave behind.
+fn is_caa_guard(content: &str) -> bool {
+    let mut parts = content.split_whitespace();
+    let (_flags, tag, value) = (parts.next(), parts.next(), parts.next());
+    matches!(tag, Some("issue" | "issuewild"))
+        && value.map(|value| value.trim_matches('"')) == Some(";")
+        && parts.next().is_none()
+}
+
 impl AcmeClient {
     pub async fn load(
         validation: ValidationMethod,
@@ -481,14 +494,57 @@ impl AcmeClient {
         let base_names = base_names(domains);
 
         for base_name in base_names {
+            // 0. Adopt guards an earlier run left behind, rather than
+            //    replacing them. A run that died between steps 1 and 4 stranded
+            //    `;` records, and step 1 would re-add byte-identical ones --
+            //    which a provider that refuses duplicates rejects, so the rerun
+            //    the operator is told to perform fails at its very first call
+            //    and the zone never recovers.
+            //
+            //    Deleting them first would fix that and open a worse hole: from
+            //    the delete until step 1 succeeds the name has no issue or
+            //    issuewild record at all, which is not "denied" but "any CA may
+            //    issue" -- and a certificate obtained in that window outlives
+            //    the window, where a stranded guard is merely a renewal that
+            //    fails until someone reruns. Reusing the record keeps the
+            //    deny-all continuous, which is the whole point of a guard.
+            //
+            //    Extras beyond the two adopted here are ordinary issue records
+            //    to step 2, which removes them.
+            let mut adopted_issue = None;
+            let mut adopted_issuewild = None;
+            for record in dns01_client.get_records(base_name).await? {
+                if record.r#type != "CAA" || !is_caa_guard(&record.content) {
+                    continue;
+                }
+                let slot = match caa_tag(&record.content) {
+                    Some("issue") => &mut adopted_issue,
+                    Some("issuewild") => &mut adopted_issuewild,
+                    _ => continue,
+                };
+                if slot.is_none() {
+                    debug!("adopting stale guard CAA record {}", record.name);
+                    *slot = Some(record.id);
+                }
+            }
             // 1. Set ";" to guard timing gap between the operations.
             debug!("setting guard CAA records for {base_name}");
-            let guard0 = dns01_client
-                .add_caa_record(base_name, 0, "issue", ";")
-                .await?;
-            let guard1 = dns01_client
-                .add_caa_record(base_name, 0, "issuewild", ";")
-                .await?;
+            let guard0 = match adopted_issue {
+                Some(id) => id,
+                None => {
+                    dns01_client
+                        .add_caa_record(base_name, 0, "issue", ";")
+                        .await?
+                }
+            };
+            let guard1 = match adopted_issuewild {
+                Some(id) => id,
+                None => {
+                    dns01_client
+                        .add_caa_record(base_name, 0, "issuewild", ";")
+                        .await?
+                }
+            };
             // 2. Remove the existing constraints
             for record in dns01_client.get_records(base_name).await? {
                 if record.id == guard0 || record.id == guard1 {
@@ -1670,6 +1726,48 @@ mod reissue_reason_tests {
     }
 }
 
+#[cfg(test)]
+mod caa_guard_tests {
+    use super::{caa_tag, is_caa_guard};
+
+    /// The guard is what an interrupted `set_caa_records` leaves behind, and a
+    /// rerun has to recognize its own leftovers to be able to replace them.
+    #[test]
+    fn a_guard_is_recognized_however_the_provider_renders_it() {
+        assert!(is_caa_guard(r#"0 issue ";""#));
+        assert!(is_caa_guard(r#"0 issuewild ";""#));
+        assert!(is_caa_guard("0 issue ;"));
+        assert!(is_caa_guard("128 issue \";\""));
+    }
+
+    /// A real issuer record must never be mistaken for a guard: sweeping one
+    /// early would delete the zone's only valid CAA before its replacement is
+    /// written.
+    #[test]
+    fn a_real_issuer_record_is_not_a_guard() {
+        assert!(!is_caa_guard(
+            r#"0 issue "letsencrypt.org;validationmethods=dns-01;accounturi=https://acme.example/acct/1""#
+        ));
+        assert!(!is_caa_guard(r#"0 issue "letsencrypt.org""#));
+        // `iodef` is neither an issuer constraint nor a guard.
+        assert!(!is_caa_guard(r#"0 iodef "mailto:x@example.com""#));
+        // A value that merely starts with the guard's character.
+        assert!(!is_caa_guard(r#"0 issue ";extra""#));
+        // Trailing junk means it is not the record this wrote.
+        assert!(!is_caa_guard(r#"0 issue ";" extra"#));
+        assert!(!is_caa_guard(""));
+    }
+
+    /// The sweep is scoped by the same tag test the replacement pass uses.
+    #[test]
+    fn the_guard_tags_are_the_ones_the_replacement_pass_replaces() {
+        for content in [r#"0 issue ";""#, r#"0 issuewild ";""#] {
+            let tag = caa_tag(content).expect("a rendered CAA record has a tag");
+            assert!(matches!(tag, "issue" | "issuewild"), "{tag}");
+        }
+    }
+}
+
 #[cfg(test)]
 mod dns_wait_tests {
     use super::advisory_dns_wait;

From 4467792fbd54d851270ec5b96822b93eac341f79 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 18:30:34 -0700
Subject: [PATCH 30/37] test(certbot): hold every config key to its own doc
 comment

`certbot cfg` labelled every key after the first absent one with its
neighbour's comment, because it walked the document by position and indexed
`FIELD_DOCS` by the same counter while `toml_edit` drops `None` options.
`max_dns_wait` came out as "Renew timeout in seconds", next to the one value
whose relationship with `renew_timeout` decides whether a missing record is
reported by name -- in the file `certbot cfg` exists to hand a new operator.

Fixed upstream in da36db82d3 with no test. These are that test: one holding
every rendered key to the comment `get_field_docs` returns for it, so a future
drift is caught wherever it lands rather than at the one key that drifted, and
one naming `max_dns_wait` because it is the key that made the bug visible.

Both fail against the positional lookup and pass against the fix.
---
 dstack/certbot/cli/src/main.rs | 59 ++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/dstack/certbot/cli/src/main.rs b/dstack/certbot/cli/src/main.rs
index f72838466..bcc41ca8f 100644
--- a/dstack/certbot/cli/src/main.rs
+++ b/dstack/certbot/cli/src/main.rs
@@ -284,3 +284,62 @@ async fn main() -> Result<()> {
     }
     Ok(())
 }
+
+#[cfg(test)]
+mod config_template_tests {
+    use super::*;
+
+    /// Every key has to carry its own doc comment. `toml_edit` drops `None`
+    /// fields from the serialized document -- `cf_api_url` is unset by default
+    /// -- so indexing `FIELD_DOCS` by position shifts every comment after the
+    /// gap onto the wrong setting.
+    #[test]
+    fn every_key_is_labelled_with_its_own_doc_comment() {
+        let rendered = Config::default()
+            .to_commented_toml()
+            .expect("the default config renders");
+
+        // Pair each key with the comment block immediately above it.
+        let mut comment = String::new();
+        let mut pairs = Vec::new();
+        for line in rendered.lines() {
+            match line.strip_prefix('#') {
+                Some(text) => comment.push_str(text.trim()),
+                None => {
+                    if let Some((key, _)) = line.split_once('=') {
+                        pairs.push((key.trim().to_string(), std::mem::take(&mut comment)));
+                    }
+                }
+            }
+        }
+        assert!(!pairs.is_empty(), "no keys rendered:\n{rendered}");
+
+        for (key, comment) in &pairs {
+            let expected = Config::get_field_docs(key)
+                .unwrap_or_else(|err| panic!("no doc comment for {key:?}: {err}"));
+            let expected: String = expected.lines().map(str::trim).collect();
+            assert_eq!(
+                comment, &expected,
+                "key {key:?} is labelled with another field's doc comment"
+            );
+        }
+    }
+
+    /// The field that made the misalignment visible: it sat after the dropped
+    /// `cf_api_url` and was labelled "Renew timeout in seconds", next to the one
+    /// value whose interaction with `renew_timeout` this crate clamps.
+    #[test]
+    fn max_dns_wait_is_not_labelled_as_a_renew_timeout() {
+        let rendered = Config::default()
+            .to_commented_toml()
+            .expect("the default config renders");
+        let (before, _) = rendered
+            .split_once("max_dns_wait")
+            .expect("max_dns_wait is rendered");
+        let label = before.lines().last().expect("it has a comment above it");
+        assert!(
+            label.contains("DNS propagation"),
+            "max_dns_wait is labelled {label:?}"
+        );
+    }
+}

From 4ec8980e9bf83080bfdedd59aad918cc270c18f8 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 18:30:34 -0700
Subject: [PATCH 31/37] test(gateway): cover dns-persist-01 up to the CA in the
 e2e suite

The challenge had no end-to-end coverage: unit tests pin the record grammar and
the tallies, but not that a domain reaches the CA without a DNS credential.

Phase 10 asserts what this harness can actually decide -- the domain is
accepted holding no credential, the record an operator must publish is rendered
on the base name with `policy=wildcard`, the challenge survives an update that
omits it, and nothing for that domain ever reaches the provider API.

Issuance itself is not asserted, because the pinned Pebble image cannot serve
it: its binary carries `dns-01`, `http-01`, `tls-alpn-01` and `dns-account-01`,
and neither `dns-persist-01` nor `_validation-persist`. A run confirms this --
the order is created, the authorization fetched, and no matching challenge is
found. Closing that gap needs a Pebble built with the draft, and the comment on
`PERSIST_DOMAIN` says so rather than leaving the next reader to find out.

The domain is kept out of `CERT_DOMAINS` so the certificate phases do not
expect one for it, and the add deletes first, since the KV store outlives a run
and an "already exists" rejection would mask whether the add is accepted.
---
 docs/certbot-dns-persist-01.md      | 13 ++++-
 dstack/gateway/test-run/e2e/test.sh | 83 +++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+), 1 deletion(-)

diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md
index 2c93e1c96..0e090fd80 100644
--- a/docs/certbot-dns-persist-01.md
+++ b/docs/certbot-dns-persist-01.md
@@ -167,7 +167,12 @@ The records name the ACME account, so the account comes first:
 Adding another dns-persist-01 domain later starts at step 3: the account already
 exists, so the new domain's records are available as soon as it is. A domain's
 challenge is chosen when it is added — in the dashboard's ZT-Domain form as well
-as over the API — and carried forward by every edit.
+as over the API — and carried forward by every edit. `UpdateZtDomain` replaces
+the whole record, so `challenge` is optional there and omitting it means "leave
+it as it is": a caller that predates the field — a cached dashboard bundle, a
+script, an older SDK — cannot downgrade a `dns-persist-01` domain to `dns-01` by
+editing something else, which its hand-published CAA would then refuse. Send the
+field explicitly to switch a domain over.
 
 ### Naming the CA
 
@@ -188,6 +193,12 @@ malformed `issue` property with nothing valid behind it — CAA that forbids eve
 issuer. Anything that is not a DNS name is refused by `SetCertbotConfig`, and by
 `certbot` at startup.
 
+Publishing CAA installs a temporary `;` guard, deletes the records it replaces,
+writes the new ones and drops the guard. A run interrupted part-way leaves the
+guard behind, which denies every issuer until a later run finishes the job — so
+each run now sweeps stale guards before installing its own, and a rerun is
+enough to recover.
+
 ### How long the check waits
 
 The self-check before each order waits for the record to become visible, but
diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh
index 223fc5219..273db5a22 100755
--- a/dstack/gateway/test-run/e2e/test.sh
+++ b/dstack/gateway/test-run/e2e/test.sh
@@ -33,6 +33,16 @@ PEBBLE_DIR="http://pebble:14000/dir"
 # Certificate domains to test (base domains, certs will be issued for *.domain)
 CERT_DOMAINS="test0.local test1.local test2.local"
 
+# A dns-persist-01 domain. Deliberately outside CERT_DOMAINS: no certificate is
+# expected for it, because the Pebble image this harness pins does not implement
+# draft-ietf-acme-dns-persist-01 (its binary carries dns-01, http-01, tls-alpn-01
+# and dns-account-01, and neither "dns-persist-01" nor "_validation-persist").
+# What is asserted here is everything up to the CA: that such a domain is
+# accepted with no DNS credential, that the record an operator must publish is
+# rendered, that the challenge survives an edit, and that certbot never writes
+# DNS for it. Issuance itself needs a Pebble built with the draft.
+PERSIST_DOMAIN="persist0.local"
+
 # Cloudflare mock settings
 CF_API_TOKEN="test-token"
 CF_API_URL="http://mock-cf-dns-api:8080/client/v4"
@@ -113,6 +123,68 @@ get_cert_san() {
     get_cert_pem "$1" "$2" | openssl x509 -noout -ext subjectAltName 2>/dev/null
 }
 
+# ==================== dns-persist-01 Helpers ====================
+
+admin_post() {
+    curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.$1" \
+        -H "${ADMIN_AUTH_HEADER}" \
+        -H "Content-Type: application/json" \
+        -d "$2"
+}
+
+# A dns-persist-01 domain must be accepted without any DNS credential: holding
+# none is the entire point of the challenge.
+test_persist_domain_needs_no_credential() {
+    # Idempotent: the KV store outlives a run, and an "already exists" rejection
+    # would mask whether the add is accepted at all.
+    admin_post DeleteZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}' > /dev/null 2>&1 || true
+    admin_post AddZtDomain \
+        '{"domain": "'"${PERSIST_DOMAIN}"'", "port": 443, "challenge": "dns-persist-01"}' \
+        > /dev/null
+}
+
+# The record is the setup under this challenge, so the API has to render it.
+test_persist_domain_reports_its_record() {
+    local info
+    info=$(admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}') || return 1
+    # Fixed strings throughout: as a regex, "_validation-persist.*.domain"
+    # matches the very name it is meant to rule out.
+    #
+    # The record sits on the base name. ACME strips the wildcard before it
+    # creates the authorization, so a record published under the "*." form is
+    # never read.
+    if echo "$info" | grep -qF "_validation-persist.*.${PERSIST_DOMAIN}"; then
+        return 1
+    fi
+    echo "$info" | grep -qF "_validation-persist.${PERSIST_DOMAIN}. IN TXT" || return 1
+    echo "$info" | grep -qF "accounturi=" || return 1
+    # policy=wildcard, because the gateway only ever orders *.{domain}.
+    echo "$info" | grep -qF "policy=wildcard"
+}
+
+# UpdateZtDomain replaces the whole record, and `challenge` is optional so that
+# a caller which does not know the field leaves it alone. A client that predates
+# it -- a cached dashboard bundle, a script, an older SDK -- must not be able to
+# downgrade the domain to dns-01, which its published CAA would then refuse.
+test_persist_challenge_survives_an_edit() {
+    admin_post UpdateZtDomain \
+        '{"domain": "'"${PERSIST_DOMAIN}"'", "port": 443, "priority": 5}' > /dev/null || return 1
+    local info
+    info=$(admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}') || return 1
+    echo "$info" | grep -q '"priority":5' || return 1
+    echo "$info" | grep -q "dns-persist-01"
+}
+
+# The claim the whole challenge exists for: certbot reads DNS and never writes
+# it, so nothing for this domain may ever reach the provider API.
+test_persist_domain_never_touches_the_provider() {
+    admin_post RenewZtDomainCert \
+        '{"domain": "'"${PERSIST_DOMAIN}"'", "force": true}' > /dev/null 2>&1 || true
+    local records
+    records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo "[]")
+    ! echo "$records" | grep -q "${PERSIST_DOMAIN}"
+}
+
 # ==================== Test Functions ====================
 
 test_http_health() {
@@ -336,6 +408,17 @@ main() {
         log_info "No DNS TXT records (expected if certs cached)"
     fi
 
+    # Phase 10: dns-persist-01 plumbing, up to but not including the CA
+    log_phase 10 "dns-persist-01 without a DNS credential"
+    run_test "Domain accepted with no DNS credential" \
+        "$(test_persist_domain_needs_no_credential; echo $?)"
+    run_test "Required validation record is reported" \
+        "$(test_persist_domain_reports_its_record; echo $?)"
+    run_test "Challenge survives an edit that omits it" \
+        "$(test_persist_challenge_survives_an_edit; echo $?)"
+    run_test "No DNS record is ever written for it" \
+        "$(test_persist_domain_never_touches_the_provider; echo $?)"
+
     # Summary
     log_section "Test Summary"
     log_info "Passed: $TESTS_PASSED"

From 63294d208e48b34949be7f1f5330ec0458e8b227 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 19:00:52 -0700
Subject: [PATCH 32/37] test(gateway): issue a real dns-persist-01 certificate
 in the e2e suite

The phase added a moment ago stopped short of the CA, because the pinned
Pebble image could not serve the challenge -- its binary carried dns-01,
http-01, tls-alpn-01 and dns-account-01, and neither "dns-persist-01" nor
"_validation-persist".

Upstream Pebble has implemented the draft since (va.validateDNSPersist01,
parsing the record as an RFC 8659 issue-value), so the image is rebuilt from
upstream v2.10.1 with only the `-http` flag this harness has always needed,
and the tag is pinned rather than floating on :latest. The flag now lives at
github.com/kvinwang/pebble branch http-flag instead of in an unrecorded
working tree.

So the challenge is covered end to end: selected out of the authorization,
posted ready, finalized, and the certificate fetched -- for a domain the
gateway holds no credential for. The DNS answer is the only part standing
in, since nothing publishes the record here and Pebble runs with
PEBBLE_VA_ALWAYS_VALID. certbot warns and proceeds by design, because the
CA's DNS view is not this node's.

Two orderings the suite has to respect, both learned by getting them wrong:

- Every record names the ACME account, so `required_dns_records` is empty
  until one exists, and in this harness issuing is what creates it. The
  record assertion therefore runs after issuance, not before.
- `renew_timeout` bounds the DNS self-check at half its value. Left at the
  default the order would sit for 150s waiting on a record that never
  arrives, so the harness sets 60s and the poll outlasts it.

`issuer_domain_name` is set to pebble.letsencrypt.org: a record naming a CA
the challenge does not list is ignored, and Pebble does not answer to
letsencrypt.org.
---
 .../gateway/test-run/e2e/docker-compose.yml   |  6 +-
 dstack/gateway/test-run/e2e/test.sh           | 67 ++++++++++++++-----
 2 files changed, 56 insertions(+), 17 deletions(-)

diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml
index 59048d84f..1039b350a 100644
--- a/dstack/gateway/test-run/e2e/docker-compose.yml
+++ b/dstack/gateway/test-run/e2e/docker-compose.yml
@@ -51,9 +51,11 @@ services:
       timeout: 3s
       retries: 5
 
-  # Pebble - Let's Encrypt test server (custom build with HTTP support)
+  # Pebble - Let's Encrypt test server. Upstream plus a `-http` flag, so a
+  # client in another container can reach it by service name without trusting
+  # the Pebble root: github.com/kvinwang/pebble branch http-flag.
   pebble:
-    image: kvin/pebble:latest
+    image: kvin/pebble:v2.10.1-http
     container_name: pebble
     command: ["-http", "-dnsserver", "172.30.0.10:53"]
     networks:
diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh
index 273db5a22..4698514b3 100755
--- a/dstack/gateway/test-run/e2e/test.sh
+++ b/dstack/gateway/test-run/e2e/test.sh
@@ -33,16 +33,15 @@ PEBBLE_DIR="http://pebble:14000/dir"
 # Certificate domains to test (base domains, certs will be issued for *.domain)
 CERT_DOMAINS="test0.local test1.local test2.local"
 
-# A dns-persist-01 domain. Deliberately outside CERT_DOMAINS: no certificate is
-# expected for it, because the Pebble image this harness pins does not implement
-# draft-ietf-acme-dns-persist-01 (its binary carries dns-01, http-01, tls-alpn-01
-# and dns-account-01, and neither "dns-persist-01" nor "_validation-persist").
-# What is asserted here is everything up to the CA: that such a domain is
-# accepted with no DNS credential, that the record an operator must publish is
-# rendered, that the challenge survives an edit, and that certbot never writes
-# DNS for it. Issuance itself needs a Pebble built with the draft.
+# A dns-persist-01 domain. Kept out of CERT_DOMAINS because the certificate
+# phases check TLS through the proxy, which needs an app behind the domain;
+# here the certificate is checked through the admin API instead.
 PERSIST_DOMAIN="persist0.local"
 
+# Pebble answers to this name, not to letsencrypt.org, and a validation record
+# naming a CA the challenge does not list is ignored.
+PERSIST_ISSUER="pebble.letsencrypt.org"
+
 # Cloudflare mock settings
 CF_API_TOKEN="test-token"
 CF_API_URL="http://mock-cf-dns-api:8080/client/v4"
@@ -175,11 +174,36 @@ test_persist_challenge_survives_an_edit() {
     echo "$info" | grep -q "dns-persist-01"
 }
 
+# The point of the whole exercise: a certificate is issued for a domain the
+# gateway holds no credential for. The record is not published anywhere in this
+# harness, so the pre-order self-check cannot pass -- it is advisory by design,
+# and certbot warns and proceeds because the CA's DNS view is not this node's.
+# Pebble runs with PEBBLE_VA_ALWAYS_VALID, so it accepts.
+#
+# That leaves the challenge itself covered end to end -- selected out of the
+# authorization, posted ready, finalized, fetched -- and the DNS answer the only
+# part standing in.
+test_persist_domain_issues_a_certificate() {
+    admin_post RenewZtDomainCert \
+        '{"domain": "'"${PERSIST_DOMAIN}"'", "force": true}' > /dev/null 2>&1 || true
+
+    # The renewal outlives the request that triggered it, and spends the
+    # advisory DNS wait -- half of renew_timeout -- before it reaches the CA.
+    local i=0
+    while [ $i -lt 60 ]; do
+        if admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}' \
+            | grep -q '"has_cert":true'; then
+            return 0
+        fi
+        sleep 2
+        i=$((i + 1))
+    done
+    return 1
+}
+
 # The claim the whole challenge exists for: certbot reads DNS and never writes
 # it, so nothing for this domain may ever reach the provider API.
 test_persist_domain_never_touches_the_provider() {
-    admin_post RenewZtDomainCert \
-        '{"domain": "'"${PERSIST_DOMAIN}"'", "force": true}' > /dev/null 2>&1 || true
     local records
     records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo "[]")
     ! echo "$records" | grep -q "${PERSIST_DOMAIN}"
@@ -236,12 +260,21 @@ test_proxy_tls_health() {
 setup_certbot_config() {
     log_info "Configuring certbot via Admin API..."
 
-    # Set ACME URL
-    log_info "Setting ACME URL: ${ACME_URL}"
+    # Set ACME URL and the name this CA is known by. The issuer name is what a
+    # dns-persist-01 record has to carry, and it is also what CAA names for
+    # dns-01, so it is one setting for the whole deployment.
+    # renew_timeout also bounds the pre-order DNS self-check, which is capped at
+    # half of it. The default 300s would have a dns-persist-01 order sit for
+    # 150s waiting on a record this harness never publishes.
+    log_info "Setting ACME URL: ${ACME_URL} (issuer ${PERSIST_ISSUER})"
     if ! curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.SetCertbotConfig" \
         -H "${ADMIN_AUTH_HEADER}" \
         -H "Content-Type: application/json" \
-        -d '{"acme_url": "'"${ACME_URL}"'"}' > /dev/null; then
+        -d '{
+            "acme_url": "'"${ACME_URL}"'",
+            "issuer_domain_name": "'"${PERSIST_ISSUER}"'",
+            "renew_timeout_secs": 60
+        }' > /dev/null; then
         log_error "Failed to set certbot config"
         return 1
     fi
@@ -412,10 +445,14 @@ main() {
     log_phase 10 "dns-persist-01 without a DNS credential"
     run_test "Domain accepted with no DNS credential" \
         "$(test_persist_domain_needs_no_credential; echo $?)"
-    run_test "Required validation record is reported" \
-        "$(test_persist_domain_reports_its_record; echo $?)"
     run_test "Challenge survives an edit that omits it" \
         "$(test_persist_challenge_survives_an_edit; echo $?)"
+    run_test "Certificate issues without a DNS credential" \
+        "$(test_persist_domain_issues_a_certificate; echo $?)"
+    # After issuance: every record names the ACME account, so there is nothing
+    # to report until one exists, and issuing is what creates it here.
+    run_test "Required validation record is reported" \
+        "$(test_persist_domain_reports_its_record; echo $?)"
     run_test "No DNS record is ever written for it" \
         "$(test_persist_domain_never_touches_the_provider; echo $?)"
 

From 102f896b753cdc7359b35bb0f6eb4f372fa764a1 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 19:21:31 -0700
Subject: [PATCH 33/37] test: answer DNS over TCP in the mock, which is all
 Pebble asks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The mock has served TXT on UDP/53 since it was written, and both e2e suites
point Pebble's -dnsserver at it -- yet both also set PEBBLE_VA_ALWAYS_VALID,
because the wiring never worked.

Pebble sets `dnsClient.Net = "tcp"` whenever it is given a custom resolver
(va/va.go), so it never sent a UDP query to answer. The DNS side of the
harness looked complete and validated nothing.

TCP is the same responder behind RFC 1035 §4.2.2 framing: a two-byte length
in front of each message, in both directions. `dns_response` is unchanged and
serves both paths, so a record is answered identically however it is asked
for.
---
 .../full-stack-compose/mock-cf-dns/Dockerfile |  2 +-
 .../full-stack-compose/mock-cf-dns/server.py  | 71 +++++++++++++++++--
 2 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/test-suites/full-stack-compose/mock-cf-dns/Dockerfile b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile
index 8c68f2ec1..679dc4164 100644
--- a/test-suites/full-stack-compose/mock-cf-dns/Dockerfile
+++ b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile
@@ -5,5 +5,5 @@
 FROM python:3.12-slim
 WORKDIR /app
 COPY server.py /app/server.py
-EXPOSE 8080/tcp 53/udp
+EXPOSE 8080/tcp 53/udp 53/tcp
 CMD ["python", "/app/server.py"]
diff --git a/test-suites/full-stack-compose/mock-cf-dns/server.py b/test-suites/full-stack-compose/mock-cf-dns/server.py
index b3bc07d51..3ba0c0766 100755
--- a/test-suites/full-stack-compose/mock-cf-dns/server.py
+++ b/test-suites/full-stack-compose/mock-cf-dns/server.py
@@ -10,8 +10,10 @@
   POST   /client/v4/zones/<zone_id>/dns_records
   DELETE /client/v4/zones/<zone_id>/dns_records/<record_id>
 
-It also serves TXT answers on UDP/53 so Pebble can validate DNS-01 when the
-ACME server is not configured with PEBBLE_VA_ALWAYS_VALID=1.
+It also serves TXT answers on 53, over both UDP and TCP, so Pebble can validate
+DNS-01 and dns-persist-01 for real rather than being run with
+PEBBLE_VA_ALWAYS_VALID=1. TCP is not optional: Pebble sets its DNS client to
+`Net = "tcp"` whenever it is given `-dnsserver`.
 """
 
 from __future__ import annotations
@@ -266,9 +268,15 @@ def dns_response(packet: bytes) -> bytes:
     return header + body
 
 
+def _debug(message: str) -> None:
+    if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}:
+        print(message, flush=True)
+
+
 def dns_loop() -> None:
     """Serve mock DNS responses over UDP."""
     sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     sock.bind(("0.0.0.0", 53))
     while True:
         packet, addr = sock.recvfrom(4096)
@@ -277,15 +285,68 @@ def dns_loop() -> None:
             if response:
                 sock.sendto(response, addr)
         except Exception as exc:  # pragma: no cover - diagnostic only
-            if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}:
-                print(f"dns error from {addr}: {exc}", flush=True)
+            _debug(f"dns error from {addr}: {exc}")
+
+
+def _serve_dns_over_tcp(conn: socket.socket, addr: Any) -> None:
+    """Answer one TCP DNS client, which frames each message with its length."""
+    try:
+        conn.settimeout(5)
+        while True:
+            head = _recv_exactly(conn, 2)
+            if head is None:
+                return
+            (length,) = struct.unpack("!H", head)
+            query = _recv_exactly(conn, length)
+            if query is None:
+                return
+            response = dns_response(query)
+            if not response:
+                return
+            conn.sendall(struct.pack("!H", len(response)) + response)
+    except Exception as exc:  # pragma: no cover - diagnostic only
+        _debug(f"dns/tcp error from {addr}: {exc}")
+    finally:
+        conn.close()
+
+
+def _recv_exactly(conn: socket.socket, count: int) -> bytes | None:
+    """Read exactly `count` bytes, or None if the peer stopped sending."""
+    buf = b""
+    while len(buf) < count:
+        chunk = conn.recv(count - len(buf))
+        if not chunk:
+            return None
+        buf += chunk
+    return buf
+
+
+def dns_tcp_loop() -> None:
+    """Serve mock DNS responses over TCP.
+
+    Pebble sets `dnsClient.Net = "tcp"` whenever it is given a custom resolver
+    (`-dnsserver`), so a UDP-only mock answers none of its queries and every
+    challenge has to be waved through with PEBBLE_VA_ALWAYS_VALID. RFC 1035
+    §4.2.2 length-prefixes each message on TCP; that framing is the only
+    difference from the UDP path.
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock.bind(("0.0.0.0", 53))
+    sock.listen(16)
+    while True:
+        conn, addr = sock.accept()
+        threading.Thread(
+            target=_serve_dns_over_tcp, args=(conn, addr), daemon=True
+        ).start()
 
 
 def main() -> None:
     """Run the mock Cloudflare API and DNS servers."""
     threading.Thread(target=dns_loop, daemon=True).start()
+    threading.Thread(target=dns_tcp_loop, daemon=True).start()
     port = int(os.environ.get("PORT", "8080"))
-    print(f"mock CF API on :{port}; zones={_zones()}", flush=True)
+    print(f"mock CF API on :{port}; DNS on :53 (udp+tcp); zones={_zones()}", flush=True)
     ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever()
 
 

From 9fa84218f166de0ff28ceab7da4b56d3044a09c4 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 19:21:41 -0700
Subject: [PATCH 34/37] test(gateway): validate the dns-persist-01 record
 against the CA, not around it

The phase issued a certificate but proved nothing about the record: Pebble ran
with PEBBLE_VA_ALWAYS_VALID, so it accepted a challenge it never looked up.
Everything the record's grammar has to satisfy -- the `; ` separator, the
issuer name the CA answers to, `policy=wildcard` covering the base name -- was
pinned only by unit tests written against a reading of Boulder.

Now the harness publishes the record the way a zone owner would, once, before
the first order, taking it verbatim from what the gateway reports; Pebble
looks it up and parses it as an RFC 8659 issue-value. A record this
implementation renders and a CA implementation rejects now fails the suite.

That is possible because the mock answers TCP, and worth doing because the
gateway itself never writes the record -- which is the claim the last
assertion checks, scoped to the `_acme-challenge` name certbot would have
written rather than to the zone being empty.

The mock is built from the repo instead of pulled as kvin/mock-cf-dns-api,
which was HTTP-only and could not have served this. Both suites now build the
same image.
---
 .../gateway/test-run/e2e/docker-compose.yml   | 14 ++++-
 dstack/gateway/test-run/e2e/test.sh           | 63 ++++++++++++++-----
 2 files changed, 60 insertions(+), 17 deletions(-)

diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml
index 1039b350a..945e98b05 100644
--- a/dstack/gateway/test-run/e2e/docker-compose.yml
+++ b/dstack/gateway/test-run/e2e/docker-compose.yml
@@ -33,9 +33,13 @@ services:
 
   # ==================== Mock Services ====================
 
-  # Mock Cloudflare DNS API
+  # Mock Cloudflare DNS API. Built from the repo rather than pulled, and the
+  # same image the full-stack suite builds, so both suites answer DNS the same
+  # way and neither depends on a prebuilt image nobody can rebuild.
   mock-cf-dns-api:
-    image: kvin/mock-cf-dns-api:latest
+    build:
+      context: ../../../../test-suites/full-stack-compose/mock-cf-dns
+    image: dstack-e2e-mock-cf-dns:local
     container_name: mock-cf-dns-api
     networks:
       certbot-test:
@@ -45,6 +49,8 @@ services:
     environment:
       - PORT=8080
       - DEBUG=true
+      # The zones certbot writes into and Pebble reads back out of.
+      - MOCK_CF_ZONES=test0.local,test1.local,test2.local,persist0.local
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"]
       interval: 5s
@@ -66,7 +72,9 @@ services:
       - "15000:15000"  # Management interface
     environment:
       - PEBBLE_VA_NOSLEEP=1
-      - PEBBLE_VA_ALWAYS_VALID=1  # Skip actual DNS validation for testing
+      # No PEBBLE_VA_ALWAYS_VALID: the mock answers DNS over TCP, which is the
+      # only transport Pebble uses once it is given -dnsserver, so challenges
+      # are validated against the records rather than waved through.
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:14000/dir"]
       interval: 5s
diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh
index 4698514b3..69b69b19d 100755
--- a/dstack/gateway/test-run/e2e/test.sh
+++ b/dstack/gateway/test-run/e2e/test.sh
@@ -38,6 +38,9 @@ CERT_DOMAINS="test0.local test1.local test2.local"
 # here the certificate is checked through the admin API instead.
 PERSIST_DOMAIN="persist0.local"
 
+# Mock zone ID, as server.py derives it from the zone name.
+PERSIST_ZONE_ID="zone-persist0-local"
+
 # Pebble answers to this name, not to letsencrypt.org, and a validation record
 # naming a CA the challenge does not list is ignored.
 PERSIST_ISSUER="pebble.letsencrypt.org"
@@ -174,15 +177,43 @@ test_persist_challenge_survives_an_edit() {
     echo "$info" | grep -q "dns-persist-01"
 }
 
-# The point of the whole exercise: a certificate is issued for a domain the
-# gateway holds no credential for. The record is not published anywhere in this
-# harness, so the pre-order self-check cannot pass -- it is advisory by design,
-# and certbot warns and proceeds because the CA's DNS view is not this node's.
-# Pebble runs with PEBBLE_VA_ALWAYS_VALID, so it accepts.
+# Publish the validation record the way a zone owner would: once, by hand,
+# before the first order. certbot never writes it -- that is the whole point --
+# so the harness stands in for the operator here.
 #
-# That leaves the challenge itself covered end to end -- selected out of the
-# authorization, posted ready, finalized, fetched -- and the DNS answer the only
-# part standing in.
+# The record has to be the one the gateway reports, byte for byte, because that
+# is what Pebble parses as an RFC 8659 issue-value and matches against its own
+# issuer name and the requesting account.
+publish_persist_record() {
+    local info rdata
+    info=$(admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}') || return 1
+    # "_validation-persist.<domain>. IN TXT \"<rdata>\"" -> <rdata>
+    rdata=$(echo "$info" \
+        | tr ',' '\n' \
+        | grep -F "_validation-persist.${PERSIST_DOMAIN}. IN TXT" \
+        | sed -e 's/.*IN TXT \\"//' -e 's/\\".*//')
+    if [ -z "$rdata" ]; then
+        log_error "no validation record reported for ${PERSIST_DOMAIN}" >&2
+        return 1
+    fi
+    # stderr, not stdout: run_test captures the function's stdout and compares
+    # the whole of it against "0".
+    log_info "Publishing: _validation-persist.${PERSIST_DOMAIN} TXT ${rdata}" >&2
+    curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${PERSIST_ZONE_ID}/dns_records" \
+        -H "Authorization: Bearer ${CF_API_TOKEN}" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "type": "TXT",
+            "name": "_validation-persist.'"${PERSIST_DOMAIN}"'",
+            "content": "'"${rdata}"'",
+            "ttl": 1
+        }' > /dev/null
+}
+
+# The point of the whole exercise: a certificate is issued for a domain the
+# gateway holds no credential for, and the CA validates the published record
+# rather than being told to skip it -- Pebble runs without
+# PEBBLE_VA_ALWAYS_VALID here.
 test_persist_domain_issues_a_certificate() {
     admin_post RenewZtDomainCert \
         '{"domain": "'"${PERSIST_DOMAIN}"'", "force": true}' > /dev/null 2>&1 || true
@@ -202,11 +233,13 @@ test_persist_domain_issues_a_certificate() {
 }
 
 # The claim the whole challenge exists for: certbot reads DNS and never writes
-# it, so nothing for this domain may ever reach the provider API.
+# it. Scoped to the record certbot would have written -- the zone is not empty,
+# because the harness published the validation record above standing in for the
+# operator, and that one is the point rather than a violation.
 test_persist_domain_never_touches_the_provider() {
     local records
     records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo "[]")
-    ! echo "$records" | grep -q "${PERSIST_DOMAIN}"
+    ! echo "$records" | grep -qF "_acme-challenge.${PERSIST_DOMAIN}"
 }
 
 # ==================== Test Functions ====================
@@ -447,12 +480,14 @@ main() {
         "$(test_persist_domain_needs_no_credential; echo $?)"
     run_test "Challenge survives an edit that omits it" \
         "$(test_persist_challenge_survives_an_edit; echo $?)"
-    run_test "Certificate issues without a DNS credential" \
-        "$(test_persist_domain_issues_a_certificate; echo $?)"
-    # After issuance: every record names the ACME account, so there is nothing
-    # to report until one exists, and issuing is what creates it here.
+    # The record names the ACME account, so it can only be rendered -- and
+    # published -- once one exists. Registering is what the first order does.
     run_test "Required validation record is reported" \
         "$(test_persist_domain_reports_its_record; echo $?)"
+    run_test "Validation record publishes" \
+        "$(publish_persist_record; echo $?)"
+    run_test "Certificate issues without a DNS credential" \
+        "$(test_persist_domain_issues_a_certificate; echo $?)"
     run_test "No DNS record is ever written for it" \
         "$(test_persist_domain_never_touches_the_provider; echo $?)"
 

From 94da8fda315554f60a84c29c905edee894dac07a Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 20:00:32 -0700
Subject: [PATCH 35/37] test: keep one mock Cloudflare DNS API, in tools/

There were two. `tools/mock-cf-dns-api` was 882 lines of Flask and gunicorn
behind a published `kvin/mock-cf-dns-api:latest`, and after the gateway suite
switched to a repo-built mock, nothing used it. The survivor is 354 lines of
stdlib with no dependencies, and it is the only one that answers DNS, which
is what lets an ACME server validate rather than be told to skip.

Nothing is lost: certbot's Cloudflare client calls four endpoints -- list
zones, list records, create, delete -- and the survivor implements exactly
those. The dropped extras (PUT, single-record GET, a log buffer, a web UI)
had no caller.

Moved to tools/ because it is no longer one suite's fixture: both build it,
and the gateway suite was reaching into the other suite's directory through
four levels of `..` to find it.
---
 .../gateway/test-run/e2e/docker-compose.yml   |   3 +-
 dstack/gateway/test-run/e2e/run-e2e.sh        |   8 +
 test-suites/full-stack-compose/compose.yml    |  15 +-
 tools/README.md                               |   4 +-
 tools/mock-cf-dns-api/Dockerfile              |  28 -
 tools/mock-cf-dns-api/app.py                  | 882 ------------------
 tools/mock-cf-dns-api/docker-compose.yml      |  17 -
 tools/mock-cf-dns-api/requirements.txt        |   2 -
 .../mock-cf-dns/Dockerfile                    |   0
 .../mock-cf-dns/server.py                     |   0
 10 files changed, 23 insertions(+), 936 deletions(-)
 delete mode 100644 tools/mock-cf-dns-api/Dockerfile
 delete mode 100644 tools/mock-cf-dns-api/app.py
 delete mode 100644 tools/mock-cf-dns-api/docker-compose.yml
 delete mode 100644 tools/mock-cf-dns-api/requirements.txt
 rename {test-suites/full-stack-compose => tools}/mock-cf-dns/Dockerfile (100%)
 rename {test-suites/full-stack-compose => tools}/mock-cf-dns/server.py (100%)

diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml
index 945e98b05..d1e2fbe38 100644
--- a/dstack/gateway/test-run/e2e/docker-compose.yml
+++ b/dstack/gateway/test-run/e2e/docker-compose.yml
@@ -36,9 +36,10 @@ services:
   # Mock Cloudflare DNS API. Built from the repo rather than pulled, and the
   # same image the full-stack suite builds, so both suites answer DNS the same
   # way and neither depends on a prebuilt image nobody can rebuild.
+  # Source: tools/mock-cf-dns.
   mock-cf-dns-api:
     build:
-      context: ../../../../test-suites/full-stack-compose/mock-cf-dns
+      context: ../../../../tools/mock-cf-dns
     image: dstack-e2e-mock-cf-dns:local
     container_name: mock-cf-dns-api
     networks:
diff --git a/dstack/gateway/test-run/e2e/run-e2e.sh b/dstack/gateway/test-run/e2e/run-e2e.sh
index 835d31fb0..7a7e0cccd 100755
--- a/dstack/gateway/test-run/e2e/run-e2e.sh
+++ b/dstack/gateway/test-run/e2e/run-e2e.sh
@@ -136,6 +136,14 @@ log_info "Starting e2e test environment..."
 
 export GATEWAY_IMAGE=dstack-gateway:test
 
+# Build the mock rather than trusting whatever carries its tag. `image:` plus
+# `build:` means compose reuses a local tag if one exists, and this tag is
+# shared with the full-stack suite, so a stale copy from an older checkout
+# would be picked up silently -- and a mock that does not answer TCP fails
+# every challenge now that Pebble actually validates them.
+log_info "Building the mock DNS/Cloudflare API..."
+docker compose build mock-cf-dns-api
+
 docker compose up -d mock-cf-dns-api pebble
 log_info "Waiting for mock services to be healthy..."
 sleep 5
diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml
index f0565e6de..aff23f7a3 100644
--- a/test-suites/full-stack-compose/compose.yml
+++ b/test-suites/full-stack-compose/compose.yml
@@ -57,7 +57,7 @@ services:
 
   mock-cf-dns-api:
     build:
-      context: ./mock-cf-dns
+      context: ../../tools/mock-cf-dns
     image: dstack-e2e-mock-cf-dns:local
     networks:
       e2e-net:
@@ -76,9 +76,11 @@ services:
       retries: 10
 
   pebble:
-    # Custom Pebble build used by the existing gateway E2E suite. It supports
-    # plain HTTP ACME URLs, which keeps the host-side Gateway bootstrap simple.
-    image: ${DSTACK_E2E_PEBBLE_IMAGE:-kvin/pebble:latest}
+    # Custom Pebble build shared with the gateway E2E suite: upstream plus a
+    # `-http` flag, which keeps the host-side Gateway bootstrap simple.
+    # github.com/kvinwang/pebble branch http-flag. Pinned rather than :latest,
+    # so a rebuild of that tag cannot change what this suite validates against.
+    image: ${DSTACK_E2E_PEBBLE_IMAGE:-kvin/pebble:v2.10.1-http}
     command: ["-http", "-dnsserver", "172.31.0.10:53"]
     networks:
       e2e-net:
@@ -88,7 +90,10 @@ services:
       - "0.0.0.0:${DSTACK_E2E_PEBBLE_MGMT_PORT:-35000}:15000"
     environment:
       PEBBLE_VA_NOSLEEP: "1"
-      PEBBLE_VA_ALWAYS_VALID: "1"
+      # No PEBBLE_VA_ALWAYS_VALID. It was needed because the mock answered only
+      # UDP while Pebble queries a custom resolver over TCP, so nothing it was
+      # pointed at could ever answer; the mock now serves both and challenges
+      # are validated against the records certbot writes.
     depends_on:
       mock-cf-dns-api:
         condition: service_healthy
diff --git a/tools/README.md b/tools/README.md
index 87a2704cb..05ab156fe 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -4,7 +4,9 @@ This directory contains developer and operator utilities that do not belong to
 one runtime component or one guest-OS backend.
 
 - `add-spdx-attribution.py` updates SPDX attribution metadata.
-- `mock-cf-dns-api/` provides a local Cloudflare DNS API test double.
+- `mock-cf-dns/` is the Cloudflare DNS API test double both E2E suites build:
+  the API endpoints certbot calls, plus TXT answers on 53 over UDP and TCP so
+  an ACME server can validate against the records it writes.
 - `sca/` builds self-contained application images.
 - `dev-stack.sh` preserves the unsupported legacy all-in-one host, guest, and
   deployment helper.
diff --git a/tools/mock-cf-dns-api/Dockerfile b/tools/mock-cf-dns-api/Dockerfile
deleted file mode 100644
index 081b9625b..000000000
--- a/tools/mock-cf-dns-api/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-# SPDX-FileCopyrightText: 2024-2025 Phala Network <dstack@phala.network>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-FROM python:3.12-slim
-
-WORKDIR /app
-
-# Install dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy application
-COPY app.py .
-
-# Environment variables
-ENV PORT=8080
-ENV DEBUG=false
-
-# Expose port
-EXPOSE 8080
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
-    CMD curl -f http://localhost:8080/health || exit 1
-
-# Run with gunicorn for production
-CMD ["gunicorn", "--bind", "0.0.0.0:8080", "--workers", "2", "--threads", "4", "app:app"]
diff --git a/tools/mock-cf-dns-api/app.py b/tools/mock-cf-dns-api/app.py
deleted file mode 100644
index daf97894f..000000000
--- a/tools/mock-cf-dns-api/app.py
+++ /dev/null
@@ -1,882 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2024-2025 Phala Network <dstack@phala.network>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-"""Mock Cloudflare DNS API server.
-
-A mock server that simulates Cloudflare's DNS API for testing purposes.
-Supports the following endpoints used by certbot:
-- POST /client/v4/zones/{zone_id}/dns_records - Create DNS record
-- GET /client/v4/zones/{zone_id}/dns_records - List DNS records
-- DELETE /client/v4/zones/{zone_id}/dns_records/{record_id} - Delete DNS record
-"""
-
-import json
-import os
-import uuid
-from datetime import datetime
-from functools import wraps
-
-from flask import Flask, jsonify, render_template_string, request
-
-app = Flask(__name__)
-
-# In-memory storage for DNS records
-# Structure: {zone_id: {record_id: record_data}}
-dns_records = {}
-
-# Request/Response logs for debugging
-request_logs = []
-MAX_LOGS = 100
-
-# Valid API tokens (for testing, accept any non-empty token or use env var)
-VALID_TOKENS = (
-    os.environ.get("CF_API_TOKENS", "").split(",")
-    if os.environ.get("CF_API_TOKENS")
-    else None
-)
-
-
-def log_request(zone_id, method, path, req_data, resp_data, status_code):
-    """Log API requests for the management UI."""
-    log_entry = {
-        "timestamp": datetime.now().isoformat(),
-        "zone_id": zone_id,
-        "method": method,
-        "path": path,
-        "request": req_data,
-        "response": resp_data,
-        "status_code": status_code,
-    }
-    request_logs.insert(0, log_entry)
-    if len(request_logs) > MAX_LOGS:
-        request_logs.pop()
-
-
-def generate_record_id():
-    """Generate a Cloudflare-style record ID."""
-    return uuid.uuid4().hex[:32]
-
-
-def get_current_time():
-    """Get current time in Cloudflare format."""
-    return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.000000Z")
-
-
-def verify_auth(f):
-    """Verify Bearer token authentication."""
-
-    @wraps(f)
-    def decorated(*args, **kwargs):
-        auth_header = request.headers.get("Authorization", "")
-        if not auth_header.startswith("Bearer "):
-            return jsonify(
-                {
-                    "success": False,
-                    "errors": [{"code": 10000, "message": "Authentication error"}],
-                    "messages": [],
-                    "result": None,
-                }
-            ), 401
-
-        token = auth_header[7:]  # Remove "Bearer " prefix
-
-        # If VALID_TOKENS is set, validate against it; otherwise accept any token
-        if VALID_TOKENS and token not in VALID_TOKENS:
-            return jsonify(
-                {
-                    "success": False,
-                    "errors": [{"code": 10000, "message": "Invalid API token"}],
-                    "messages": [],
-                    "result": None,
-                }
-            ), 403
-
-        return f(*args, **kwargs)
-
-    return decorated
-
-
-def cf_response(result, success=True, errors=None, messages=None):
-    """Create a Cloudflare-style API response."""
-    return {
-        "success": success,
-        "errors": errors or [],
-        "messages": messages or [],
-        "result": result,
-    }
-
-
-def cf_error(message, code=1000):
-    """Create a Cloudflare-style error response."""
-    return cf_response(None, success=False, errors=[{"code": code, "message": message}])
-
-
-# ==================== DNS Record Endpoints ====================
-
-
-@app.route("/client/v4/zones/<zone_id>/dns_records", methods=["POST"])
-@verify_auth
-def create_dns_record(zone_id):
-    """Create a new DNS record."""
-    data = request.get_json()
-
-    if not data:
-        resp = cf_error("Invalid request body")
-        log_request(zone_id, "POST", f"/zones/{zone_id}/dns_records", None, resp, 400)
-        return jsonify(resp), 400
-
-    record_type = data.get("type")
-    name = data.get("name")
-
-    if not record_type or not name:
-        resp = cf_error("Missing required fields: type, name")
-        log_request(zone_id, "POST", f"/zones/{zone_id}/dns_records", data, resp, 400)
-        return jsonify(resp), 400
-
-    # Initialize zone if not exists
-    if zone_id not in dns_records:
-        dns_records[zone_id] = {}
-
-    record_id = generate_record_id()
-    now = get_current_time()
-
-    # Build record based on type
-    record = {
-        "id": record_id,
-        "zone_id": zone_id,
-        "zone_name": f"zone-{zone_id[:8]}.example.com",
-        "name": name,
-        "type": record_type,
-        "ttl": data.get("ttl", 1),
-        "proxied": data.get("proxied", False),
-        "proxiable": False,
-        "locked": False,
-        "created_on": now,
-        "modified_on": now,
-        "meta": {
-            "auto_added": False,
-            "managed_by_apps": False,
-            "managed_by_argo_tunnel": False,
-        },
-    }
-
-    # Handle different record types
-    if record_type == "TXT":
-        record["content"] = data.get("content", "")
-    elif record_type == "CAA":
-        caa_data = data.get("data", {})
-        record["data"] = caa_data
-        # Format content as Cloudflare does
-        flags = caa_data.get("flags", 0)
-        tag = caa_data.get("tag", "")
-        value = caa_data.get("value", "")
-        record["content"] = f'{flags} {tag} "{value}"'
-    elif record_type == "A":
-        record["content"] = data.get("content", "")
-    elif record_type == "AAAA":
-        record["content"] = data.get("content", "")
-    elif record_type == "CNAME":
-        record["content"] = data.get("content", "")
-    else:
-        record["content"] = data.get("content", "")
-
-    dns_records[zone_id][record_id] = record
-
-    resp = cf_response(record)
-    log_request(zone_id, "POST", f"/zones/{zone_id}/dns_records", data, resp, 200)
-
-    print(
-        f"[CREATE] Zone: {zone_id}, Record: {record_id}, Type: {record_type}, Name: {name}"
-    )
-
-    return jsonify(resp), 200
-
-
-@app.route("/client/v4/zones/<zone_id>/dns_records", methods=["GET"])
-@verify_auth
-def list_dns_records(zone_id):
-    """List DNS records for a zone."""
-    zone_records = dns_records.get(zone_id, {})
-    records_list = list(zone_records.values())
-
-    # Filter by type if specified
-    record_type = request.args.get("type")
-    if record_type:
-        records_list = [r for r in records_list if r["type"] == record_type]
-
-    # Filter by name if specified
-    name = request.args.get("name")
-    if name:
-        records_list = [r for r in records_list if r["name"] == name]
-
-    # Get pagination params
-    page = int(request.args.get("page", 1))
-    per_page = int(request.args.get("per_page", 100))
-
-    # Pagination
-    total_count = len(records_list)
-    total_pages = max(1, (total_count + per_page - 1) // per_page)
-    start_idx = (page - 1) * per_page
-    end_idx = start_idx + per_page
-    page_records = records_list[start_idx:end_idx]
-
-    resp = {
-        "success": True,
-        "errors": [],
-        "messages": [],
-        "result": page_records,
-        "result_info": {
-            "page": page,
-            "per_page": per_page,
-            "count": len(page_records),
-            "total_count": total_count,
-            "total_pages": total_pages,
-        },
-    }
-    log_request(
-        zone_id, "GET", f"/zones/{zone_id}/dns_records", dict(request.args), resp, 200
-    )
-
-    return jsonify(resp), 200
-
-
-@app.route("/client/v4/zones/<zone_id>/dns_records/<record_id>", methods=["GET"])
-@verify_auth
-def get_dns_record(zone_id, record_id):
-    """Get a specific DNS record."""
-    zone_records = dns_records.get(zone_id, {})
-    record = zone_records.get(record_id)
-
-    if not record:
-        resp = cf_error("Record not found", 81044)
-        log_request(
-            zone_id, "GET", f"/zones/{zone_id}/dns_records/{record_id}", None, resp, 404
-        )
-        return jsonify(resp), 404
-
-    resp = cf_response(record)
-    log_request(
-        zone_id, "GET", f"/zones/{zone_id}/dns_records/{record_id}", None, resp, 200
-    )
-
-    return jsonify(resp), 200
-
-
-@app.route("/client/v4/zones/<zone_id>/dns_records/<record_id>", methods=["PUT"])
-@verify_auth
-def update_dns_record(zone_id, record_id):
-    """Update a DNS record."""
-    zone_records = dns_records.get(zone_id, {})
-    record = zone_records.get(record_id)
-
-    if not record:
-        resp = cf_error("Record not found", 81044)
-        log_request(
-            zone_id, "PUT", f"/zones/{zone_id}/dns_records/{record_id}", None, resp, 404
-        )
-        return jsonify(resp), 404
-
-    data = request.get_json()
-    if not data:
-        resp = cf_error("Invalid request body")
-        log_request(
-            zone_id, "PUT", f"/zones/{zone_id}/dns_records/{record_id}", None, resp, 400
-        )
-        return jsonify(resp), 400
-
-    # Update allowed fields
-    for field in ["name", "type", "content", "ttl", "proxied", "data"]:
-        if field in data:
-            record[field] = data[field]
-
-    record["modified_on"] = get_current_time()
-
-    resp = cf_response(record)
-    log_request(
-        zone_id, "PUT", f"/zones/{zone_id}/dns_records/{record_id}", data, resp, 200
-    )
-
-    print(f"[UPDATE] Zone: {zone_id}, Record: {record_id}")
-
-    return jsonify(resp), 200
-
-
-@app.route("/client/v4/zones/<zone_id>/dns_records/<record_id>", methods=["DELETE"])
-@verify_auth
-def delete_dns_record(zone_id, record_id):
-    """Delete a DNS record."""
-    zone_records = dns_records.get(zone_id, {})
-
-    if record_id not in zone_records:
-        resp = cf_error("Record not found", 81044)
-        log_request(
-            zone_id,
-            "DELETE",
-            f"/zones/{zone_id}/dns_records/{record_id}",
-            None,
-            resp,
-            404,
-        )
-        return jsonify(resp), 404
-
-    del zone_records[record_id]
-
-    resp = cf_response({"id": record_id})
-    log_request(
-        zone_id, "DELETE", f"/zones/{zone_id}/dns_records/{record_id}", None, resp, 200
-    )
-
-    print(f"[DELETE] Zone: {zone_id}, Record: {record_id}")
-
-    return jsonify(resp), 200
-
-
-# ==================== Zone Endpoints ====================
-
-# Pre-configured zones for testing
-# Can be configured via MOCK_ZONES environment variable (JSON format)
-# Example: MOCK_ZONES='[{"id":"zone123","name":"example.com"},{"id":"zone456","name":"test.local"}]'
-DEFAULT_ZONES = [
-    {"id": "mock-zone-test-local", "name": "test.local"},
-    {"id": "mock-zone-example-com", "name": "example.com"},
-    {"id": "mock-zone-test0-local", "name": "test0.local"},
-    {"id": "mock-zone-test1-local", "name": "test1.local"},
-    {"id": "mock-zone-test2-local", "name": "test2.local"},
-]
-
-
-def get_configured_zones():
-    """Get zones from environment or use defaults."""
-    zones_json = os.environ.get("MOCK_ZONES")
-    if zones_json:
-        try:
-            return json.loads(zones_json)
-        except json.JSONDecodeError:
-            print("Warning: Invalid MOCK_ZONES JSON, using defaults")
-    return DEFAULT_ZONES
-
-
-@app.route("/client/v4/zones", methods=["GET"])
-@verify_auth
-def list_zones():
-    """List all zones (paginated)."""
-    page = int(request.args.get("page", 1))
-    per_page = int(request.args.get("per_page", 50))
-    name_filter = request.args.get("name")
-
-    zones = get_configured_zones()
-
-    # Filter by name if specified
-    if name_filter:
-        zones = [z for z in zones if z["name"] == name_filter]
-
-    # Build full zone objects
-    full_zones = []
-    for z in zones:
-        full_zones.append(
-            {
-                "id": z["id"],
-                "name": z["name"],
-                "status": "active",
-                "paused": False,
-                "type": "full",
-                "development_mode": 0,
-                "name_servers": ["ns1.mock-cloudflare.com", "ns2.mock-cloudflare.com"],
-                "created_on": "2024-01-01T00:00:00.000000Z",
-                "modified_on": get_current_time(),
-            }
-        )
-
-    # Pagination
-    total_count = len(full_zones)
-    total_pages = max(1, (total_count + per_page - 1) // per_page)
-    start_idx = (page - 1) * per_page
-    end_idx = start_idx + per_page
-    page_zones = full_zones[start_idx:end_idx]
-
-    result = {
-        "success": True,
-        "errors": [],
-        "messages": [],
-        "result": page_zones,
-        "result_info": {
-            "page": page,
-            "per_page": per_page,
-            "count": len(page_zones),
-            "total_count": total_count,
-            "total_pages": total_pages,
-        },
-    }
-
-    log_request("*", "GET", "/zones", dict(request.args), result, 200)
-    print(
-        f"[LIST ZONES] page={page}, per_page={per_page}, count={len(page_zones)}, total={total_count}"
-    )
-
-    return jsonify(result), 200
-
-
-@app.route("/client/v4/zones/<zone_id>", methods=["GET"])
-@verify_auth
-def get_zone(zone_id):
-    """Get zone details (mock)."""
-    # Try to find zone in configured zones
-    zones = get_configured_zones()
-    zone_info = next((z for z in zones if z["id"] == zone_id), None)
-
-    if zone_info:
-        zone_name = zone_info["name"]
-    else:
-        # Fallback for unknown zone IDs
-        zone_name = f"zone-{zone_id[:8]}.example.com"
-
-    zone = {
-        "id": zone_id,
-        "name": zone_name,
-        "status": "active",
-        "paused": False,
-        "type": "full",
-        "development_mode": 0,
-        "name_servers": ["ns1.mock-cloudflare.com", "ns2.mock-cloudflare.com"],
-        "created_on": "2024-01-01T00:00:00.000000Z",
-        "modified_on": get_current_time(),
-    }
-
-    resp = cf_response(zone)
-    log_request(zone_id, "GET", f"/zones/{zone_id}", None, resp, 200)
-
-    return jsonify(resp), 200
-
-
-# ==================== Management UI ====================
-
-MANAGEMENT_HTML = """
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Mock Cloudflare DNS API - Management</title>
-    <style>
-        * {
-            box-sizing: border-box;
-            margin: 0;
-            padding: 0;
-        }
-        body {
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
-            background: #1a1a2e;
-            color: #eaeaea;
-            min-height: 100vh;
-        }
-        .container {
-            max-width: 1400px;
-            margin: 0 auto;
-            padding: 20px;
-        }
-        header {
-            background: linear-gradient(135deg, #f6821f 0%, #faad3f 100%);
-            padding: 20px;
-            margin-bottom: 20px;
-            border-radius: 8px;
-        }
-        header h1 {
-            color: #1a1a2e;
-            font-size: 1.5em;
-        }
-        header p {
-            color: #333;
-            margin-top: 5px;
-        }
-        .stats {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
-            gap: 15px;
-            margin-bottom: 20px;
-        }
-        .stat-card {
-            background: #25274d;
-            padding: 20px;
-            border-radius: 8px;
-            text-align: center;
-        }
-        .stat-card h3 {
-            font-size: 2em;
-            color: #f6821f;
-        }
-        .stat-card p {
-            color: #888;
-            margin-top: 5px;
-        }
-        .section {
-            background: #25274d;
-            border-radius: 8px;
-            margin-bottom: 20px;
-            overflow: hidden;
-        }
-        .section-header {
-            background: #2e3156;
-            padding: 15px 20px;
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-        }
-        .section-header h2 {
-            font-size: 1.1em;
-        }
-        .section-content {
-            padding: 20px;
-        }
-        table {
-            width: 100%;
-            border-collapse: collapse;
-        }
-        th, td {
-            padding: 12px;
-            text-align: left;
-            border-bottom: 1px solid #3a3d6b;
-        }
-        th {
-            background: #2e3156;
-            font-weight: 600;
-        }
-        tr:hover {
-            background: #2e3156;
-        }
-        .record-type {
-            display: inline-block;
-            padding: 2px 8px;
-            border-radius: 4px;
-            font-size: 0.85em;
-            font-weight: 600;
-        }
-        .record-type.TXT { background: #3b82f6; }
-        .record-type.CAA { background: #8b5cf6; }
-        .record-type.A { background: #10b981; }
-        .record-type.AAAA { background: #06b6d4; }
-        .record-type.CNAME { background: #f59e0b; }
-        .method {
-            display: inline-block;
-            padding: 2px 8px;
-            border-radius: 4px;
-            font-size: 0.85em;
-            font-weight: 600;
-        }
-        .method.GET { background: #10b981; }
-        .method.POST { background: #3b82f6; }
-        .method.PUT { background: #f59e0b; }
-        .method.DELETE { background: #ef4444; }
-        .btn {
-            background: #f6821f;
-            color: #1a1a2e;
-            border: none;
-            padding: 8px 16px;
-            border-radius: 4px;
-            cursor: pointer;
-            font-weight: 600;
-        }
-        .btn:hover {
-            background: #faad3f;
-        }
-        .btn-danger {
-            background: #ef4444;
-            color: white;
-        }
-        .btn-danger:hover {
-            background: #dc2626;
-        }
-        .content-preview {
-            max-width: 300px;
-            overflow: hidden;
-            text-overflow: ellipsis;
-            white-space: nowrap;
-            font-family: monospace;
-            font-size: 0.9em;
-        }
-        .log-entry {
-            background: #1a1a2e;
-            border-radius: 4px;
-            padding: 10px;
-            margin-bottom: 10px;
-            font-family: monospace;
-            font-size: 0.85em;
-        }
-        .log-time {
-            color: #888;
-        }
-        .empty-state {
-            text-align: center;
-            padding: 40px;
-            color: #666;
-        }
-        .tabs {
-            display: flex;
-            border-bottom: 1px solid #3a3d6b;
-            margin-bottom: 15px;
-        }
-        .tab {
-            padding: 10px 20px;
-            cursor: pointer;
-            border-bottom: 2px solid transparent;
-        }
-        .tab.active {
-            border-bottom-color: #f6821f;
-            color: #f6821f;
-        }
-        .tab-content {
-            display: none;
-        }
-        .tab-content.active {
-            display: block;
-        }
-        pre {
-            background: #1a1a2e;
-            padding: 10px;
-            border-radius: 4px;
-            overflow-x: auto;
-            font-size: 0.85em;
-        }
-        .refresh-btn {
-            position: fixed;
-            bottom: 20px;
-            right: 20px;
-            width: 50px;
-            height: 50px;
-            border-radius: 50%;
-            font-size: 1.5em;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-        }
-    </style>
-</head>
-<body>
-    <div class="container">
-        <header>
-            <h1>Mock Cloudflare DNS API</h1>
-            <p>Testing server for ACME DNS-01 challenges</p>
-        </header>
-
-        <div class="stats">
-            <div class="stat-card">
-                <h3 id="zone-count">{{ zone_count }}</h3>
-                <p>Zones</p>
-            </div>
-            <div class="stat-card">
-                <h3 id="record-count">{{ record_count }}</h3>
-                <p>DNS Records</p>
-            </div>
-            <div class="stat-card">
-                <h3 id="request-count">{{ request_count }}</h3>
-                <p>API Requests</p>
-            </div>
-        </div>
-
-        <div class="section">
-            <div class="section-header">
-                <h2>DNS Records</h2>
-                <button class="btn btn-danger" onclick="clearAllRecords()">Clear All</button>
-            </div>
-            <div class="section-content">
-                {% if records %}
-                <table>
-                    <thead>
-                        <tr>
-                            <th>Zone ID</th>
-                            <th>Type</th>
-                            <th>Name</th>
-                            <th>Content</th>
-                            <th>Created</th>
-                            <th>Actions</th>
-                        </tr>
-                    </thead>
-                    <tbody>
-                        {% for record in records %}
-                        <tr>
-                            <td><code>{{ record.zone_id[:12] }}...</code></td>
-                            <td><span class="record-type {{ record.type }}">{{ record.type }}</span></td>
-                            <td>{{ record.name }}</td>
-                            <td class="content-preview" title="{{ record.content }}">{{ record.content }}</td>
-                            <td>{{ record.created_on[:19] }}</td>
-                            <td>
-                                <button class="btn btn-danger" onclick="deleteRecord('{{ record.zone_id }}', '{{ record.id }}')">Delete</button>
-                            </td>
-                        </tr>
-                        {% endfor %}
-                    </tbody>
-                </table>
-                {% else %}
-                <div class="empty-state">
-                    <p>No DNS records yet. Records created via API will appear here.</p>
-                </div>
-                {% endif %}
-            </div>
-        </div>
-
-        <div class="section">
-            <div class="section-header">
-                <h2>Recent API Requests</h2>
-                <button class="btn" onclick="clearLogs()">Clear Logs</button>
-            </div>
-            <div class="section-content">
-                {% if logs %}
-                {% for log in logs %}
-                <div class="log-entry">
-                    <span class="log-time">{{ log.timestamp }}</span>
-                    <span class="method {{ log.method }}">{{ log.method }}</span>
-                    <span>{{ log.path }}</span>
-                    <span style="color: {% if log.status_code == 200 %}#10b981{% else %}#ef4444{% endif %}">
-                        ({{ log.status_code }})
-                    </span>
-                    {% if log.request %}
-                    <details>
-                        <summary>Request/Response</summary>
-                        <pre>Request: {{ log.request | tojson(indent=2) }}</pre>
-                        <pre>Response: {{ log.response | tojson(indent=2) }}</pre>
-                    </details>
-                    {% endif %}
-                </div>
-                {% endfor %}
-                {% else %}
-                <div class="empty-state">
-                    <p>No API requests yet.</p>
-                </div>
-                {% endif %}
-            </div>
-        </div>
-
-        <div class="section">
-            <div class="section-header">
-                <h2>API Usage</h2>
-            </div>
-            <div class="section-content">
-                <p style="margin-bottom: 15px;">Base URL: <code id="base-url"></code></p>
-                <pre id="api-examples"></pre>
-            </div>
-        </div>
-    </div>
-
-    <button class="btn refresh-btn" onclick="location.reload()">&#x21bb;</button>
-
-    <script>
-        function deleteRecord(zoneId, recordId) {
-            if (!confirm('Delete this record?')) return;
-            fetch(`/api/records/${zoneId}/${recordId}`, { method: 'DELETE' })
-                .then(() => location.reload());
-        }
-
-        function clearAllRecords() {
-            if (!confirm('Clear all DNS records?')) return;
-            fetch('/api/records', { method: 'DELETE' })
-                .then(() => location.reload());
-        }
-
-        function clearLogs() {
-            fetch('/api/logs', { method: 'DELETE' })
-                .then(() => location.reload());
-        }
-
-        // Auto-refresh every 5 seconds
-        setTimeout(() => location.reload(), 5000);
-
-        // Set API base URL from browser location
-        const baseUrl = window.location.origin + '/client/v4';
-        document.getElementById('base-url').textContent = baseUrl;
-        document.getElementById('api-examples').textContent = `# Create TXT record
-curl -X POST "${baseUrl}/zones/YOUR_ZONE_ID/dns_records" \\
-     -H "Authorization: Bearer YOUR_API_TOKEN" \\
-     -H "Content-Type: application/json" \\
-     --data '{"type":"TXT","name":"_acme-challenge.example.com","content":"test-value"}'
-
-# List records
-curl -X GET "${baseUrl}/zones/YOUR_ZONE_ID/dns_records" \\
-     -H "Authorization: Bearer YOUR_API_TOKEN"
-
-# Delete record
-curl -X DELETE "${baseUrl}/zones/YOUR_ZONE_ID/dns_records/RECORD_ID" \\
-     -H "Authorization: Bearer YOUR_API_TOKEN"`;
-    </script>
-</body>
-</html>
-"""
-
-
-@app.route("/")
-def management_ui():
-    """Render the management UI."""
-    all_records = []
-    for zone_id, records in dns_records.items():
-        all_records.extend(records.values())
-
-    # Sort by created time, newest first
-    all_records.sort(key=lambda r: r.get("created_on", ""), reverse=True)
-
-    return render_template_string(
-        MANAGEMENT_HTML,
-        zone_count=len(dns_records),
-        record_count=sum(len(r) for r in dns_records.values()),
-        request_count=len(request_logs),
-        records=all_records,
-        logs=request_logs[:20],
-        port=os.environ.get("PORT", 8080),
-    )
-
-
-# ==================== Management API ====================
-
-
-@app.route("/api/records", methods=["DELETE"])
-def clear_all_records():
-    """Clear all DNS records."""
-    dns_records.clear()
-    return jsonify({"success": True})
-
-
-@app.route("/api/records/<zone_id>/<record_id>", methods=["DELETE"])
-def delete_record_ui(zone_id, record_id):
-    """Delete a specific record from UI."""
-    if zone_id in dns_records and record_id in dns_records[zone_id]:
-        del dns_records[zone_id][record_id]
-    return jsonify({"success": True})
-
-
-@app.route("/api/logs", methods=["DELETE"])
-def clear_logs():
-    """Clear request logs."""
-    request_logs.clear()
-    return jsonify({"success": True})
-
-
-@app.route("/api/records", methods=["GET"])
-def get_all_records():
-    """Get all records as JSON."""
-    all_records = []
-    for zone_id, records in dns_records.items():
-        all_records.extend(records.values())
-    return jsonify(all_records)
-
-
-@app.route("/health")
-def health():
-    """Health check endpoint."""
-    return jsonify(
-        {"status": "healthy", "records": sum(len(r) for r in dns_records.values())}
-    )
-
-
-if __name__ == "__main__":
-    port = int(os.environ.get("PORT", 8080))
-    debug = os.environ.get("DEBUG", "false").lower() == "true"
-
-    print(f"""
-    ╔═══════════════════════════════════════════════════════════════╗
-    ║         Mock Cloudflare DNS API Server                        ║
-    ╠═══════════════════════════════════════════════════════════════╣
-    ║  Management UI:  http://localhost:{port}/                         ║
-    ║  API Base URL:   http://localhost:{port}/client/v4                ║
-    ╚═══════════════════════════════════════════════════════════════╝
-    """)
-
-    app.run(host="0.0.0.0", port=port, debug=debug)
diff --git a/tools/mock-cf-dns-api/docker-compose.yml b/tools/mock-cf-dns-api/docker-compose.yml
deleted file mode 100644
index 951bd68b7..000000000
--- a/tools/mock-cf-dns-api/docker-compose.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-# SPDX-FileCopyrightText: 2024-2025 Phala Network <dstack@phala.network>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-version: "3.8"
-
-services:
-  mock-cf-dns-api:
-    image: kvin/mock-cf-dns-api:latest
-    ports:
-      - "8080:8080"
-    environment:
-      - PORT=8080
-      - DEBUG=false
-      # Optional: comma-separated list of valid API tokens
-      # - CF_API_TOKENS=token1,token2
-    restart: unless-stopped
diff --git a/tools/mock-cf-dns-api/requirements.txt b/tools/mock-cf-dns-api/requirements.txt
deleted file mode 100644
index a36229c7e..000000000
--- a/tools/mock-cf-dns-api/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-flask>=3.0.0
-gunicorn>=21.0.0
diff --git a/test-suites/full-stack-compose/mock-cf-dns/Dockerfile b/tools/mock-cf-dns/Dockerfile
similarity index 100%
rename from test-suites/full-stack-compose/mock-cf-dns/Dockerfile
rename to tools/mock-cf-dns/Dockerfile
diff --git a/test-suites/full-stack-compose/mock-cf-dns/server.py b/tools/mock-cf-dns/server.py
similarity index 100%
rename from test-suites/full-stack-compose/mock-cf-dns/server.py
rename to tools/mock-cf-dns/server.py

From 4c53736f896a0e37585d1c5d4df93dd0bf378551 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 20:26:53 -0700
Subject: [PATCH 36/37] test: let the mock's bind address be narrowed

CodeQL flags the DNS listeners for binding every interface. In a container
that is the only address that answers -- peers reach this by an address on a
private compose network that the process cannot know in advance -- so the
default stays, but it is now one documented function rather than a literal
repeated at each listener, and MOCK_CF_BIND narrows it for anyone running the
mock outside a container, where a test double that trusts every caller should
not be on every interface.
---
 tools/mock-cf-dns/server.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/tools/mock-cf-dns/server.py b/tools/mock-cf-dns/server.py
index 3ba0c0766..9208f9adb 100755
--- a/tools/mock-cf-dns/server.py
+++ b/tools/mock-cf-dns/server.py
@@ -268,6 +268,19 @@ def dns_response(packet: bytes) -> bytes:
     return header + body
 
 
+def bind_host() -> str:
+    """Address the mock listens on, for both the HTTP API and DNS.
+
+    Defaults to every interface. This runs as a container on a private compose
+    network and its peers -- certbot writing records, and the ACME server
+    resolving them -- reach it by an address the process cannot know in
+    advance, so narrowing the default would answer nobody. Set MOCK_CF_BIND to
+    something specific when running it outside a container, where the default
+    would expose a test double that trusts every caller.
+    """
+    return os.environ.get("MOCK_CF_BIND", "0.0.0.0")
+
+
 def _debug(message: str) -> None:
     if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}:
         print(message, flush=True)
@@ -277,7 +290,7 @@ def dns_loop() -> None:
     """Serve mock DNS responses over UDP."""
     sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    sock.bind(("0.0.0.0", 53))
+    sock.bind((bind_host(), 53))
     while True:
         packet, addr = sock.recvfrom(4096)
         try:
@@ -332,7 +345,7 @@ def dns_tcp_loop() -> None:
     """
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    sock.bind(("0.0.0.0", 53))
+    sock.bind((bind_host(), 53))
     sock.listen(16)
     while True:
         conn, addr = sock.accept()
@@ -346,8 +359,12 @@ def main() -> None:
     threading.Thread(target=dns_loop, daemon=True).start()
     threading.Thread(target=dns_tcp_loop, daemon=True).start()
     port = int(os.environ.get("PORT", "8080"))
-    print(f"mock CF API on :{port}; DNS on :53 (udp+tcp); zones={_zones()}", flush=True)
-    ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever()
+    host = bind_host()
+    print(
+        f"mock CF API on {host}:{port}; DNS on {host}:53 (udp+tcp); zones={_zones()}",
+        flush=True,
+    )
+    ThreadingHTTPServer((host, port), Handler).serve_forever()
 
 
 if __name__ == "__main__":

From 5ed087c6356ee268c28c4baef4370303452b6ef5 Mon Sep 17 00:00:00 2001
From: Kevin Wang <wy721@qq.com>
Date: Tue, 25 Aug 2026 22:38:16 -0700
Subject: [PATCH 37/37] test(gateway): pin what the CA refuses, not only what
 it accepts

The phase proved a certificate issues from a published record. It could not
prove the record had to be that record: the only failing case it exercised was
the domain having no record at all.

Three cases the CA now decides, because it looks the record up rather than
being run with PEBBLE_VA_ALWAYS_VALID:

- A record correct in every respect except `policy=wildcard` does not answer
  the order. The gateway only ever orders `*.{domain}`, so every authorization
  it answers is a wildcard one, and the draft has a CA accept those only from
  a record carrying the parameter.
- Adding that one parameter, and nothing else, issues. Without this the case
  above could pass for any reason at all.
- A record naming a different account does not authorize. If it did, the
  record would authorize anybody who found it and the challenge would prove
  nothing.

Plus the two gateway operations that change shape for a domain the gateway
cannot write: `SetCaa` skips it rather than failing the whole run or writing
to a provider it has no credential for, and `RotateAcmeCredentials` returns
the records to republish, naming the account it just registered -- the old
records name the account it replaced, and nothing else reports them.

Each refusal gets its own domain. `DeleteZtDomain` keeps the certificate on
purpose, so a domain that has already issued reports that certificate after
being re-added, and a refusal would read as a pass. The assertions are
inverted rather than matched against an error string, because what is being
claimed is that no certificate appears.

The CAA assertion splits the response into records before matching, rather
than looking for `"type"` and `"name"` adjacent in one pattern: the mock
serializes with sorted keys, so `"type"` never follows `"name"`, and a pattern
spanning the two can only fail to match -- which in a negative assertion is a
test that passes whatever the gateway did.
---
 .../gateway/test-run/e2e/docker-compose.yml   |   2 +-
 dstack/gateway/test-run/e2e/test.sh           | 163 ++++++++++++++++++
 2 files changed, 164 insertions(+), 1 deletion(-)

diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml
index d1e2fbe38..bfe8dc378 100644
--- a/dstack/gateway/test-run/e2e/docker-compose.yml
+++ b/dstack/gateway/test-run/e2e/docker-compose.yml
@@ -51,7 +51,7 @@ services:
       - PORT=8080
       - DEBUG=true
       # The zones certbot writes into and Pebble reads back out of.
-      - MOCK_CF_ZONES=test0.local,test1.local,test2.local,persist0.local
+      - MOCK_CF_ZONES=test0.local,test1.local,test2.local,persist0.local,persist1.local,persist2.local
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"]
       interval: 5s
diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh
index 69b69b19d..04962c93e 100755
--- a/dstack/gateway/test-run/e2e/test.sh
+++ b/dstack/gateway/test-run/e2e/test.sh
@@ -41,6 +41,19 @@ PERSIST_DOMAIN="persist0.local"
 # Mock zone ID, as server.py derives it from the zone name.
 PERSIST_ZONE_ID="zone-persist0-local"
 
+# A second dns-persist-01 domain, used for the cases that must NOT issue. Kept
+# apart from PERSIST_DOMAIN so a deliberately unanswerable order cannot be
+# confused with the working one.
+PERSIST_NEG_DOMAIN="persist1.local"
+PERSIST_NEG_ZONE_ID="zone-persist1-local"
+
+# A third domain, for the second refusal. Deleting a ZT domain deliberately
+# keeps its certificate ("kept for historical purposes"), so reusing a domain
+# that has already issued would report the old certificate and read as a
+# refusal that did not happen.
+PERSIST_NEG2_DOMAIN="persist2.local"
+PERSIST_NEG2_ZONE_ID="zone-persist2-local"
+
 # Pebble answers to this name, not to letsencrypt.org, and a validation record
 # naming a CA the challenge does not list is ignored.
 PERSIST_ISSUER="pebble.letsencrypt.org"
@@ -232,6 +245,142 @@ test_persist_domain_issues_a_certificate() {
     return 1
 }
 
+# ---- Cases that must NOT issue -------------------------------------------
+#
+# These are the reason the mock answers DNS at all. With PEBBLE_VA_ALWAYS_VALID
+# the CA accepts any challenge, so a wrong record and a right one are
+# indistinguishable and the record's grammar is pinned only by unit tests
+# written against a reading of the CA. Here the CA reads it.
+
+# Publish one TXT record verbatim at the negative domain's validation name,
+# replacing whatever is there.
+publish_neg_record() {
+    local domain="$1" zone="$2" rdata="$3"
+    local records id
+    records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo '{"records":[]}')
+    for id in $(echo "$records" \
+        | tr '{' '\n' \
+        | grep -F "_validation-persist.${domain}" \
+        | sed -e 's/.*"id": "//' -e 's/".*//'); do
+        curl -sf -X DELETE \
+            "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records/${id}" \
+            -H "Authorization: Bearer ${CF_API_TOKEN}" > /dev/null 2>&1 || true
+    done
+    curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records" \
+        -H "Authorization: Bearer ${CF_API_TOKEN}" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "type": "TXT",
+            "name": "_validation-persist.'"${domain}"'",
+            "content": "'"${rdata}"'",
+            "ttl": 1
+        }' > /dev/null
+}
+
+# The account URI the cluster actually holds, read off the domain the gateway
+# already renders records for.
+persist_account_uri() {
+    admin_post GetZtDomain '{"domain": "'"${PERSIST_DOMAIN}"'"}' \
+        | tr ',' '\n' \
+        | grep -F "accounturi=" \
+        | head -1 \
+        | sed -e 's/.*accounturi=//' -e 's/[";\\].*//'
+}
+
+# Try to issue for the negative domain and report whether a certificate
+# appeared. Deliberately short: the interesting outcome is "no certificate",
+# and an order that is going to be refused says so in one pass.
+neg_domain_issued() {
+    local domain="$1"
+    admin_post RenewZtDomainCert \
+        '{"domain": "'"${domain}"'", "force": true}' > /dev/null 2>&1 || true
+    local i=0
+    while [ $i -lt 45 ]; do
+        if admin_post GetZtDomain '{"domain": "'"${domain}"'"}' \
+            | grep -q '"has_cert":true'; then
+            return 0
+        fi
+        sleep 2
+        i=$((i + 1))
+    done
+    return 1
+}
+
+# The gateway only ever orders `*.{domain}`, so every authorization it answers
+# is a wildcard one -- and the draft has a CA accept a wildcard authorization
+# only from a record carrying `policy=wildcard`. A record that is correct in
+# every other respect must therefore not be enough.
+test_persist_record_without_wildcard_policy_is_refused() {
+    admin_post DeleteZtDomain '{"domain": "'"${PERSIST_NEG_DOMAIN}"'"}' > /dev/null 2>&1 || true
+    admin_post AddZtDomain \
+        '{"domain": "'"${PERSIST_NEG_DOMAIN}"'", "port": 443, "challenge": "dns-persist-01"}' \
+        > /dev/null || return 1
+    local uri
+    uri=$(persist_account_uri) || return 1
+    [ -n "$uri" ] || return 1
+    publish_neg_record "${PERSIST_NEG_DOMAIN}" "${PERSIST_NEG_ZONE_ID}" \
+        "${PERSIST_ISSUER}; accounturi=${uri}" || return 1
+    # Inverted: issuing here would mean the policy parameter is not enforced.
+    ! neg_domain_issued "${PERSIST_NEG_DOMAIN}"
+}
+
+# Adding the one parameter that was missing, and nothing else, has to flip the
+# outcome. That is what makes the previous case a statement about `policy`
+# rather than about some unrelated breakage.
+test_persist_record_with_wildcard_policy_issues() {
+    local uri
+    uri=$(persist_account_uri) || return 1
+    publish_neg_record "${PERSIST_NEG_DOMAIN}" "${PERSIST_NEG_ZONE_ID}" \
+        "${PERSIST_ISSUER}; accounturi=${uri}; policy=wildcard" || return 1
+    neg_domain_issued "${PERSIST_NEG_DOMAIN}"
+}
+
+# The account URI in the record is the whole authorization. A record naming a
+# different account must not let this one issue -- otherwise publishing a record
+# authorizes anybody, and the challenge proves nothing.
+test_persist_record_for_another_account_is_refused() {
+    admin_post DeleteZtDomain '{"domain": "'"${PERSIST_NEG2_DOMAIN}"'"}' > /dev/null 2>&1 || true
+    admin_post AddZtDomain \
+        '{"domain": "'"${PERSIST_NEG2_DOMAIN}"'", "port": 443, "challenge": "dns-persist-01"}' \
+        > /dev/null || return 1
+    publish_neg_record "${PERSIST_NEG2_DOMAIN}" "${PERSIST_NEG2_ZONE_ID}" \
+        "${PERSIST_ISSUER}; accounturi=${ACME_URL%/dir}/my-account/0; policy=wildcard" || return 1
+    ! neg_domain_issued "${PERSIST_NEG2_DOMAIN}"
+}
+
+# ---- Gateway operations that change shape for such a domain ---------------
+
+# SetCaa reconciles CAA through the DNS provider, which the gateway has no
+# credential for here. It must skip the domain rather than fail the call, and
+# must not write anything.
+test_set_caa_skips_a_persist_domain() {
+    admin_post SetCaa '{}' > /dev/null 2>&1 || true
+    local records
+    records=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null || echo "[]")
+    # Split into records first and require both fields within one of them. The
+    # mock serializes with sorted keys, so "type" never follows "name" and a
+    # single fixed pattern spanning the two can only ever fail to match -- which
+    # in a negative assertion is a test that passes no matter what.
+    ! echo "$records" \
+        | tr '{' '\n' \
+        | grep -F "\"name\": \"${PERSIST_DOMAIN}\"" \
+        | grep -qF '"type": "CAA"'
+}
+
+# Rotation registers a new account, and every published record names the old
+# one. The response has to carry the replacements, because the gateway cannot
+# publish them and nothing else reports them.
+test_rotation_reports_the_records_to_republish() {
+    local out
+    out=$(admin_post RotateAcmeCredentials '{}') || return 1
+    echo "$out" | grep -qF "_validation-persist.${PERSIST_DOMAIN}" || return 1
+    # Naming the new account, not the one it replaced.
+    local uri
+    uri=$(echo "$out" | sed -e 's/.*"account_uri": *"//' -e 's/".*//')
+    [ -n "$uri" ] || return 1
+    echo "$out" | grep -qF "accounturi=${uri}"
+}
+
 # The claim the whole challenge exists for: certbot reads DNS and never writes
 # it. Scoped to the record certbot would have written -- the zone is not empty,
 # because the harness published the validation record above standing in for the
@@ -491,6 +640,20 @@ main() {
     run_test "No DNS record is ever written for it" \
         "$(test_persist_domain_never_touches_the_provider; echo $?)"
 
+    # What the CA refuses, now that it actually looks the record up.
+    run_test "A record without policy=wildcard cannot answer a wildcard order" \
+        "$(test_persist_record_without_wildcard_policy_is_refused; echo $?)"
+    run_test "Adding policy=wildcard is what makes it answerable" \
+        "$(test_persist_record_with_wildcard_policy_issues; echo $?)"
+    run_test "A record naming another account does not authorize" \
+        "$(test_persist_record_for_another_account_is_refused; echo $?)"
+
+    # Gateway operations that change shape for a domain it cannot write.
+    run_test "SetCaa skips it instead of failing or writing" \
+        "$(test_set_caa_skips_a_persist_domain; echo $?)"
+    run_test "Rotation reports the records to republish" \
+        "$(test_rotation_reports_the_records_to_republish; echo $?)"
+
     # Summary
     log_section "Test Summary"
     log_info "Passed: $TESTS_PASSED"