From 47d055a4e3a20e3f2b9638c31648f986f2ce381e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 01:36:00 -0700 Subject: [PATCH] test(gateway): add the port a ZT domain needs, and cover the CAA guard rerun Phase 4 has been adding the dns-01 domains without a port, which the server refuses -- `proto_to_zt_domain_config` requires 1..65535 -- and the call swallowed the error with `|| true`. So `test0.local` and friends were never configured, and everything downstream of them failed: CAA records reconciled for 0 domains (3 on dns-persist-01 left ...) That is 22 of the suite's assertions, reported as failures for long enough to read as the environment. They are not: nothing had been added to fail on. The port is supplied and the swallow becomes a warning, so the next time this breaks it says so. With a dns-01 domain that exists, the CAA guard sequence can be exercised. A run that dies between installing the `;` guards and dropping them leaves them behind, and the operator is told to rerun; the rerun has to work, and has to get there without ever lifting the deny-all, because a name with no issuer CAA is one any CA may issue for. The stranded state is planted directly -- it is exactly what the dead run left -- so this needs no way to kill a run midway. The half that matters is the second one. Both reusing the stranded guard and deleting it to add a fresh one end with the same records, and only one of them stays denied throughout, so the gap is invisible in the result. The mock therefore records names whose issuer CAA set emptied out, and the test asserts the domain is not among them. Reverting to delete-then-add fails it. --- dstack/gateway/test-run/e2e/test.sh | 59 ++++++++++++++++++++++++++++- tools/mock-cf-dns/server.py | 49 +++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh index 59e9f0ca7..b055a33c1 100755 --- a/dstack/gateway/test-run/e2e/test.sh +++ b/dstack/gateway/test-run/e2e/test.sh @@ -428,6 +428,60 @@ test_set_caa_skips_a_persist_domain() { | grep -qF '"type": "CAA"' } +# A run of SetCaa that dies between installing the `;` guards and dropping them +# leaves the guards behind, and the operator is told to rerun. The rerun has to +# work -- re-adding a byte-identical guard is what a provider that refuses +# duplicates rejects -- and it has to get there without ever lifting the +# deny-all, because a name with no issuer CAA is one any CA may issue for. +# +# The stranded state is planted directly: that is exactly what the dead run +# left, and it needs no way to kill a run mid-flight. +test_caa_rerun_recovers_without_lifting_deny_all() { + local domain="${CERT_DOMAINS%% *}" + local zone="zone-${domain//./-}" + local id tag + + # Plant the guards a dead run would have left, and remove the real records + # it had already deleted by that point. + for id in $(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${domain}\"" \ + | grep -F '"type": "CAA"' \ + | sed -e 's/.*"id": "//' -e 's/".*//'); do + curl -sf -X DELETE "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records/${id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" > /dev/null 2>&1 || true + done + for tag in issue issuewild; do + curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"type": "CAA", "name": "'"${domain}"'", "content": "0 '"${tag}"' \";\"", "ttl": 60}' \ + > /dev/null || return 1 + done + + # From here the name is deny-all, and must stay that way. + curl -sf -X DELETE "${MOCK_CF_API}/api/caa-gaps" > /dev/null 2>&1 || true + admin_post SetCaa '{}' > /dev/null || return 1 + + # Recovered: real issuer records, and none of the guards left behind. + local caa + caa=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${domain}\"" \ + | grep -F '"type": "CAA"') + echo "$caa" | grep -qF 'accounturi=' || return 1 + # An `if`, not `&& return 1`: under `set -e` a failing left-hand side makes + # the whole list non-zero and aborts the function. + if echo "$caa" | grep -qF '0 issue \";\"'; then + return 1 + fi + + # And never fell open on the way. This is the half that separates reusing + # the stranded guard from deleting it and adding a fresh one: both end here, + # only one of them stays denied throughout. + ! curl -sf "${MOCK_CF_API}/api/caa-gaps" 2>/dev/null | grep -qF "\"${domain}\"" +} + # Rotation registers a new account, and every published record names the old # one. The response has to carry the replacements, because the gateway cannot # publish them and nothing else reports them. @@ -546,7 +600,8 @@ setup_certbot_config() { curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.AddZtDomain" \ -H "${ADMIN_AUTH_HEADER}" \ -H "Content-Type: application/json" \ - -d '{"domain": "'"${domain}"'"}' > /dev/null || true + -d '{"domain": "'"${domain}"'", "port": 443}' > /dev/null \ + || log_warn "AddZtDomain failed for $domain (may already exist)" log_info "Triggering renewal for: $domain" curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ @@ -712,6 +767,8 @@ main() { # Gateway operations that change shape for a domain it cannot write. run_test "SetCaa skips it instead of failing or writing" \ "$(test_set_caa_skips_a_persist_domain; echo $?)" + run_test "A stranded CAA guard is recovered without falling open" \ + "$(test_caa_rerun_recovers_without_lifting_deny_all; echo $?)" run_test "Rotation reports the records to republish" \ "$(test_rotation_reports_the_records_to_republish; echo $?)" diff --git a/tools/mock-cf-dns/server.py b/tools/mock-cf-dns/server.py index d8c0e9253..70fc8dd7c 100755 --- a/tools/mock-cf-dns/server.py +++ b/tools/mock-cf-dns/server.py @@ -10,6 +10,10 @@ POST /client/v4/zones//dns_records DELETE /client/v4/zones//dns_records/ +It tracks one thing beyond storing records: names whose issuer CAA set was +emptied, served at /api/caa-gaps, because replacing CAA is meant to leave one +record standing throughout and a gap is invisible once the replacement is done. + It also serves TXT answers on 53, over both UDP and TCP, so Pebble can validate DNS-01 and dns-persist-01 for real rather than being run with PEBBLE_VA_ALWAYS_VALID=1. TCP is not optional: Pebble sets its DNS client to @@ -42,6 +46,14 @@ QUERIES: list[dict[str, Any]] = [] MAX_QUERIES = 2000 NEXT_ID = 1 +# Names whose issuer CAA set went from non-empty to empty at some point. +# +# A client replacing those records is meant to keep at least one in place +# throughout: absent CAA is not "no issuer may" but "any issuer may", so a gap +# is a window in which any CA could have issued. It closes as soon as the +# replacement finishes and is invisible in the result, so it is recorded as it +# happens. +CAA_GAPS: set[str] = set() def _zones() -> list[dict[str, str]]: @@ -76,6 +88,27 @@ def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None: handler.wfile.write(data) +def _issuer_caa_count(name: str) -> int: + """How many issue/issuewild CAA records currently sit at `name`.""" + wanted = name.strip(".").lower() + return sum( + 1 + for r in RECORDS + if r["type"] == "CAA" + and r["name"].strip(".").lower() == wanted + and (r["content"].split() + ["", ""])[1] in ("issue", "issuewild") + ) + + +def _note_caa_gap(name: str, before: int) -> None: + """Record `name` if its issuer CAA set just emptied out. + + Called with STATE_LOCK held, `before` sampled ahead of the mutation. + """ + if before > 0 and _issuer_caa_count(name) == 0: + CAA_GAPS.add(name.strip(".").lower()) + + def _record_content(payload: dict[str, Any]) -> str: if "content" in payload: return str(payload.get("content") or "") @@ -116,6 +149,10 @@ def do_GET(self) -> None: # noqa: N802 if path == "/health": _json(self, 200, {"ok": True}) return + if path == "/api/caa-gaps": + with STATE_LOCK: + _json(self, 200, {"names": sorted(CAA_GAPS)}) + return if path == "/api/records": with STATE_LOCK: _json(self, 200, {"records": RECORDS}) @@ -200,9 +237,14 @@ def do_POST(self) -> None: # noqa: N802 _json(self, 200, {"success": True, "result": record}) def do_DELETE(self) -> None: # noqa: N802 - """Delete a mock DNS record.""" + """Delete a mock DNS record, or forget the CAA gaps seen so far.""" parsed = urllib.parse.urlparse(self.path) path = parsed.path.rstrip("/") or "/" + if path == "/api/caa-gaps": + with STATE_LOCK: + CAA_GAPS.clear() + _json(self, 200, {"success": True}) + return m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path) if not m: _json( @@ -215,9 +257,14 @@ def do_DELETE(self) -> None: # noqa: N802 return record_id = urllib.parse.unquote(m.group(2)) with STATE_LOCK: + doomed = next((r for r in RECORDS if r["id"] == record_id), None) + caa_name = doomed["name"] if doomed and doomed["type"] == "CAA" else None + caa_before = _issuer_caa_count(caa_name) if caa_name else 0 before = len(RECORDS) RECORDS[:] = [r for r in RECORDS if r["id"] != record_id] removed = before != len(RECORDS) + if caa_name: + _note_caa_gap(caa_name, caa_before) _json( self, 200,