diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh index 59e9f0ca7..b055a33c1 100755 --- a/dstack/gateway/test-run/e2e/test.sh +++ b/dstack/gateway/test-run/e2e/test.sh @@ -428,6 +428,60 @@ test_set_caa_skips_a_persist_domain() { | grep -qF '"type": "CAA"' } +# A run of SetCaa that dies between installing the `;` guards and dropping them +# leaves the guards behind, and the operator is told to rerun. The rerun has to +# work -- re-adding a byte-identical guard is what a provider that refuses +# duplicates rejects -- and it has to get there without ever lifting the +# deny-all, because a name with no issuer CAA is one any CA may issue for. +# +# The stranded state is planted directly: that is exactly what the dead run +# left, and it needs no way to kill a run mid-flight. +test_caa_rerun_recovers_without_lifting_deny_all() { + local domain="${CERT_DOMAINS%% *}" + local zone="zone-${domain//./-}" + local id tag + + # Plant the guards a dead run would have left, and remove the real records + # it had already deleted by that point. + for id in $(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${domain}\"" \ + | grep -F '"type": "CAA"' \ + | sed -e 's/.*"id": "//' -e 's/".*//'); do + curl -sf -X DELETE "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records/${id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" > /dev/null 2>&1 || true + done + for tag in issue issuewild; do + curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"type": "CAA", "name": "'"${domain}"'", "content": "0 '"${tag}"' \";\"", "ttl": 60}' \ + > /dev/null || return 1 + done + + # From here the name is deny-all, and must stay that way. + curl -sf -X DELETE "${MOCK_CF_API}/api/caa-gaps" > /dev/null 2>&1 || true + admin_post SetCaa '{}' > /dev/null || return 1 + + # Recovered: real issuer records, and none of the guards left behind. + local caa + caa=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \ + | tr '{' '\n' \ + | grep -F "\"name\": \"${domain}\"" \ + | grep -F '"type": "CAA"') + echo "$caa" | grep -qF 'accounturi=' || return 1 + # An `if`, not `&& return 1`: under `set -e` a failing left-hand side makes + # the whole list non-zero and aborts the function. + if echo "$caa" | grep -qF '0 issue \";\"'; then + return 1 + fi + + # And never fell open on the way. This is the half that separates reusing + # the stranded guard from deleting it and adding a fresh one: both end here, + # only one of them stays denied throughout. + ! curl -sf "${MOCK_CF_API}/api/caa-gaps" 2>/dev/null | grep -qF "\"${domain}\"" +} + # Rotation registers a new account, and every published record names the old # one. The response has to carry the replacements, because the gateway cannot # publish them and nothing else reports them. @@ -546,7 +600,8 @@ setup_certbot_config() { curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.AddZtDomain" \ -H "${ADMIN_AUTH_HEADER}" \ -H "Content-Type: application/json" \ - -d '{"domain": "'"${domain}"'"}' > /dev/null || true + -d '{"domain": "'"${domain}"'", "port": 443}' > /dev/null \ + || log_warn "AddZtDomain failed for $domain (may already exist)" log_info "Triggering renewal for: $domain" curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ @@ -712,6 +767,8 @@ main() { # Gateway operations that change shape for a domain it cannot write. run_test "SetCaa skips it instead of failing or writing" \ "$(test_set_caa_skips_a_persist_domain; echo $?)" + run_test "A stranded CAA guard is recovered without falling open" \ + "$(test_caa_rerun_recovers_without_lifting_deny_all; echo $?)" run_test "Rotation reports the records to republish" \ "$(test_rotation_reports_the_records_to_republish; echo $?)" diff --git a/tools/mock-cf-dns/server.py b/tools/mock-cf-dns/server.py index d8c0e9253..70fc8dd7c 100755 --- a/tools/mock-cf-dns/server.py +++ b/tools/mock-cf-dns/server.py @@ -10,6 +10,10 @@ POST /client/v4/zones//dns_records DELETE /client/v4/zones//dns_records/ +It tracks one thing beyond storing records: names whose issuer CAA set was +emptied, served at /api/caa-gaps, because replacing CAA is meant to leave one +record standing throughout and a gap is invisible once the replacement is done. + It also serves TXT answers on 53, over both UDP and TCP, so Pebble can validate DNS-01 and dns-persist-01 for real rather than being run with PEBBLE_VA_ALWAYS_VALID=1. TCP is not optional: Pebble sets its DNS client to @@ -42,6 +46,14 @@ QUERIES: list[dict[str, Any]] = [] MAX_QUERIES = 2000 NEXT_ID = 1 +# Names whose issuer CAA set went from non-empty to empty at some point. +# +# A client replacing those records is meant to keep at least one in place +# throughout: absent CAA is not "no issuer may" but "any issuer may", so a gap +# is a window in which any CA could have issued. It closes as soon as the +# replacement finishes and is invisible in the result, so it is recorded as it +# happens. +CAA_GAPS: set[str] = set() def _zones() -> list[dict[str, str]]: @@ -76,6 +88,27 @@ def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None: handler.wfile.write(data) +def _issuer_caa_count(name: str) -> int: + """How many issue/issuewild CAA records currently sit at `name`.""" + wanted = name.strip(".").lower() + return sum( + 1 + for r in RECORDS + if r["type"] == "CAA" + and r["name"].strip(".").lower() == wanted + and (r["content"].split() + ["", ""])[1] in ("issue", "issuewild") + ) + + +def _note_caa_gap(name: str, before: int) -> None: + """Record `name` if its issuer CAA set just emptied out. + + Called with STATE_LOCK held, `before` sampled ahead of the mutation. + """ + if before > 0 and _issuer_caa_count(name) == 0: + CAA_GAPS.add(name.strip(".").lower()) + + def _record_content(payload: dict[str, Any]) -> str: if "content" in payload: return str(payload.get("content") or "") @@ -116,6 +149,10 @@ def do_GET(self) -> None: # noqa: N802 if path == "/health": _json(self, 200, {"ok": True}) return + if path == "/api/caa-gaps": + with STATE_LOCK: + _json(self, 200, {"names": sorted(CAA_GAPS)}) + return if path == "/api/records": with STATE_LOCK: _json(self, 200, {"records": RECORDS}) @@ -200,9 +237,14 @@ def do_POST(self) -> None: # noqa: N802 _json(self, 200, {"success": True, "result": record}) def do_DELETE(self) -> None: # noqa: N802 - """Delete a mock DNS record.""" + """Delete a mock DNS record, or forget the CAA gaps seen so far.""" parsed = urllib.parse.urlparse(self.path) path = parsed.path.rstrip("/") or "/" + if path == "/api/caa-gaps": + with STATE_LOCK: + CAA_GAPS.clear() + _json(self, 200, {"success": True}) + return m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path) if not m: _json( @@ -215,9 +257,14 @@ def do_DELETE(self) -> None: # noqa: N802 return record_id = urllib.parse.unquote(m.group(2)) with STATE_LOCK: + doomed = next((r for r in RECORDS if r["id"] == record_id), None) + caa_name = doomed["name"] if doomed and doomed["type"] == "CAA" else None + caa_before = _issuer_caa_count(caa_name) if caa_name else 0 before = len(RECORDS) RECORDS[:] = [r for r in RECORDS if r["id"] != record_id] removed = before != len(RECORDS) + if caa_name: + _note_caa_gap(caa_name, caa_before) _json( self, 200,