test(gateway): add the port a ZT domain needs, and cover the CAA guard rerun

dstack/gateway/test-run/e2e/test.sh

@@ -428,6 +428,60 @@ test_set_caa_skips_a_persist_domain() {
         | grep -qF '"type": "CAA"'
 }
 
+# A run of SetCaa that dies between installing the `;` guards and dropping them
+# leaves the guards behind, and the operator is told to rerun. The rerun has to
+# work -- re-adding a byte-identical guard is what a provider that refuses
+# duplicates rejects -- and it has to get there without ever lifting the
+# deny-all, because a name with no issuer CAA is one any CA may issue for.
+#
+# The stranded state is planted directly: that is exactly what the dead run
+# left, and it needs no way to kill a run mid-flight.
+test_caa_rerun_recovers_without_lifting_deny_all() {
+    local domain="${CERT_DOMAINS%% *}"
+    local zone="zone-${domain//./-}"
+    local id tag
+
+    # Plant the guards a dead run would have left, and remove the real records
+    # it had already deleted by that point.
+    for id in $(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \
+        | tr '{' '\n' \
+        | grep -F "\"name\": \"${domain}\"" \
+        | grep -F '"type": "CAA"' \
+        | sed -e 's/.*"id": "//' -e 's/".*//'); do
+        curl -sf -X DELETE "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records/${id}" \
+            -H "Authorization: Bearer ${CF_API_TOKEN}" > /dev/null 2>&1 || true
+    done
+    for tag in issue issuewild; do
+        curl -sf -X POST "${MOCK_CF_API}/client/v4/zones/${zone}/dns_records" \
+            -H "Authorization: Bearer ${CF_API_TOKEN}" \
+            -H "Content-Type: application/json" \
+            -d '{"type": "CAA", "name": "'"${domain}"'", "content": "0 '"${tag}"' \";\"", "ttl": 60}' \
+            > /dev/null || return 1
+    done
+
+    # From here the name is deny-all, and must stay that way.
+    curl -sf -X DELETE "${MOCK_CF_API}/api/caa-gaps" > /dev/null 2>&1 || true
+    admin_post SetCaa '{}' > /dev/null || return 1
+
+    # Recovered: real issuer records, and none of the guards left behind.
+    local caa
+    caa=$(curl -sf "${MOCK_CF_API}/api/records" 2>/dev/null \
+        | tr '{' '\n' \
+        | grep -F "\"name\": \"${domain}\"" \
+        | grep -F '"type": "CAA"')
+    echo "$caa" | grep -qF 'accounturi=' || return 1
+    # An `if`, not `&& return 1`: under `set -e` a failing left-hand side makes
+    # the whole list non-zero and aborts the function.
+    if echo "$caa" | grep -qF '0 issue \";\"'; then
+        return 1
+    fi
+
+    # And never fell open on the way. This is the half that separates reusing
+    # the stranded guard from deleting it and adding a fresh one: both end here,
+    # only one of them stays denied throughout.
+    ! curl -sf "${MOCK_CF_API}/api/caa-gaps" 2>/dev/null | grep -qF "\"${domain}\""
+}
+
 # Rotation registers a new account, and every published record names the old
 # one. The response has to carry the replacements, because the gateway cannot
 # publish them and nothing else reports them.
@@ -546,7 +600,8 @@ setup_certbot_config() {
         curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.AddZtDomain" \
             -H "${ADMIN_AUTH_HEADER}" \
             -H "Content-Type: application/json" \
-            -d '{"domain": "'"${domain}"'"}' > /dev/null || true
+            -d '{"domain": "'"${domain}"'", "port": 443}' > /dev/null \
+            || log_warn "AddZtDomain failed for $domain (may already exist)"
 
         log_info "Triggering renewal for: $domain"
         curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \
@@ -712,6 +767,8 @@ main() {
     # Gateway operations that change shape for a domain it cannot write.
     run_test "SetCaa skips it instead of failing or writing" \
         "$(test_set_caa_skips_a_persist_domain; echo $?)"
+    run_test "A stranded CAA guard is recovered without falling open" \
+        "$(test_caa_rerun_recovers_without_lifting_deny_all; echo $?)"
     run_test "Rotation reports the records to republish" \
         "$(test_rotation_reports_the_records_to_republish; echo $?)"
 

tools/mock-cf-dns/server.py

@@ -10,6 +10,10 @@
   POST   /client/v4/zones/<zone_id>/dns_records
   DELETE /client/v4/zones/<zone_id>/dns_records/<record_id>
 
+It tracks one thing beyond storing records: names whose issuer CAA set was
+emptied, served at /api/caa-gaps, because replacing CAA is meant to leave one
+record standing throughout and a gap is invisible once the replacement is done.
+
 It also serves TXT answers on 53, over both UDP and TCP, so Pebble can validate
 DNS-01 and dns-persist-01 for real rather than being run with
 PEBBLE_VA_ALWAYS_VALID=1. TCP is not optional: Pebble sets its DNS client to
@@ -42,6 +46,14 @@
 QUERIES: list[dict[str, Any]] = []
 MAX_QUERIES = 2000
 NEXT_ID = 1
+# Names whose issuer CAA set went from non-empty to empty at some point.
+#
+# A client replacing those records is meant to keep at least one in place
+# throughout: absent CAA is not "no issuer may" but "any issuer may", so a gap
+# is a window in which any CA could have issued. It closes as soon as the
+# replacement finishes and is invisible in the result, so it is recorded as it
+# happens.
+CAA_GAPS: set[str] = set()
 
 
 def _zones() -> list[dict[str, str]]:
@@ -76,6 +88,27 @@ def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None:
     handler.wfile.write(data)
 
 
+def _issuer_caa_count(name: str) -> int:
+    """How many issue/issuewild CAA records currently sit at `name`."""
+    wanted = name.strip(".").lower()
+    return sum(
+        1
+        for r in RECORDS
+        if r["type"] == "CAA"
+        and r["name"].strip(".").lower() == wanted
+        and (r["content"].split() + ["", ""])[1] in ("issue", "issuewild")
+    )
+
+
+def _note_caa_gap(name: str, before: int) -> None:
+    """Record `name` if its issuer CAA set just emptied out.
+
+    Called with STATE_LOCK held, `before` sampled ahead of the mutation.
+    """
+    if before > 0 and _issuer_caa_count(name) == 0:
+        CAA_GAPS.add(name.strip(".").lower())
+
+
 def _record_content(payload: dict[str, Any]) -> str:
     if "content" in payload:
         return str(payload.get("content") or "")
@@ -116,6 +149,10 @@ def do_GET(self) -> None:  # noqa: N802
         if path == "/health":
             _json(self, 200, {"ok": True})
             return
+        if path == "/api/caa-gaps":
+            with STATE_LOCK:
+                _json(self, 200, {"names": sorted(CAA_GAPS)})
+            return
         if path == "/api/records":
             with STATE_LOCK:
                 _json(self, 200, {"records": RECORDS})
@@ -200,9 +237,14 @@ def do_POST(self) -> None:  # noqa: N802
         _json(self, 200, {"success": True, "result": record})
 
     def do_DELETE(self) -> None:  # noqa: N802
-        """Delete a mock DNS record."""
+        """Delete a mock DNS record, or forget the CAA gaps seen so far."""
         parsed = urllib.parse.urlparse(self.path)
         path = parsed.path.rstrip("/") or "/"
+        if path == "/api/caa-gaps":
+            with STATE_LOCK:
+                CAA_GAPS.clear()
+            _json(self, 200, {"success": True})
+            return
         m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path)
         if not m:
             _json(
@@ -215,9 +257,14 @@ def do_DELETE(self) -> None:  # noqa: N802
             return
         record_id = urllib.parse.unquote(m.group(2))
         with STATE_LOCK:
+            doomed = next((r for r in RECORDS if r["id"] == record_id), None)
+            caa_name = doomed["name"] if doomed and doomed["type"] == "CAA" else None
+            caa_before = _issuer_caa_count(caa_name) if caa_name else 0
             before = len(RECORDS)
             RECORDS[:] = [r for r in RECORDS if r["id"] != record_id]
             removed = before != len(RECORDS)
+            if caa_name:
+                _note_caa_gap(caa_name, caa_before)
         _json(
             self,
             200,