test: resolve challenge records through the mock, and check that it happened

dstack/gateway/test-run/e2e/docker-compose.yml

@@ -51,7 +51,7 @@ services:
       - PORT=8080
       - DEBUG=true
       # The zones certbot writes into and Pebble reads back out of.
-      - MOCK_CF_ZONES=test0.local,test1.local,test2.local,persist0.local,persist1.local,persist2.local
+      - MOCK_CF_ZONES=test0.local,test1.local,test2.local,persist0.local,persist1.local,persist2.local,selfcheck0.local
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"]
       interval: 5s
@@ -117,6 +117,12 @@ services:
       timeout: 3s
       retries: 10
       start_period: 30s
+    # Resolve through the mock, which owns the test zones and forwards
+    # everything else. certbot's pre-order self-check reads DNS like any
+    # client; without this it queries a resolver that has never heard of the
+    # challenge names, so the check can only ever time out.
+    dns:
+      - 172.30.0.10
     cap_add:
       - NET_ADMIN
     extra_hosts:
@@ -152,6 +158,12 @@ services:
       timeout: 3s
       retries: 10
       start_period: 30s
+    # Resolve through the mock, which owns the test zones and forwards
+    # everything else. certbot's pre-order self-check reads DNS like any
+    # client; without this it queries a resolver that has never heard of the
+    # challenge names, so the check can only ever time out.
+    dns:
+      - 172.30.0.10
     cap_add:
       - NET_ADMIN
 
@@ -184,6 +196,12 @@ services:
       timeout: 3s
       retries: 10
       start_period: 30s
+    # Resolve through the mock, which owns the test zones and forwards
+    # everything else. certbot's pre-order self-check reads DNS like any
+    # client; without this it queries a resolver that has never heard of the
+    # challenge names, so the check can only ever time out.
+    dns:
+      - 172.30.0.10
     cap_add:
       - NET_ADMIN
 

dstack/gateway/test-run/e2e/test.sh

@@ -348,6 +348,67 @@ test_persist_record_for_another_account_is_refused() {
     ! neg_domain_issued "${PERSIST_NEG2_DOMAIN}"
 }
 
+# ---- The pre-order DNS self-check -----------------------------------------
+#
+# certbot resolves the challenge name itself before telling the CA to go and
+# look, so that a record that has not propagated is reported by name instead of
+# as an order failure. The check is advisory: it warns and proceeds either way,
+# which is exactly why nothing downstream reveals whether it worked. The mock
+# logs the questions it is asked, so the check is observed directly.
+
+dns_queries_for() {
+    curl -sf "${MOCK_CF_API}/api/dns-queries" 2>/dev/null \
+        | tr '{' '\n' \
+        | grep -F "\"name\": \"$1\"" || true
+}
+
+answered_queries_for() {
+    dns_queries_for "$1" | grep -cvF '"answers": 0'
+}
+
+# The happy path, observed rather than triggered. By the time this runs the
+# dns-persist-01 domain has issued from a record this suite published, so the
+# self-check must have resolved it -- and an answered question is the only
+# direct evidence, because the check warns and proceeds either way.
+#
+# Deliberately passive: forcing another renewal would race the periodic one,
+# which picks a domain up as soon as it is added and can leave nothing for the
+# forced run to do.
+test_self_check_resolves_a_published_record() {
+    [ "$(answered_queries_for "_validation-persist.${PERSIST_DOMAIN}")" -gt 0 ]
+}
+
+# The unhappy path, and the reason the check is advisory at all. A name with no
+# record has to be polled and given up on -- the record may still be
+# propagating -- rather than asked once and abandoned.
+#
+# Its own domain, and one nothing else queries, so this needs no clearing and
+# does not care what ran before it.
+test_self_check_gives_up_on_a_missing_record() {
+    local domain="selfcheck0.local"
+    local name="_validation-persist.${domain}"
+    admin_post DeleteZtDomain '{"domain": "'"${domain}"'"}' > /dev/null 2>&1 || true
+    # dns-persist-01 because nothing writes its record: the name stays empty for
+    # the whole wait without the test racing certbot's own cleanup.
+    admin_post AddZtDomain \
+        '{"domain": "'"${domain}"'", "port": 443, "challenge": "dns-persist-01"}' \
+        > /dev/null || return 1
+    admin_post RenewZtDomainCert \
+        '{"domain": "'"${domain}"'", "force": true}' > /dev/null 2>&1 || true
+
+    local i=0 asked=0
+    while [ $i -lt 45 ]; do
+        asked=$(dns_queries_for "$name" | wc -l)
+        [ "$asked" -ge 3 ] && break
+        sleep 2
+        i=$((i + 1))
+    done
+    # Polled, not asked once: the retry loop is what makes the wait a wait.
+    [ "$asked" -ge 3 ] || return 1
+    # Every one a miss, or the record was not actually absent.
+    [ "$(answered_queries_for "$name")" -eq 0 ]
+}
+
 # ---- Gateway operations that change shape for such a domain ---------------
 
 # SetCaa reconciles CAA through the DNS provider, which the gateway has no
@@ -654,6 +715,12 @@ main() {
     run_test "Rotation reports the records to republish" \
         "$(test_rotation_reports_the_records_to_republish; echo $?)"
 
+    # The pre-order self-check, both ways round.
+    run_test "Self-check resolves a published challenge record" \
+        "$(test_self_check_resolves_a_published_record; echo $?)"
+    run_test "Self-check polls and gives up when the record is absent" \
+        "$(test_self_check_gives_up_on_a_missing_record; echo $?)"
+
     # Summary
     log_section "Test Summary"
     log_info "Passed: $TESTS_PASSED"

tools/mock-cf-dns/server.py

@@ -14,6 +14,12 @@
 DNS-01 and dns-persist-01 for real rather than being run with
 PEBBLE_VA_ALWAYS_VALID=1. TCP is not optional: Pebble sets its DNS client to
 `Net = "tcp"` whenever it is given `-dnsserver`.
+
+Questions outside the configured zones are forwarded upstream, so this can be a
+client's only resolver rather than only a CA's `-dnsserver`: certbot resolves
+its own challenge records through it while still reaching the other containers
+by name. Every question is logged and served at /api/dns-queries, so a test can
+assert that a name was actually looked up.
 """
 
 from __future__ import annotations
@@ -31,6 +37,10 @@
 
 STATE_LOCK = threading.RLock()
 RECORDS: list[dict[str, Any]] = []
+# Every DNS question this mock was asked, so a test can assert that the client
+# under test resolved a name rather than inferring it from what happened next.
+QUERIES: list[dict[str, Any]] = []
+MAX_QUERIES = 2000
 NEXT_ID = 1
 
 
@@ -110,6 +120,10 @@ def do_GET(self) -> None:  # noqa: N802
             with STATE_LOCK:
                 _json(self, 200, {"records": RECORDS})
             return
+        if path == "/api/dns-queries":
+            with STATE_LOCK:
+                _json(self, 200, {"queries": QUERIES})
+            return
         if path.startswith("/client/v4/") and not _authorized(self):
             return
         if path == "/client/v4/zones":
@@ -233,6 +247,55 @@ def txt_rdata(text: str) -> bytes:
     return b"".join(bytes([len(c)]) + c for c in chunks)
 
 
+def _upstream() -> str:
+    """Where to send questions this mock is not authoritative for.
+
+    Docker's embedded resolver, which is what this container's own
+    `/etc/resolv.conf` points at, so service names keep resolving for whoever
+    is pointed here.
+    """
+    return os.environ.get("MOCK_DNS_UPSTREAM", "127.0.0.11")
+
+
+def _is_ours(name: str) -> bool:
+    """Whether `name` falls inside one of the configured mock zones."""
+    return any(
+        name == zone["name"] or name.endswith("." + zone["name"]) for zone in _zones()
+    )
+
+
+def _forward(packet: bytes) -> bytes:
+    """Ask the upstream resolver and hand back its answer verbatim.
+
+    Without this the mock is only usable as a CA's `-dnsserver`, because a name
+    it does not know is answered NOERROR with no records -- which a resolver
+    reads as an authoritative "no such record" and does not retry elsewhere.
+    A client configured to use this as its only resolver would then fail to
+    resolve the other containers.
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(5)
+    try:
+        sock.sendto(packet, (_upstream(), 53))
+        return sock.recvfrom(65535)[0]
+    finally:
+        sock.close()
+
+
+def _record_query(name: str, qtype: int, answers: int, forwarded: bool) -> None:
+    with STATE_LOCK:
+        QUERIES.append(
+            {
+                "name": name,
+                "type": qtype,
+                "answers": answers,
+                "forwarded": forwarded,
+                "at": int(time.time()),
+            }
+        )
+        del QUERIES[:-MAX_QUERIES]
+
+
 def dns_response(packet: bytes) -> bytes:
     """Build a DNS response containing matching TXT records."""
     if len(packet) < 12:
@@ -248,6 +311,18 @@ def dns_response(packet: bytes) -> bytes:
         if qend + 4 <= len(packet)
         else 16
     )
+    # Only answer for the zones this mock owns; anything else is the caller's
+    # ordinary name resolution and belongs upstream.
+    if not _is_ours(name):
+        try:
+            reply = _forward(packet)
+            _record_query(name, qtype, -1, True)
+            return reply
+        except Exception as exc:  # pragma: no cover - diagnostic only
+            _debug(f"forwarding {name} failed: {exc}")
+            _record_query(name, qtype, 0, True)
+            return txid + struct.pack("!HHHHH", 0x8182, 1, 0, 0, 0) + packet[12:qend + 4]
+
     with STATE_LOCK:
         answers = [
             r
@@ -256,6 +331,7 @@ def dns_response(packet: bytes) -> bytes:
         ]
     if qtype not in (16, 255):
         answers = []
+    _record_query(name, qtype, len(answers), False)
     header = txid + struct.pack("!HHHHH", 0x8180, 1, len(answers), 0, 0)
     body = question
     for record in answers: