ci(TC-959): read the full node usage inventory in quota restore

docs/releasing.md

@@ -154,7 +154,10 @@ CVM or whitelist mutation.
 
 `authority=node-override` currently applies the reviewed limits after the
 last restart and required health. Both exact spaces must exist once with
-positive usage in authenticated `/admin/usage`. Before ANY PUT, both billing
+positive usage in authenticated `/admin/usage`, which lists every space on
+the node unpaginated (~107 bytes a space); the script reads up to 16 MiB of
+it and fails closed above that, while other quota bodies stay capped at
+64 KiB. Before ANY PUT, both billing
 storage-comp reads must prove no active comp: 404 logs only
 `billing storage-comp unavailable`; valid `active:false` also permits the
 legacy restore. Active comp fails `authority mismatch: flip to billing`.

scripts/ci/reconcile-quota.py

@@ -13,6 +13,11 @@
 OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412"
 LIMITS = {"applications": 5368709120, "secrets": 268721121}
 NODE_HOST = "node.tinycloud.xyz"
+# Per-space and billing bodies are a few hundred bytes. GET /admin/usage lists
+# every space on the node, unpaginated, at ~107 bytes a row (~1,034 spaces,
+# ~110 KiB, in October 2026): bounded at 16 MiB (~150,000 spaces).
+RESPONSE_CAP = 64 * 1024
+USAGE_RESPONSE_CAP = 16 * 1024 * 1024
 
 
 class QuotaError(Exception):
@@ -67,7 +72,19 @@ def credentials():
     return node, billing, (url.hostname, port)
 
 
-def request_json(host, method, path, secret, limit=None, port=None, allow_404=False):
+def read_bounded(response, cap):
+    """Stream at most cap+1 bytes; the caller rejects a body longer than cap."""
+    chunks, size = [], 0
+    while size <= cap:
+        chunk = response.read(min(64 * 1024, cap + 1 - size))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        size += len(chunk)
+    return b"".join(chunks)
+
+
+def request_json(host, method, path, secret, limit=None, port=None, allow_404=False, cap=RESPONSE_CAP):
     """Fixed TLS requests, no redirects; retry transport only, never HTTP errors."""
     headers = {"Authorization": "Bearer " + secret, "Content-Type": "application/json"}
     body = None if limit is None else json.dumps({"limit_bytes": limit}).encode()
@@ -80,7 +97,7 @@ def request_json(host, method, path, secret, limit=None, port=None, allow_404=Fa
             connection = http.client.HTTPSConnection(host, **kwargs)
             connection.request(method, path, body=body, headers=headers)
             response = connection.getresponse()
-            payload = response.read(65537)
+            payload = read_bounded(response, cap)
         except (OSError, http.client.HTTPException):
             if attempt == 2:
                 raise QuotaError("quota transport failed after bounded attempts") from None
@@ -94,7 +111,8 @@ def request_json(host, method, path, secret, limit=None, port=None, allow_404=Fa
         if response.status != 200:
             raise QuotaError("quota API rejected the request")
         try:
-            result = json.loads(payload) if len(payload) <= 65536 else None
+            # Strict UTF-8: json.loads(bytes) would auto-accept a BOM, UTF-16 or UTF-32.
+            result = json.loads(payload.decode("utf-8")) if len(payload) <= cap else None
             if not isinstance(result, dict):
                 raise ValueError
             return result
@@ -104,7 +122,7 @@ def request_json(host, method, path, secret, limit=None, port=None, allow_404=Fa
 
 def prove_inventory(overrides, node_secret):
     """A quota GET echoes arbitrary IDs; only usage enumerates existing spaces."""
-    inventory = request_json(NODE_HOST, "GET", "/admin/usage", node_secret)
+    inventory = request_json(NODE_HOST, "GET", "/admin/usage", node_secret, cap=USAGE_RESPONSE_CAP)
     rows = inventory.get("spaces")
     if (not isinstance(rows, list) or type(inventory.get("count")) is not int
         or inventory["count"] != len(rows)):

scripts/ci/test-deploy-recovery.py

@@ -117,7 +117,7 @@ def event(value):
 else: raise SystemExit('Unexpected CLI boundary')
 '''
 
-HTTP = r'''import http.client, json, os, socket
+HTTP = r'''import http.client, io, json, os, random, socket
 from pathlib import Path
 from urllib.parse import unquote
 socket.create_connection=lambda *a,**k: (_ for _ in ()).throw(AssertionError('Real network forbidden'))
@@ -147,12 +147,17 @@ def getresponse(self):
                 payload={'active':True,'comp_bytes':1,'account_limit_bytes':floor+100,'storage_limit_bytes':floor+100}
             else: status,payload=404,os.environ['QUOTA_BILLING_ADMIN_SECRET']
         elif os.environ['FAIL_AT']=='quota': status,payload=503,os.environ['QUOTA_ADMIN_SECRET']
-        elif self.path=='/admin/usage': payload={'count':len(targets),'spaces':[{'space_id':space,'usage_bytes':10} for space in targets]}
+        elif self.path=='/admin/usage':
+            # The node's real, unpaginated inventory: every space, ~107 bytes a row (prod: ~1,034 spaces).
+            rng=random.Random(959); spaces=[{'space_id':space,'usage_bytes':10} for space in targets]
+            spaces+=[{'space_id':'tinycloud:pkh:eip155:1:0x%040x:default'%rng.getrandbits(160),'usage_bytes':rng.choice([None,rng.randint(1,700000000)])} for _ in range(3000)]
+            payload={'spaces':spaces,'count':len(spaces)}
         else:
             value=data['values'].get(self.path)
             payload={'space_id':unquote(self.path.removeprefix('/admin/quota/')),'override_bytes':value,'effective_limit_bytes':value if value is not None else 123}
+        stream=io.BytesIO(json.dumps(payload).encode())
         class Response:
-            def read(self,count): return json.dumps(payload).encode()[:count]
+            def read(self,count): return stream.read(count)
         response=Response(); response.status=status; return response
     def close(self): pass
 http.client.HTTPSConnection=Connection

scripts/ci/test-quota-authority.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 """Synthetic HTTP contracts for reviewed node/billing quota authority."""
 import contextlib
+import gzip
 import http.client
 import io
 import json
 from pathlib import Path
+import random
 import secrets
 import tempfile
 import unittest
@@ -19,6 +21,24 @@
 OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412"
 LIMITS = {OWNER + ":applications": 5368709120, OWNER + ":secrets": 268721121}
 CANARIES = []
+USAGE_CAP = 16 * 1024 * 1024
+
+
+def realistic_inventory(total):
+    """The node's real GET /admin/usage body (routes/admin.rs `UsageResponse`):
+    every space on the node, unpaginated, as compact serde JSON with `spaces`
+    before `count`, sorted by usage descending with unknown (null) last.
+    Production held about 1,034 spaces on 2026-10-12 (~107 bytes a row)."""
+    rng = random.Random(959)
+    rows = [{"space_id": space, "usage_bytes": usage}
+            for space, usage in zip(LIMITS, (640743590, 1048576))]
+    while len(rows) < total:
+        owner = "tinycloud:pkh:eip155:1:0x" + "".join(rng.choice("0123456789abcdef") for _ in range(40))
+        name = rng.choice(("default", "applications", "secrets", "kv", "public", "notes"))
+        rows.append({"space_id": owner + ":" + name,
+                     "usage_bytes": None if rng.random() < 0.4 else rng.randint(1, 700_000_000)})
+    rows.sort(key=lambda row: (row["usage_bytes"] is None, -(row["usage_bytes"] or 0)))
+    return json.dumps({"spaces": rows, "count": len(rows)}, separators=(",", ":")).encode()
 
 
 class SafeResult(unittest.TextTestResult):
@@ -46,6 +66,8 @@ def setUp(self):
         self.node_status = 200
         self.bad_effective = False
         self.usage = [{"space_id": space, "usage_bytes": 10} for space in LIMITS]
+        self.usage_body = None
+        self.stream = None
         self.directory = tempfile.TemporaryDirectory()
         self.addCleanup(self.directory.cleanup)
         self.config = Path(self.directory.name) / "quota.json"
@@ -106,6 +128,8 @@ def getresponse(self):
                     body = test.billing_body[space] if status == 200 else test.billing_secret
                 elif test.node_status != 200:
                     status, body = test.node_status, test.node_secret
+                elif self.path == "/admin/usage" and test.usage_body is not None:
+                    status, body = 200, test.usage_body
                 elif self.path == "/admin/usage":
                     status, body = 200, {"count": len(test.usage), "spaces": test.usage}
                 else:
@@ -115,10 +139,11 @@ def getresponse(self):
                                          "effective_limit_bytes": (value + int(test.bad_effective)) if value is not None else 123,
                                          "current_usage_bytes": 10}
 
-                class Response:
-                    def read(self, count):
-                        return json.dumps(body).encode()[:count]
-                response = Response()
+                # Stream semantics of http.client.HTTPResponse.read(amt): at most
+                # amt bytes per call, b"" once the body is exhausted.
+                stream = io.BytesIO(body if isinstance(body, bytes) else json.dumps(body).encode())
+                test.stream = stream
+                response = type("Response", (), {"read": lambda _, count: stream.read(count)})()
                 response.status = status
                 return response
 
@@ -318,6 +343,82 @@ def test_invalid_inventory_never_applies_either_override(self):
                 self.assertEqual(self.run_main()[0], 1)
                 self.assertEqual(self.puts(), [])
 
+    def test_prod_sized_inventory_drives_the_full_restore(self):
+        self.usage_body = realistic_inventory(3000)
+        self.assertGreater(len(self.usage_body), 4 * 65536)
+        for authority in ("node-override", "billing"):
+            with self.subTest(authority=authority):
+                self.calls.clear(); self.values.clear()
+                self.configure(authority)
+                if authority == "billing":
+                    self.billing_active()
+                code, logs = self.run_main()
+                self.assertEqual(code, 0)
+                if authority == "billing":
+                    self.assertIn("billing-authoritative", logs)
+                    self.assertEqual(self.puts(), [])
+                else:
+                    self.assertIn("Quota overrides verified", logs)
+                    self.assertEqual(sorted(self.values.values()), sorted(LIMITS.values()))
+                    self.assertEqual([call[1] for call in self.calls], ["GET"] * 3 + ["PUT", "GET", "PUT", "GET"])
+
+    def test_prod_sized_inventory_keeps_exact_target_invariants(self):
+        document = json.loads(realistic_inventory(3000))
+        first = next(iter(LIMITS))
+        target = next(row for row in document["spaces"] if row["space_id"] == first)
+        variants = {
+            "missing": [row for row in document["spaces"] if row is not target],
+            "duplicate": document["spaces"] + [dict(target)],
+            "unknown-usage": [dict(row, usage_bytes=None) if row is target else row for row in document["spaces"]],
+        }
+        for name, rows in variants.items():
+            with self.subTest(case=name):
+                self.calls.clear()
+                self.usage_body = json.dumps({"spaces": rows, "count": len(rows)}).encode()
+                code, logs = self.run_main()
+                self.assertEqual(code, 1)
+                self.assertIn("no overrides applied", logs)
+                self.assertEqual(self.puts(), [])
+
+    def test_usage_body_is_bounded_at_the_documented_cap(self):
+        inventory = realistic_inventory(3000)
+        at_cap = inventory + b" " * (USAGE_CAP - len(inventory))
+        self.usage_body = at_cap
+        code, logs = self.run_main()
+        self.assertEqual(code, 0, "A valid inventory of exactly the cap is accepted")
+        self.calls.clear(); self.values.clear()
+        self.usage_body = at_cap + b" "
+        code, logs = self.run_main()
+        self.assertEqual(code, 1)
+        self.assertIn("quota API returned an invalid response", logs)
+        self.assertEqual(self.calls, [("node.tinycloud.xyz", "GET", "/admin/usage")])
+        self.assertLessEqual(self.stream.tell(), USAGE_CAP + 1, "The read stops at cap+1 bytes")
+
+    def test_invalid_usage_body_fails_closed(self):
+        inventory = realistic_inventory(3000)
+        text = inventory.decode()
+        # json.loads(bytes) auto-detects UTF-8-sig/16/32; the node only ever sends
+        # plain UTF-8, so every other encoding (and gzip, which the script never
+        # requests or decodes) must fail closed.
+        for name, body in (("array", json.dumps(json.loads(inventory)["spaces"]).encode()),
+                           ("truncated", inventory[:-1]), ("empty", b""), ("string", b'"spaces"'),
+                           ("html", b"<html><body>502 Bad Gateway</body></html>"),
+                           ("invalid-utf8", b"\xff" * 70000),
+                           ("utf8-bom", b"\xef\xbb\xbf" + inventory),
+                           ("utf16", text.encode("utf-16")), ("utf16-le", text.encode("utf-16-le")),
+                           ("utf32-le", text.encode("utf-32-le")),
+                           ("gzip", gzip.compress(inventory))):
+            with self.subTest(case=name):
+                self.calls.clear()
+                self.usage_body = body
+                code, logs = self.run_main()
+                self.assertEqual(code, 1)
+                self.assertIn("quota API returned an invalid response", logs)
+                self.assertEqual(self.calls, [("node.tinycloud.xyz", "GET", "/admin/usage")])
+        self.calls.clear()
+        self.usage_body = inventory
+        self.assertEqual(self.run_main()[0], 0, "Plain UTF-8 JSON is still accepted")
+
     def test_wrong_owner_config_fails_before_any_request(self):
         document = json.loads(self.config.read_text())
         document["overrides"] = {