diff --git a/docs/releasing.md b/docs/releasing.md index f033fee6..1cbf8f24 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -154,7 +154,10 @@ CVM or whitelist mutation. `authority=node-override` currently applies the reviewed limits after the last restart and required health. Both exact spaces must exist once with -positive usage in authenticated `/admin/usage`. Before ANY PUT, both billing +positive usage in authenticated `/admin/usage`, which lists every space on +the node unpaginated (~107 bytes a space); the script reads up to 16 MiB of +it and fails closed above that, while other quota bodies stay capped at +64 KiB. Before ANY PUT, both billing storage-comp reads must prove no active comp: 404 logs only `billing storage-comp unavailable`; valid `active:false` also permits the legacy restore. Active comp fails `authority mismatch: flip to billing`. diff --git a/scripts/ci/reconcile-quota.py b/scripts/ci/reconcile-quota.py index d5eab63d..7a16046c 100755 --- a/scripts/ci/reconcile-quota.py +++ b/scripts/ci/reconcile-quota.py @@ -13,6 +13,11 @@ OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412" LIMITS = {"applications": 5368709120, "secrets": 268721121} NODE_HOST = "node.tinycloud.xyz" +# Per-space and billing bodies are a few hundred bytes. GET /admin/usage lists +# every space on the node, unpaginated, at ~107 bytes a row (~1,034 spaces, +# ~110 KiB, in October 2026): bounded at 16 MiB (~150,000 spaces). +RESPONSE_CAP = 64 * 1024 +USAGE_RESPONSE_CAP = 16 * 1024 * 1024 class QuotaError(Exception): @@ -67,7 +72,19 @@ def credentials(): return node, billing, (url.hostname, port) -def request_json(host, method, path, secret, limit=None, port=None, allow_404=False): +def read_bounded(response, cap): + """Stream at most cap+1 bytes; the caller rejects a body longer than cap.""" + chunks, size = [], 0 + while size <= cap: + chunk = response.read(min(64 * 1024, cap + 1 - size)) + if not chunk: + break + chunks.append(chunk) + size += len(chunk) + return b"".join(chunks) + + +def request_json(host, method, path, secret, limit=None, port=None, allow_404=False, cap=RESPONSE_CAP): """Fixed TLS requests, no redirects; retry transport only, never HTTP errors.""" headers = {"Authorization": "Bearer " + secret, "Content-Type": "application/json"} body = None if limit is None else json.dumps({"limit_bytes": limit}).encode() @@ -80,7 +97,7 @@ def request_json(host, method, path, secret, limit=None, port=None, allow_404=Fa connection = http.client.HTTPSConnection(host, **kwargs) connection.request(method, path, body=body, headers=headers) response = connection.getresponse() - payload = response.read(65537) + payload = read_bounded(response, cap) except (OSError, http.client.HTTPException): if attempt == 2: raise QuotaError("quota transport failed after bounded attempts") from None @@ -94,7 +111,8 @@ def request_json(host, method, path, secret, limit=None, port=None, allow_404=Fa if response.status != 200: raise QuotaError("quota API rejected the request") try: - result = json.loads(payload) if len(payload) <= 65536 else None + # Strict UTF-8: json.loads(bytes) would auto-accept a BOM, UTF-16 or UTF-32. + result = json.loads(payload.decode("utf-8")) if len(payload) <= cap else None if not isinstance(result, dict): raise ValueError return result @@ -104,7 +122,7 @@ def request_json(host, method, path, secret, limit=None, port=None, allow_404=Fa def prove_inventory(overrides, node_secret): """A quota GET echoes arbitrary IDs; only usage enumerates existing spaces.""" - inventory = request_json(NODE_HOST, "GET", "/admin/usage", node_secret) + inventory = request_json(NODE_HOST, "GET", "/admin/usage", node_secret, cap=USAGE_RESPONSE_CAP) rows = inventory.get("spaces") if (not isinstance(rows, list) or type(inventory.get("count")) is not int or inventory["count"] != len(rows)): diff --git a/scripts/ci/test-deploy-recovery.py b/scripts/ci/test-deploy-recovery.py index 37654593..4df46cdf 100755 --- a/scripts/ci/test-deploy-recovery.py +++ b/scripts/ci/test-deploy-recovery.py @@ -117,7 +117,7 @@ def event(value): else: raise SystemExit('Unexpected CLI boundary') ''' -HTTP = r'''import http.client, json, os, socket +HTTP = r'''import http.client, io, json, os, random, socket from pathlib import Path from urllib.parse import unquote socket.create_connection=lambda *a,**k: (_ for _ in ()).throw(AssertionError('Real network forbidden')) @@ -147,12 +147,17 @@ def getresponse(self): payload={'active':True,'comp_bytes':1,'account_limit_bytes':floor+100,'storage_limit_bytes':floor+100} else: status,payload=404,os.environ['QUOTA_BILLING_ADMIN_SECRET'] elif os.environ['FAIL_AT']=='quota': status,payload=503,os.environ['QUOTA_ADMIN_SECRET'] - elif self.path=='/admin/usage': payload={'count':len(targets),'spaces':[{'space_id':space,'usage_bytes':10} for space in targets]} + elif self.path=='/admin/usage': + # The node's real, unpaginated inventory: every space, ~107 bytes a row (prod: ~1,034 spaces). + rng=random.Random(959); spaces=[{'space_id':space,'usage_bytes':10} for space in targets] + spaces+=[{'space_id':'tinycloud:pkh:eip155:1:0x%040x:default'%rng.getrandbits(160),'usage_bytes':rng.choice([None,rng.randint(1,700000000)])} for _ in range(3000)] + payload={'spaces':spaces,'count':len(spaces)} else: value=data['values'].get(self.path) payload={'space_id':unquote(self.path.removeprefix('/admin/quota/')),'override_bytes':value,'effective_limit_bytes':value if value is not None else 123} + stream=io.BytesIO(json.dumps(payload).encode()) class Response: - def read(self,count): return json.dumps(payload).encode()[:count] + def read(self,count): return stream.read(count) response=Response(); response.status=status; return response def close(self): pass http.client.HTTPSConnection=Connection diff --git a/scripts/ci/test-quota-authority.py b/scripts/ci/test-quota-authority.py index bdc26c7b..95736323 100755 --- a/scripts/ci/test-quota-authority.py +++ b/scripts/ci/test-quota-authority.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 """Synthetic HTTP contracts for reviewed node/billing quota authority.""" import contextlib +import gzip import http.client import io import json from pathlib import Path +import random import secrets import tempfile import unittest @@ -19,6 +21,24 @@ OWNER = "tinycloud:pkh:eip155:1:0xd559CCd9EB87c530A9a349262669386dE93cf412" LIMITS = {OWNER + ":applications": 5368709120, OWNER + ":secrets": 268721121} CANARIES = [] +USAGE_CAP = 16 * 1024 * 1024 + + +def realistic_inventory(total): + """The node's real GET /admin/usage body (routes/admin.rs `UsageResponse`): + every space on the node, unpaginated, as compact serde JSON with `spaces` + before `count`, sorted by usage descending with unknown (null) last. + Production held about 1,034 spaces on 2026-10-12 (~107 bytes a row).""" + rng = random.Random(959) + rows = [{"space_id": space, "usage_bytes": usage} + for space, usage in zip(LIMITS, (640743590, 1048576))] + while len(rows) < total: + owner = "tinycloud:pkh:eip155:1:0x" + "".join(rng.choice("0123456789abcdef") for _ in range(40)) + name = rng.choice(("default", "applications", "secrets", "kv", "public", "notes")) + rows.append({"space_id": owner + ":" + name, + "usage_bytes": None if rng.random() < 0.4 else rng.randint(1, 700_000_000)}) + rows.sort(key=lambda row: (row["usage_bytes"] is None, -(row["usage_bytes"] or 0))) + return json.dumps({"spaces": rows, "count": len(rows)}, separators=(",", ":")).encode() class SafeResult(unittest.TextTestResult): @@ -46,6 +66,8 @@ def setUp(self): self.node_status = 200 self.bad_effective = False self.usage = [{"space_id": space, "usage_bytes": 10} for space in LIMITS] + self.usage_body = None + self.stream = None self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) self.config = Path(self.directory.name) / "quota.json" @@ -106,6 +128,8 @@ def getresponse(self): body = test.billing_body[space] if status == 200 else test.billing_secret elif test.node_status != 200: status, body = test.node_status, test.node_secret + elif self.path == "/admin/usage" and test.usage_body is not None: + status, body = 200, test.usage_body elif self.path == "/admin/usage": status, body = 200, {"count": len(test.usage), "spaces": test.usage} else: @@ -115,10 +139,11 @@ def getresponse(self): "effective_limit_bytes": (value + int(test.bad_effective)) if value is not None else 123, "current_usage_bytes": 10} - class Response: - def read(self, count): - return json.dumps(body).encode()[:count] - response = Response() + # Stream semantics of http.client.HTTPResponse.read(amt): at most + # amt bytes per call, b"" once the body is exhausted. + stream = io.BytesIO(body if isinstance(body, bytes) else json.dumps(body).encode()) + test.stream = stream + response = type("Response", (), {"read": lambda _, count: stream.read(count)})() response.status = status return response @@ -318,6 +343,82 @@ def test_invalid_inventory_never_applies_either_override(self): self.assertEqual(self.run_main()[0], 1) self.assertEqual(self.puts(), []) + def test_prod_sized_inventory_drives_the_full_restore(self): + self.usage_body = realistic_inventory(3000) + self.assertGreater(len(self.usage_body), 4 * 65536) + for authority in ("node-override", "billing"): + with self.subTest(authority=authority): + self.calls.clear(); self.values.clear() + self.configure(authority) + if authority == "billing": + self.billing_active() + code, logs = self.run_main() + self.assertEqual(code, 0) + if authority == "billing": + self.assertIn("billing-authoritative", logs) + self.assertEqual(self.puts(), []) + else: + self.assertIn("Quota overrides verified", logs) + self.assertEqual(sorted(self.values.values()), sorted(LIMITS.values())) + self.assertEqual([call[1] for call in self.calls], ["GET"] * 3 + ["PUT", "GET", "PUT", "GET"]) + + def test_prod_sized_inventory_keeps_exact_target_invariants(self): + document = json.loads(realistic_inventory(3000)) + first = next(iter(LIMITS)) + target = next(row for row in document["spaces"] if row["space_id"] == first) + variants = { + "missing": [row for row in document["spaces"] if row is not target], + "duplicate": document["spaces"] + [dict(target)], + "unknown-usage": [dict(row, usage_bytes=None) if row is target else row for row in document["spaces"]], + } + for name, rows in variants.items(): + with self.subTest(case=name): + self.calls.clear() + self.usage_body = json.dumps({"spaces": rows, "count": len(rows)}).encode() + code, logs = self.run_main() + self.assertEqual(code, 1) + self.assertIn("no overrides applied", logs) + self.assertEqual(self.puts(), []) + + def test_usage_body_is_bounded_at_the_documented_cap(self): + inventory = realistic_inventory(3000) + at_cap = inventory + b" " * (USAGE_CAP - len(inventory)) + self.usage_body = at_cap + code, logs = self.run_main() + self.assertEqual(code, 0, "A valid inventory of exactly the cap is accepted") + self.calls.clear(); self.values.clear() + self.usage_body = at_cap + b" " + code, logs = self.run_main() + self.assertEqual(code, 1) + self.assertIn("quota API returned an invalid response", logs) + self.assertEqual(self.calls, [("node.tinycloud.xyz", "GET", "/admin/usage")]) + self.assertLessEqual(self.stream.tell(), USAGE_CAP + 1, "The read stops at cap+1 bytes") + + def test_invalid_usage_body_fails_closed(self): + inventory = realistic_inventory(3000) + text = inventory.decode() + # json.loads(bytes) auto-detects UTF-8-sig/16/32; the node only ever sends + # plain UTF-8, so every other encoding (and gzip, which the script never + # requests or decodes) must fail closed. + for name, body in (("array", json.dumps(json.loads(inventory)["spaces"]).encode()), + ("truncated", inventory[:-1]), ("empty", b""), ("string", b'"spaces"'), + ("html", b"502 Bad Gateway"), + ("invalid-utf8", b"\xff" * 70000), + ("utf8-bom", b"\xef\xbb\xbf" + inventory), + ("utf16", text.encode("utf-16")), ("utf16-le", text.encode("utf-16-le")), + ("utf32-le", text.encode("utf-32-le")), + ("gzip", gzip.compress(inventory))): + with self.subTest(case=name): + self.calls.clear() + self.usage_body = body + code, logs = self.run_main() + self.assertEqual(code, 1) + self.assertIn("quota API returned an invalid response", logs) + self.assertEqual(self.calls, [("node.tinycloud.xyz", "GET", "/admin/usage")]) + self.calls.clear() + self.usage_body = inventory + self.assertEqual(self.run_main()[0], 0, "Plain UTF-8 JSON is still accepted") + def test_wrong_owner_config_fails_before_any_request(self): document = json.loads(self.config.read_text()) document["overrides"] = {