From f2b768fea97b24053cc49e33455c085e896d1a24 Mon Sep 17 00:00:00 2001 From: Miro Date: Wed, 14 Oct 2026 16:36:57 +0800 Subject: [PATCH 1/5] test(evolver): add isolated same-image campaign takeover validator --- services/evolver/README.md | 12 + .../evolver/scripts/takeover_fixture/app.py | 151 +++++ .../evolver/scripts/takeover_fixture/seed.py | 79 +++ .../scripts/takeover_fixture/stale_write.py | 67 ++ .../scripts/validate_campaign_takeover.py | 583 ++++++++++++++++++ .../tests/test_campaign_takeover_validator.py | 81 +++ 6 files changed, 973 insertions(+) create mode 100644 services/evolver/scripts/takeover_fixture/app.py create mode 100644 services/evolver/scripts/takeover_fixture/seed.py create mode 100644 services/evolver/scripts/takeover_fixture/stale_write.py create mode 100644 services/evolver/scripts/validate_campaign_takeover.py create mode 100644 services/evolver/tests/test_campaign_takeover_validator.py diff --git a/services/evolver/README.md b/services/evolver/README.md index 37739114..b18a0d70 100644 --- a/services/evolver/README.md +++ b/services/evolver/README.md @@ -253,3 +253,15 @@ Loop 每代两批模型请求分别持久化 dispatch、响应与费用预留， 请求已登记但响应未保存时，系统无法证明供应商是否计费：保留预留费用并以 `LOOP_PROPOSAL_RESPONSE_UNCONFIRMED` 停止自动重发，也不会启动同代另一批新请求来绕过此状态。应先核对用量，再明确准备并批准新实验；不清除旧预留或伪装为免费重试。响应已保存但缺少 usage 时仍保留未知费用。 该功能先迁移 `0068`（依赖 `0067`），再部署 Evolver；存在批次记录时禁止降级删除。独立 non-Loop campaign 不使用此缓存。53 项相关回归及离线进程注入已通过，生产部署/正常 owner 续跑尚待，详细范围见 [回执](../../docs/validation/e2-proposal-response-recovery-2026-10-12.json)。 + +### 固定镜像接手与旧租约写入检查 + +```bash +# 先缓存对应业务 SHA 的 linux/amd64 镜像；本命令只使用已有缓存的固定 image ID。 +DATABASE_URL="" .venv/bin/python scripts/validate_campaign_takeover.py \ + --image "ghcr.io/mirror29/inalpha-evolver:<40-character-business-sha>" +``` + +需要本地 PostgreSQL、Docker、Evolver dev 依赖和 migrations 虚拟环境。工具新建并迁移 UUID 审计库，启动两个完整 Evolver 应用；暂停旧进程并等待真实租约过期，观察新 token，再从该镜像直接尝试旧 token 的 `proposal_responses.complete()`。旧请求必须被拒绝，未知费用预留及阶段不得改变。外围 Data、签发与模型 HTTP 是合成夹具，不购买真实模型调用；不是正常 owner 或生产授权验收，也不证明旧 SDK 回调成功收包。 + +回执和私有日志写入仓库 `.tmp/e2-takeover-validation/`；审计库保留，只清理本次 UUID 容器及临时 env。工具明确记录清理错误，不把 Docker 查询失败当作容器已删除。单次检查仍不能替代完整故障矩阵、真实五代搜索和长期 Forward/holdout；剩余清单见 [当前状态](../../docs/04-current-state.md#未完成--下一步)。 diff --git a/services/evolver/scripts/takeover_fixture/app.py b/services/evolver/scripts/takeover_fixture/app.py new file mode 100644 index 00000000..ec6f0b73 --- /dev/null +++ b/services/evolver/scripts/takeover_fixture/app.py @@ -0,0 +1,151 @@ +"""Same-image Evolver probe: only peripheral HTTP fixtures and provider transport are replaced.""" + +import importlib +import json +import os +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import httpx +import jwt +import psycopg +from inalpha_shared_llm import LLMClient +from openai import AsyncOpenAI +from psycopg.rows import dict_row + +exchange = Path("/probe/exchange") +fixture = json.loads((exchange / "fixture.json").read_text()) +secret = os.environ["JWT_SECRET"] +mutex = threading.Lock() + + +def append(name, value): + with mutex: + with (exchange / name).open("a") as out: + out.write(value + "\n") + out.flush() + os.fsync(out.fileno()) + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def respond(self, payload, status=200): + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + if self.path == "/events/snapshots/" + fixture["snapshot"]["snapshot_id"]: + claims = jwt.decode( + self.headers.get("Authorization", "").removeprefix("Bearer "), + secret, + algorithms=["HS256"], + options={"verify_aud": False}, + ) + assert claims["token_purpose"] == "event_snapshot_read" + append("requests", "snapshot") + self.respond(fixture["snapshot"]) + elif self.path == "/api/internal/llm-config/config-1": + assert self.headers.get("Authorization") == "Bearer isolated-test-grant" + append("requests", "credential") + self.respond( + {"config_id": "config-1", "provider": "deepseek", "api_key": "offline-fixture-only"} + ) + else: + self.respond({}, 404) + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + if self.path == "/offline-provider": + append("calls", os.environ["INSTANCE_STAGE"]) + deadline = time.monotonic() + 90 + while not (exchange / "release").exists(): + if time.monotonic() > deadline: + self.respond({}, 503) + return + time.sleep(0.05) + self.respond( + { + "id": "offline-fixture", + "object": "chat.completion", + "created": 1, + "model": "deepseek-chat", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "[{}, {}, {}, {}]"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140}, + } + ) + elif self.path == "/internal/evolution-loops/" + fixture["loop_id"] + "/credential-grant": + claims = jwt.decode( + self.headers.get("Authorization", "").removeprefix("Bearer "), + secret, + algorithms=["HS256"], + audience="inalpha-orchestration", + ) + assert ( + claims["token_purpose"] == "evolution_loop_credential" + and claims["loop_id"] == fixture["loop_id"] + and claims["owner_account_id"] == fixture["owner"] + ) + db = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") + with psycopg.connect(db, row_factory=dict_row) as conn: + conn.execute("BEGIN READ ONLY") + conn.execute("SET LOCAL statement_timeout='5s'") + assert conn.info.dbname.startswith("inalpha_container_audit_") + row = conn.execute( + "SELECT lease_token,lease_expires_at>clock_timestamp() AS valid FROM evolution_campaigns WHERE campaign_id=%s", + (fixture["campaign_id"],), + ).fetchone() + assert row["valid"] and str(row["lease_token"]) == claims["lease_token"] + append("requests", "local-fixture-grant") + self.respond({"credential_grant": "isolated-test-grant"}) + else: + self.respond({}, 404) + + +server = ThreadingHTTPServer(("127.0.0.1", 9011), Handler) +threading.Thread(target=server.serve_forever, daemon=True).start() + + +async def isolated_client(self): + if self._client is None: + + async def transport(request): + assert request.url.host == "api.deepseek.com" + async with httpx.AsyncClient(trust_env=False, timeout=120) as client: + response = await client.post( + "http://127.0.0.1:9011/offline-provider", content=request.content + ) + return httpx.Response( + response.status_code, + content=response.content, + headers={"content-type": "application/json"}, + request=request, + ) + + self._client = AsyncOpenAI( + api_key="isolated-fixture-only", + base_url=self.settings.llm_base_url, + max_retries=0, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(transport), trust_env=False + ), + ) + return self._client + + +LLMClient._ensure_client = isolated_client + +app = importlib.import_module("inalpha_evolver.main").app diff --git a/services/evolver/scripts/takeover_fixture/seed.py b/services/evolver/scripts/takeover_fixture/seed.py new file mode 100644 index 00000000..a208ca69 --- /dev/null +++ b/services/evolver/scripts/takeover_fixture/seed.py @@ -0,0 +1,79 @@ +import asyncio +import json +import sys +from pathlib import Path +from uuid import uuid4 + +from inalpha_shared.db import close_pool, get_conn, init_pool + +from inalpha_evolver.config import EvolverSettings +from inalpha_evolver.runtime import loop_campaign + +sys.path.insert(0, str(Path.cwd() / "services/evolver")) +from tests.test_loop_handoff import ready_loop + + +async def main(): + import os + + await init_pool(os.environ["DATABASE_URL"], min_size=1, max_size=2) + try: + async with get_conn() as conn: + worker, _, snapshot = await ready_loop(conn) + types = [ + "listing", + "delisting", + "exploit", + "chain_halt", + "regulatory", + "upgrade", + "macro", + "other", + ] + snapshot["facts"] = [ + { + "fact_id": str(uuid4()), + "event_type": kind, + "severity": 0.8, + "confidence": 0.9, + "effective_at": "2026-01-01T07:00:00Z", + "available_at": "2026-01-01T07:00:00Z", + "assets": ["BTC"], + "asset_ids": ["asset:BTC"], + "extractor_version": "deterministic-event-extractor-v3", + } + for kind in types + ] + + async def fetch(*args, **kwargs): + return snapshot + + loop_campaign.fetch_event_snapshot = fetch + campaign = await loop_campaign.handoff_campaign( + worker, EvolverSettings(EVENT_EVOLUTION_ENABLED=True) + ) + async with get_conn() as conn: + await conn.execute( + "UPDATE evolution_campaigns SET requested_by_sub=%s WHERE campaign_id=%s", + (str(worker["owner_account_id"]), campaign), + ) + await conn.execute( + "UPDATE evolution_loops SET requested_by_sub=%s WHERE loop_id=%s", + (str(worker["owner_account_id"]), worker["loop_id"]), + ) + await asyncio.to_thread( + Path(sys.argv[1]).write_text, + json.dumps( + { + "campaign_id": str(campaign), + "owner": str(worker["owner_account_id"]), + "loop_id": str(worker["loop_id"]), + "snapshot": snapshot, + } + ), + ) + finally: + await close_pool() + + +asyncio.run(main()) diff --git a/services/evolver/scripts/takeover_fixture/stale_write.py b/services/evolver/scripts/takeover_fixture/stale_write.py new file mode 100644 index 00000000..5c5e0f85 --- /dev/null +++ b/services/evolver/scripts/takeover_fixture/stale_write.py @@ -0,0 +1,67 @@ +"""Attempt the real checkpoint write with an old lease in the isolated business image.""" + +import asyncio +import json +import os +from decimal import Decimal +from pathlib import Path +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +from inalpha_evolver.storage import loop_authorizations, proposal_responses + + +async def main(): + fixture = json.loads( + await asyncio.to_thread(Path("/probe/exchange/stale_scope.json").read_text) + ) + url = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") + async with await psycopg.AsyncConnection.connect(url, row_factory=dict_row) as conn: + assert conn.info.dbname.startswith("inalpha_container_audit_") + await conn.execute("SET LOCAL statement_timeout='5s'") + row = await conn.execute( + "SELECT status,lease_token,lease_expires_at,clock_timestamp() AS observed_at FROM evolution_campaigns WHERE campaign_id=%s AND owner_account_id=%s", + (UUID(fixture["campaign_id"]), UUID(fixture["owner"])), + ) + live = await row.fetchone() + assert ( + live["status"] == "replaying" and str(live["lease_token"]) == fixture["new_lease_token"] + ) + assert ( + str(live["lease_token"]) != fixture["old_lease_token"] + and live["lease_expires_at"] > live["observed_at"] + ) + try: + await proposal_responses.complete( + conn, + scope={ + "loop_id": UUID(fixture["loop_id"]), + "owner_account_id": UUID(fixture["owner"]), + "lease_token": UUID(fixture["old_lease_token"]), + "phase": "campaign", + }, + reservation_id=UUID(fixture["reservation_id"]), + response={"text": "offline stale response"}, + actual=Decimal("0.000001"), + ) + except loop_authorizations.LoopAuthorizationUnavailable as exc: + await conn.rollback() + print( + json.dumps( + { + "result": "rejected", + "code": exc.code, + "operation": "proposal_responses.complete", + "active_lease_at_attempt": {k: str(v) for k, v in live.items()}, + "scope": "direct same-image storage write attempt, not an old SDK callback", + } + ) + ) + return + await conn.rollback() + raise AssertionError("stale checkpoint write accepted") + + +asyncio.run(main()) diff --git a/services/evolver/scripts/validate_campaign_takeover.py b/services/evolver/scripts/validate_campaign_takeover.py new file mode 100644 index 00000000..86f7689c --- /dev/null +++ b/services/evolver/scripts/validate_campaign_takeover.py @@ -0,0 +1,583 @@ +"""Validate campaign takeover in a fixed local Evolver image, without paid calls. + +Requires Docker, local PostgreSQL and installed dev/migration environments. +Creates a fresh migrated audit database and keeps it. Peripheral services are +synthetic HTTP fixtures. Does not authorize production or prove old SDK receipt. +""" + +import argparse +import hashlib +import json +import os +import re +import secrets +import signal +import socket +import subprocess +import sys +import time +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.request import urlopen +from uuid import uuid4 + +import psycopg +from dotenv import dotenv_values +from psycopg import sql +from psycopg.rows import dict_row + + +def validate_image(value): + """Require an immutable own-project business tag, never latest or a foreign registry.""" + if not re.fullmatch(r"ghcr\.io/mirror29/inalpha-evolver:[a-f0-9]{40}", value): + raise ValueError("expected fixed ghcr.io/mirror29/inalpha-evolver business SHA") + return value + + +def container_database_url(url): + """Map only the validated local database address, preserving optional credentials.""" + parsed = urlsplit(url) + if parsed.hostname not in {"localhost", "127.0.0.1", "::1"}: + raise ValueError("container validation requires a validated local URL") + credentials = parsed.netloc.rsplit("@", 1)[0] + "@" if "@" in parsed.netloc else "" + options = [(k, v) for k, v in parse_qsl(parsed.query) if k != "hostaddr"] + return urlunsplit( + parsed._replace( + netloc=credentials + "host.docker.internal:" + str(parsed.port), + query=urlencode(options), + ) + ) + + +def cleanup_owned_process(name, process): + """Bound each owned-process cleanup step without skipping later resources.""" + record = {"exit_code": None, "errors": []} + if process is None: + return record + for sig in ("CONT", "TERM"): + try: + subprocess.run( + ["docker", "kill", "--signal", sig, name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + except (subprocess.TimeoutExpired, OSError) as exc: + record["errors"].append(type(exc).__name__) + try: + if process.poll() is None: + process.wait(timeout=30) + except (subprocess.TimeoutExpired, OSError) as exc: + record["errors"].append(type(exc).__name__) + if process.poll() is None: + try: + subprocess.run( + ["docker", "kill", "--signal", "KILL", name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + process.wait(timeout=10) + except (subprocess.TimeoutExpired, OSError) as exc: + record["errors"].append(type(exc).__name__) + record["exit_code"] = process.poll() + return record + + +def terminate_validation(signum, _frame): + """Unwind through owned-container cleanup on SIGTERM.""" + raise SystemExit(128 + signum) + + +def main(): + """Run only in an isolated database; retain evidence and remove owned containers.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--image", + required=True, + help="Fixed own Evolver image tagged with its 40-character business SHA", + ) + args = parser.parse_args() + image = validate_image(args.image) + signal.signal(signal.SIGTERM, terminate_validation) + root = Path(__file__).resolve().parents[3] + sys.path.insert(0, str(root / "services/evolver/scripts")) + from validate_durable_loop import local_database_urls + + name = "inalpha_container_audit_" + uuid4().hex + source_url = os.environ.get("DATABASE_URL") or dotenv_values(root / ".env").get("DATABASE_URL") + if not source_url: + raise ValueError("local DATABASE_URL required") + admin, url = local_database_urls(source_url, name) + raw = url.replace("postgresql+psycopg://", "postgresql://") + logs = root / ".tmp/e2-takeover-validation" / name + logs.mkdir(parents=True, mode=0o700) + probe_dir = root / "services/evolver/scripts/takeover_fixture" + secret = secrets.token_urlsafe(32) + fixture = {} + exchange = logs / "exchange" + exchange.mkdir(mode=0o777) + exchange.chmod(0o777) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + endpoint = "http://127.0.0.1:9011" + env = {k: os.environ[k] for k in ("PATH", "LANG", "TMPDIR") if k in os.environ} + env.update( + DATABASE_URL=url, + JWT_SECRET=secret, + EVENT_EVOLUTION_ENABLED="true", + CAMPAIGN_LEASE_TTL_S="30", + EVOLVER_JOB_TIMEOUT_S="60", + CANDIDATE_EVALUATION_CONCURRENCY="2", + EVOLVER_POOL_SIZE="5", + DATA_SERVICE_URL=endpoint, + PAPER_SERVICE_URL=endpoint, + DASHBOARD_SERVICE_URL=endpoint, + ORCHESTRATION_SERVICE_URL=endpoint, + FIXTURE_URL=endpoint, + OMP_NUM_THREADS="1", + OPENBLAS_NUM_THREADS="1", + VECLIB_MAXIMUM_THREADS="1", + ) + python = str(root / "services/evolver/.venv/bin/python") + child = None + old_child = None + old_name = None + lock = None + container_name = "inalpha-e2-response-" + uuid4().hex + container_env = {k: v for k, v in env.items() if k not in ("PATH", "LANG", "TMPDIR")} + + container_env["DATABASE_URL"] = container_database_url(url) + image_id = subprocess.check_output( + ["docker", "image", "inspect", image, "--format", "{{.Id}}"], text=True, timeout=10 + ).strip() + if not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id): + raise ValueError("invalid cached Docker image ID") + for key in ( + "DATA_SERVICE_URL", + "PAPER_SERVICE_URL", + "DASHBOARD_SERVICE_URL", + "ORCHESTRATION_SERVICE_URL", + "FIXTURE_URL", + ): + container_env[key] = endpoint + env_path = logs / "container.env" + fd = os.open(env_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as f: + for k, v in container_env.items(): + assert "\n" not in v + f.write(k + "=" + v + "\n") + report = { + "database": name, + "result": "failed", + "scope": "full actual Evolver application lifespan and dispatchers; provider SDK HTTP transport replaced, synthetic local signer/Dashboard/Data; actual fixed business image, local AMD64 emulation, not real owner/production", + } + observer_conn = None + observer_transient_failures = [] + + def observe_once(): + nonlocal observer_conn + if observer_conn is None or observer_conn.closed: + observer_conn = psycopg.connect(raw, row_factory=dict_row, autocommit=True) + with observer_conn.transaction(): + observer_conn.execute("SET TRANSACTION READ ONLY") + observer_conn.execute("SET LOCAL statement_timeout='5s'") + row = dict( + observer_conn.execute( + "SELECT status,active_generation,lease_token,lease_expires_at,locked_candidate_id,forward_sandbox_id,holdout_consumed_at,failure_code FROM evolution_campaigns WHERE campaign_id=%s", + (fixture["campaign_id"],), + ).fetchone() + ) + row["observed_at"] = observer_conn.execute("SELECT clock_timestamp() AS t").fetchone()[ + "t" + ] + row["proposal_count"] = ( + observer_conn.execute( + "SELECT count(*) AS n FROM evolution_proposal_checkpoints WHERE campaign_id=%s", + (fixture["campaign_id"],), + ).fetchone()["n"] + if lock is None + else "locked" + ) + row["stored_response_count"] = observer_conn.execute( + "SELECT count(*) AS n FROM evolution_loop_proposal_responses WHERE loop_id=%s AND response IS NOT NULL", + (fixture["loop_id"],), + ).fetchone()["n"] + row["response_count"] = observer_conn.execute( + "SELECT count(*) AS n FROM evolution_loop_proposal_responses WHERE loop_id=%s", + (fixture["loop_id"],), + ).fetchone()["n"] + row["succeeded"] = observer_conn.execute( + "SELECT count(*) AS n FROM evolution_implementations WHERE campaign_id=%s AND outcome='succeeded'", + (fixture["campaign_id"],), + ).fetchone()["n"] + row["ledger"] = dict( + observer_conn.execute( + "SELECT spent_usd,reserved_usd,max_cost_usd,(SELECT count(*) FROM evolution_loop_cost_reservations r WHERE r.loop_id=a.loop_id) AS attempts FROM evolution_loop_authorizations a WHERE loop_id=%s", + (fixture["loop_id"],), + ).fetchone() + ) + row["expired"] = observer_conn.execute( + "SELECT lease_expires_at= 2: + break + assert child.poll() is None + time.sleep(0.1) + assert (exchange / "calls").exists() and len( + (exchange / "calls").read_text().splitlines() + ) == 2 + before = observe() + assert before["response_count"] == 2 and before["stored_response_count"] == 0 + assert before["ledger"]["attempts"] == 2 and before["ledger"]["spent_usd"] == 0 + assert ( + before["ledger"]["reserved_usd"] > 0 + and before["ledger"]["max_cost_usd"] > before["ledger"]["reserved_usd"] * 2 + ) + kill_container("STOP") + report["sigstop_completed_at"] = datetime.now(UTC).isoformat() + old_child = child + old_name = container_name + report["fault"] = ( + "SIGSTOP old application; wait natural lease expiry; second application claims" + ) + before = observe() + assert before["proposal_count"] == 0 + report["before"] = before + deadline = time.monotonic() + 45 + while not observe()["expired"]: + if time.monotonic() > deadline: + raise TimeoutError("lease did not expire") + time.sleep(0.5) + expired = observe() + assert ( + expired["expired"] is True and expired["observed_at"] > expired["lease_expires_at"] + ) + report["expired_lease_observation"] = expired + lock = psycopg.connect(raw) + lock.execute("SET LOCAL statement_timeout='5s'") + lock.execute("LOCK TABLE evolution_loop_proposal_responses IN ACCESS EXCLUSIVE MODE") + container_name = "inalpha-e2-response-" + uuid4().hex + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + report["second_application_started_at"] = datetime.now(UTC).isoformat() + child = start(out, "after") + report["restart_health"] = health() + deadline = time.monotonic() + 30 + with psycopg.connect(raw, row_factory=dict_row) as claim_conn: + while time.monotonic() < deadline: + with claim_conn.transaction(): + claim_conn.execute("SET TRANSACTION READ ONLY") + claim_conn.execute("SET LOCAL statement_timeout='5s'") + claim = dict( + claim_conn.execute( + "SELECT status,lease_token,lease_expires_at,updated_at,clock_timestamp() AS observed_at FROM evolution_campaigns WHERE campaign_id=%s", + (fixture["campaign_id"],), + ).fetchone() + ) + if ( + claim["lease_token"] is not None + and claim["lease_token"] != before["lease_token"] + ): + break + time.sleep(0.1) + assert ( + claim["lease_token"] is not None + and claim["lease_token"] != before["lease_token"] + and claim["status"] == "replaying" + ) + assert ( + claim["updated_at"] > expired["lease_expires_at"] + and claim["observed_at"] > expired["observed_at"] + ) + report["second_application_claim"] = claim + with psycopg.connect(raw, row_factory=dict_row) as stale_conn: + stale_conn.execute("SET TRANSACTION READ ONLY") + stale_conn.execute("SET LOCAL statement_timeout='5s'") + # The response table is locked, so read the independent reservation ledger. + reservation = stale_conn.execute( + "SELECT reservation_id FROM evolution_loop_cost_reservations WHERE loop_id=%s ORDER BY reservation_id LIMIT 1", + (fixture["loop_id"],), + ).fetchone() + (exchange / "stale_scope.json").write_text( + json.dumps( + { + **fixture, + "old_lease_token": str(before["lease_token"]), + "reservation_id": str(reservation["reservation_id"]), + "new_lease_token": str(claim["lease_token"]), + } + ) + ) + stale_run = subprocess.run( + ["docker", "exec", container_name, ".venv/bin/python", "/probe/stale_write.py"], + capture_output=True, + text=True, + timeout=20, + ) + (logs / "stale-write.log").write_text(stale_run.stdout + stale_run.stderr) + if stale_run.returncode != 0: + raise RuntimeError("isolated stale-write probe failed; see private subprocess log") + report["direct_stale_write"] = json.loads(stale_run.stdout.strip()) + assert report["direct_stale_write"]["result"] == "rejected" + lock.rollback() + lock.close() + lock = None + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + after = observe() + if after["status"] in ( + "insufficient_evidence", + "failed", + "candidate_locked", + "waiting_forward", + ): + break + assert child.poll() is None + time.sleep(0.5) + calls = (exchange / "calls").read_text().splitlines() + requests = (exchange / "requests").read_text().splitlines() + report["after"] = after + report["offline_provider_calls"] = calls + report["local_peripheral_requests"] = requests + assert ( + after["status"] == "failed" + and after["failure_code"] == "LOOP_PROPOSAL_RESPONSE_UNCONFIRMED" + ) + assert after["active_generation"] == 1 and after["succeeded"] == 0 + assert after["proposal_count"] == 0 and after["stored_response_count"] == 0 + assert after["response_count"] == 2 and after["lease_token"] != before["lease_token"] + assert len(calls) == 2 and calls.count("before") == 2 + assert after["ledger"] == before["ledger"] + subprocess.run( + ["docker", "kill", "--signal", "CONT", old_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + timeout=10, + ) + (exchange / "release").touch() + report["old_application_resumed_and_fixture_released_at"] = datetime.now( + UTC + ).isoformat() + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + final = observe() + assert ( + final["status"] == after["status"] + and final["failure_code"] == after["failure_code"] + ) + assert ( + final["ledger"] == after["ledger"] + and final["response_count"] == 2 + and final["stored_response_count"] == 0 + ) + assert ( + final["proposal_count"] == 0 + and final["succeeded"] == 0 + and final["lease_token"] == after["lease_token"] + ) + assert (exchange / "calls").read_text().splitlines() == calls + assert old_child.poll() is None and child.poll() is None + time.sleep(0.5) + report["after_old_worker_return"] = final + report["late_response_observation_seconds"] = 20 + report["purpose"] = ( + "Observe takeover/resume invariance and direct old-token checkpoint write rejection; not successful old SDK receipt or full fencing acceptance" + ) + report["result"] = "passed" + except Exception as exc: + report["error_type"] = type(exc).__name__ + finally: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + report["cleanup_errors"] = [] + for connection in (lock, observer_conn): + if connection is not None: + try: + connection.close() + except psycopg.Error as exc: + report["cleanup_errors"].append(type(exc).__name__) + report["observer_transient_failures"] = observer_transient_failures + report["old_worker_cleanup"] = cleanup_owned_process(old_name, old_child) + if child is not old_child: + report["new_worker_cleanup"] = cleanup_owned_process(container_name, child) + try: + env_path.unlink(missing_ok=True) + except OSError as exc: + report["cleanup_errors"].append(type(exc).__name__) + report["env_file_removed"] = not env_path.exists() + try: + check = subprocess.run( + ["docker", "ps", "-a", "--format", "{{.Names}}"], + capture_output=True, + text=True, + timeout=10, + ) + report["owned_container_removed"] = check.returncode == 0 and not ( + {container_name, old_name} & set(check.stdout.splitlines()) + ) + except (subprocess.TimeoutExpired, OSError) as exc: + report["owned_container_removed"] = False + report["cleanup_errors"].append(type(exc).__name__) + if not report["owned_container_removed"] or not report["env_file_removed"]: + report["result"] = "failed" + report["cleanup_error"] = "owned-resource cleanup not confirmed" + report["image"] = image + report["actual_image_id"] = image_id + report["runtime"] = ( + "local Docker AMD64 emulation on ARM host; CPUs2/memory4GB; no production container restarted" + ) + report["entrypoint_shim_sha256"] = hashlib.sha256( + (probe_dir / "app.py").read_bytes() + ).hexdigest() + (logs / "receipt.json").write_text(json.dumps(report, default=str, indent=2) + "\n") + print( + json.dumps( + { + "result": report["result"], + "error_type": report.get("error_type"), + "database": name, + "receipt": str(logs / "receipt.json"), + } + ) + ) + return 0 if report["result"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/evolver/tests/test_campaign_takeover_validator.py b/services/evolver/tests/test_campaign_takeover_validator.py new file mode 100644 index 00000000..38d009c5 --- /dev/null +++ b/services/evolver/tests/test_campaign_takeover_validator.py @@ -0,0 +1,81 @@ +"""Protect the offline container validator's image and local-connection boundaries.""" + +import importlib.util +from pathlib import Path + +import pytest + +path = Path(__file__).parents[1] / "scripts" / "validate_campaign_takeover.py" +spec = importlib.util.spec_from_file_location("takeover_validator", path) +validator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validator) + + +@pytest.mark.parametrize( + "image", + [ + "ghcr.io/mirror29/inalpha-evolver:latest", + "foreign/image:" + "a" * 40, + "ghcr.io/mirror29/inalpha-evolver:" + "a" * 39, + ], +) +def test_requires_fixed_own_business_image(image): + """Reject moving aliases and foreign images before starting containers.""" + with pytest.raises(ValueError): + validator.validate_image(image) + + +def test_container_connection_mapping_preserves_credentials_without_hostaddr_override(): + """Only remap the host; do not rewrite address-like values inside credentials.""" + result = validator.container_database_url( + "postgresql+psycopg://fixture:offline-127.0.0.1@localhost:5432/a?hostaddr=127.0.0.1&connect_timeout=5" + ) + assert ( + result + == "postgresql+psycopg://fixture:offline-127.0.0.1@host.docker.internal:5432/a?connect_timeout=5" + ) + assert ( + validator.container_database_url("postgresql+psycopg://localhost:5432/a?hostaddr=127.0.0.1") + == "postgresql+psycopg://host.docker.internal:5432/a" + ) + + +def test_remote_database_address_is_rejected(): + """A remote database must not be adapted into an apparently local target.""" + with pytest.raises(ValueError): + validator.container_database_url("postgresql+psycopg://remote.example:5432/a") + + +def test_cleanup_continues_after_signal_timeout(monkeypatch): + """A timed-out signal must not prevent the remaining cleanup sequence.""" + import subprocess + from unittest.mock import Mock + + process = Mock() + process.poll.side_effect = [None, None, 0] + process.wait.side_effect = [subprocess.TimeoutExpired("wait", 30), 0] + runner = Mock( + side_effect=[ + subprocess.TimeoutExpired("docker", 10), + Mock(returncode=0), + Mock(returncode=0), + ] + ) + monkeypatch.setattr(validator.subprocess, "run", runner) + result = validator.cleanup_owned_process("inalpha-e2-response-fixture", process) + assert result["exit_code"] == 0 + assert result["errors"] == ["TimeoutExpired", "TimeoutExpired"] + assert runner.call_args_list[-1].args[0][-2:] == ["KILL", "inalpha-e2-response-fixture"] + + +def test_cleanup_tolerates_already_exited_container(monkeypatch): + """Missing-container signal results do not abort receipt and env cleanup.""" + from unittest.mock import Mock + + process = Mock() + process.poll.return_value = 0 + monkeypatch.setattr(validator.subprocess, "run", Mock(return_value=Mock(returncode=1))) + assert validator.cleanup_owned_process("inalpha-e2-response-fixture", process) == { + "exit_code": 0, + "errors": [], + } From 900e11be381eda53657c4d05c79a395666ac3021 Mon Sep 17 00:00:00 2001 From: Miro Date: Wed, 14 Oct 2026 16:39:53 +0800 Subject: [PATCH 2/5] docs(e2): record maintained takeover validator acceptance --- docs/04-current-state.md | 2 + ...ntained-takeover-validator-2026-10-14.json | 209 ++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 docs/validation/e2-maintained-takeover-validator-2026-10-14.json diff --git a/docs/04-current-state.md b/docs/04-current-state.md index 001e1965..7ccee27c 100644 --- a/docs/04-current-state.md +++ b/docs/04-current-state.md @@ -39,6 +39,8 @@ Inalpha 是 AI agent 编排与多 Python kernel 的量化实验框架。D-12 与 | 在途响应未知时退出 | 已有费用/响应回归 | 同镜像重启及再次重启均停在`LOOP_PROPOSAL_RESPONSE_UNCONFIRMED`；调用总数2，未知预留0.0340608保持，额度1充足 | 未取得真实供应商对账与生产未知费用恢复证据 | | 其他故障、交接与一次性holdout | 已有相关状态机与自动化 | 部分源码进程/受控数据库证据；完整矩阵待补 | 生产恢复、冠军与长期正向链路未完成 | +已将双进程探索整理为 [固定镜像接手工具](../services/evolver/scripts/validate_campaign_takeover.py)，新增镜像/连接边界与清理超时、已退出竞争7项回归。最终同镜像运行在新租约仍为`replaying`且未过期时，直接旧token调用`proposal_responses.complete()`收到`LOOP_AUTHORIZATION_UNAVAILABLE`；预留0.0340608、非空响应0和阶段保持不变。两容器退出0，env删除及独立Docker清理确认；7项测试、Ruff/格式、一致性19/0/12通过。见 [维护工具回执](validation/e2-maintained-takeover-validator-2026-10-14.json)。这是实际存储调用拒绝证据，不是旧SDK回调收包、完整外围授权栈或生产fencing通过；其他故障场景尚未整理为维护套件。 + 同镜像双进程补充：旧应用SIGSTOP后租约自然过期，新应用接手并拒绝未知响应重发；恢复旧应用、释放离线响应后连续20秒观察费用预留、零非空响应、零提案及阶段均未变化。未直接采样旧SDK成功收包或旧complete()调用，不能区分心跳取消与写入fencing；仍不视为整个旧worker矩阵完成。见 [暂停worker接手回执](validation/e2-paused-worker-takeover-2026-10-14.json)。 同镜像回执：[已知响应恢复](validation/e2-same-image-response-recovery-2026-10-14.json) / [未知响应阻断](validation/e2-same-image-unknown-response-2026-10-14.json)。无效DSL使用既有回退，成功实现数不代表有效变异或改善；两项都是临时探索脚本，尚非CI维护套件。前期失败与观察端超时保留在回执，不删除审计库。 diff --git a/docs/validation/e2-maintained-takeover-validator-2026-10-14.json b/docs/validation/e2-maintained-takeover-validator-2026-10-14.json new file mode 100644 index 00000000..67b56687 --- /dev/null +++ b/docs/validation/e2-maintained-takeover-validator-2026-10-14.json @@ -0,0 +1,209 @@ +{ + "database": "inalpha_container_audit_a891ed653b0a4fcc9e8dddd31de5e225", + "result": "passed", + "scope": "full actual Evolver application lifespan and dispatchers; provider SDK HTTP transport replaced, synthetic local signer/Dashboard/Data; actual fixed business image, local AMD64 emulation, not real owner/production", + "initial_health": { + "status": "ok", + "service": "inalpha-evolver", + "event_evolution": "enabled" + }, + "sigstop_completed_at": "2026-10-14T08:36:33.618504+00:00", + "fault": "SIGSTOP old application; wait natural lease expiry; second application claims", + "before": { + "status": "replaying", + "active_generation": 1, + "lease_token": "01b2e32e-e00e-4328-bf34-9e8e2ae8f361", + "lease_expires_at": "2026-10-14 08:37:01.562946+00:00", + "locked_candidate_id": null, + "forward_sandbox_id": null, + "holdout_consumed_at": null, + "failure_code": null, + "observed_at": "2026-10-14 08:36:33.623580+00:00", + "proposal_count": 0, + "stored_response_count": 0, + "response_count": 2, + "succeeded": 0, + "ledger": { + "spent_usd": "0", + "reserved_usd": "0.0340608", + "max_cost_usd": "1", + "attempts": 2 + }, + "expired": false + }, + "expired_lease_observation": { + "status": "replaying", + "active_generation": 1, + "lease_token": "01b2e32e-e00e-4328-bf34-9e8e2ae8f361", + "lease_expires_at": "2026-10-14 08:37:01.562946+00:00", + "locked_candidate_id": null, + "forward_sandbox_id": null, + "holdout_consumed_at": null, + "failure_code": null, + "observed_at": "2026-10-14 08:37:01.890314+00:00", + "proposal_count": 0, + "stored_response_count": 0, + "response_count": 2, + "succeeded": 0, + "ledger": { + "spent_usd": "0", + "reserved_usd": "0.0340608", + "max_cost_usd": "1", + "attempts": 2 + }, + "expired": true + }, + "second_application_started_at": "2026-10-14T08:37:01.911612+00:00", + "restart_health": { + "status": "ok", + "service": "inalpha-evolver", + "event_evolution": "enabled" + }, + "second_application_claim": { + "status": "replaying", + "lease_token": "7475e1e6-49cf-4730-9734-40fa0a96a0f7", + "lease_expires_at": "2026-10-14 08:37:45.614037+00:00", + "updated_at": "2026-10-14 08:37:15.614044+00:00", + "observed_at": "2026-10-14 08:37:15.948605+00:00" + }, + "direct_stale_write": { + "result": "rejected", + "code": "LOOP_AUTHORIZATION_UNAVAILABLE", + "operation": "proposal_responses.complete", + "active_lease_at_attempt": { + "status": "replaying", + "lease_token": "7475e1e6-49cf-4730-9734-40fa0a96a0f7", + "lease_expires_at": "2026-10-14 08:37:45.614037+00:00", + "observed_at": "2026-10-14 08:37:19.070242+00:00" + }, + "scope": "direct same-image storage write attempt, not an old SDK callback" + }, + "after": { + "status": "failed", + "active_generation": 1, + "lease_token": null, + "lease_expires_at": null, + "locked_candidate_id": null, + "forward_sandbox_id": null, + "holdout_consumed_at": null, + "failure_code": "LOOP_PROPOSAL_RESPONSE_UNCONFIRMED", + "observed_at": "2026-10-14 08:37:19.795621+00:00", + "proposal_count": 0, + "stored_response_count": 0, + "response_count": 2, + "succeeded": 0, + "ledger": { + "spent_usd": "0", + "reserved_usd": "0.0340608", + "max_cost_usd": "1", + "attempts": 2 + }, + "expired": null + }, + "offline_provider_calls": [ + "before", + "before" + ], + "local_peripheral_requests": [ + "snapshot", + "local-fixture-grant", + "credential", + "snapshot", + "local-fixture-grant", + "credential" + ], + "old_application_resumed_and_fixture_released_at": "2026-10-14T08:37:19.917544+00:00", + "after_old_worker_return": { + "status": "failed", + "active_generation": 1, + "lease_token": null, + "lease_expires_at": null, + "locked_candidate_id": null, + "forward_sandbox_id": null, + "holdout_consumed_at": null, + "failure_code": "LOOP_PROPOSAL_RESPONSE_UNCONFIRMED", + "observed_at": "2026-10-14 08:37:39.842318+00:00", + "proposal_count": 0, + "stored_response_count": 0, + "response_count": 2, + "succeeded": 0, + "ledger": { + "spent_usd": "0", + "reserved_usd": "0.0340608", + "max_cost_usd": "1", + "attempts": 2 + }, + "expired": null + }, + "late_response_observation_seconds": 20, + "purpose": "Observe takeover/resume invariance and direct old-token checkpoint write rejection; not successful old SDK receipt or full fencing acceptance", + "cleanup_errors": [], + "observer_transient_failures": [], + "old_worker_cleanup": { + "exit_code": 0, + "errors": [] + }, + "new_worker_cleanup": { + "exit_code": 0, + "errors": [] + }, + "env_file_removed": true, + "owned_container_removed": true, + "image": "ghcr.io/mirror29/inalpha-evolver:a275f8be7679578e4aef62cffb3ff5c6f13a988e", + "actual_image_id": "sha256:5aadcaf79089ece40137a11ad39085c3f190b2eff645219e9c9324e19e5be7df", + "runtime": "local Docker AMD64 emulation on ARM host; CPUs2/memory4GB; no production container restarted", + "entrypoint_shim_sha256": "e025efe7315ee9c80196bad678673aba78b10b4a80a13698dd264be182202b90", + "harness_commit": "f2b768fe", + "business_sha": "a275f8be7679578e4aef62cffb3ff5c6f13a988e", + "new_production_model_calls": 0, + "production_e2_searches_started": 0, + "harness_files_sha256": { + "services/evolver/scripts/validate_campaign_takeover.py": "94feedae0d1320e5902ec6e02ef69a5241ef0664869e58cb517cd54111850abd", + "services/evolver/scripts/takeover_fixture/app.py": "e025efe7315ee9c80196bad678673aba78b10b4a80a13698dd264be182202b90", + "services/evolver/scripts/takeover_fixture/seed.py": "7c8fe7002efdb4d702298b8472b365b5f4683106c25f472e86fabab8347e59c6", + "services/evolver/scripts/takeover_fixture/stale_write.py": "58d6aca522353f310fee342f4ec8f732def09c53a92ec148f64851f8058bb845" + }, + "acceptance_limits": [ + "Direct complete() invocation runs inside the actual business image while a different current lease is live. This proves the observed storage-call rejection, not successful receipt by an old SDK callback.", + "Full Evolver lifespan and dispatchers with offline peripheral HTTP/provider fixtures, local AMD64 emulation on an ARM host. Not full peripheral service stack, normal-owner authorization, production recovery, strategy improvement or supplier billing.", + "Audit databases retained; only owned UUID containers and private temporary environment files removed.", + "Correct fresh migration chain through 0069; no legacy audit database upgrade or force stamp." + ], + "local_validation": { + "boundary_and_cleanup_tests": 7, + "ruff": "passed", + "ruff_format": "passed", + "consistency": { + "passed": 19, + "failed": 0, + "existing_warnings": 12 + } + }, + "development_attempts": [ + { + "database": "inalpha_container_audit_df417ccdcaca4dbbbc67c76d506bf52e", + "result": "failed", + "cause": "Fixture directory bind conflicted with writable exchange child mount; no app startup." + }, + { + "database": "inalpha_container_audit_9101162e638447edb2b60247d5594f33", + "result": "failed", + "cause": "Same fixture directory bind conflict; no app startup." + }, + { + "database": "inalpha_container_audit_1b3ded83010b44798fa9d94454888349", + "result": "failed", + "cause": "Observer closure used global instead of nonlocal after conversion to main; startup passed, observation failed." + }, + { + "database": "inalpha_container_audit_5f91711d698a473ab6ab202e0580d43f", + "result": "passed", + "scope": "Intermediate direct stale write probe before cleanup robustness revision." + }, + { + "database": "inalpha_container_audit_1bcfcca48d54466db4df125f25b20687", + "result": "passed", + "scope": "Cleanup revision probe before supported event-type fixture alignment." + } + ] +} From 1fae2a189f90ac5cf7c98792f343ed16e5ebca7a Mon Sep 17 00:00:00 2001 From: Miro Date: Wed, 14 Oct 2026 16:56:59 +0800 Subject: [PATCH 3/5] test(evolver): strengthen takeover controls and failure diagnostics --- services/evolver/README.md | 4 +- .../evolver/scripts/takeover_fixture/seed.py | 2 +- .../scripts/takeover_fixture/stale_write.py | 23 +++++- .../scripts/validate_campaign_takeover.py | 77 ++++++++++++++++--- .../tests/test_campaign_takeover_validator.py | 16 ++++ 5 files changed, 106 insertions(+), 16 deletions(-) diff --git a/services/evolver/README.md b/services/evolver/README.md index b18a0d70..86291434 100644 --- a/services/evolver/README.md +++ b/services/evolver/README.md @@ -262,6 +262,6 @@ DATABASE_URL="" .venv/bin/python scripts/validate_campaign --image "ghcr.io/mirror29/inalpha-evolver:<40-character-business-sha>" ``` -需要本地 PostgreSQL、Docker、Evolver dev 依赖和 migrations 虚拟环境。工具新建并迁移 UUID 审计库，启动两个完整 Evolver 应用；暂停旧进程并等待真实租约过期，观察新 token，再从该镜像直接尝试旧 token 的 `proposal_responses.complete()`。旧请求必须被拒绝，未知费用预留及阶段不得改变。外围 Data、签发与模型 HTTP 是合成夹具，不购买真实模型调用；不是正常 owner 或生产授权验收，也不证明旧 SDK 回调成功收包。 +需要本地 PostgreSQL、Docker、Evolver dev 依赖和 migrations 虚拟环境。工具新建并迁移 UUID 审计库，启动两个完整 Evolver 应用；暂停旧进程并等待真实租约过期，先以当前有效 token 对同一 reservation 执行并回滚 `proposal_responses.complete()` 作为正向对照；再观察接手的新 token，并尝试旧 token 的相同写入。旧请求必须被拒绝，未知费用预留及阶段不得改变。外围 Data、签发与模型 HTTP 是合成夹具，不购买真实模型调用；不是正常 owner 或生产授权验收，也不证明旧 SDK 回调成功收包。 -回执和私有日志写入仓库 `.tmp/e2-takeover-validation/`；审计库保留，只清理本次 UUID 容器及临时 env。工具明确记录清理错误，不把 Docker 查询失败当作容器已删除。单次检查仍不能替代完整故障矩阵、真实五代搜索和长期 Forward/holdout；剩余清单见 [当前状态](../../docs/04-current-state.md#未完成--下一步)。 +回执和私有日志写入仓库 `.tmp/e2-takeover-validation/`；审计库保留，只清理本次 UUID 容器及临时 env。工具明确记录清理错误，不把 Docker 查询失败当作容器已删除。临时隔离响应表锁用于观察领取状态，回执记录取得/释放时间；失败阶段写入回执，traceback 只保存在私有日志中。合成 baseline 故意复用 checkout 的 `tests.test_loop_handoff.ready_loop`（需要 dev 依赖），由路径定位和 CI 契约测试防止无声漂移，不是可安装的生产 seed 接口。单次检查仍不能替代完整故障矩阵、真实五代搜索和长期 Forward/holdout；剩余清单见 [当前状态](../../docs/04-current-state.md#未完成--下一步)。 diff --git a/services/evolver/scripts/takeover_fixture/seed.py b/services/evolver/scripts/takeover_fixture/seed.py index a208ca69..4477e706 100644 --- a/services/evolver/scripts/takeover_fixture/seed.py +++ b/services/evolver/scripts/takeover_fixture/seed.py @@ -9,7 +9,7 @@ from inalpha_evolver.config import EvolverSettings from inalpha_evolver.runtime import loop_campaign -sys.path.insert(0, str(Path.cwd() / "services/evolver")) +sys.path.insert(0, str(Path(__file__).resolve().parents[4] / "services/evolver")) from tests.test_loop_handoff import ready_loop diff --git a/services/evolver/scripts/takeover_fixture/stale_write.py b/services/evolver/scripts/takeover_fixture/stale_write.py index 5c5e0f85..7b4255d1 100644 --- a/services/evolver/scripts/takeover_fixture/stale_write.py +++ b/services/evolver/scripts/takeover_fixture/stale_write.py @@ -3,6 +3,7 @@ import asyncio import json import os +import sys from decimal import Decimal from pathlib import Path from uuid import UUID @@ -14,6 +15,7 @@ async def main(): + positive = "--positive-control" in sys.argv fixture = json.loads( await asyncio.to_thread(Path("/probe/exchange/stale_scope.json").read_text) ) @@ -29,10 +31,9 @@ async def main(): assert ( live["status"] == "replaying" and str(live["lease_token"]) == fixture["new_lease_token"] ) - assert ( - str(live["lease_token"]) != fixture["old_lease_token"] - and live["lease_expires_at"] > live["observed_at"] - ) + assert (positive or str(live["lease_token"]) != fixture["old_lease_token"]) and live[ + "lease_expires_at" + ] > live["observed_at"] try: await proposal_responses.complete( conn, @@ -47,6 +48,8 @@ async def main(): actual=Decimal("0.000001"), ) except loop_authorizations.LoopAuthorizationUnavailable as exc: + if positive: + raise AssertionError("live scope rejected") from exc await conn.rollback() print( json.dumps( @@ -61,6 +64,18 @@ async def main(): ) return await conn.rollback() + if positive: + print( + json.dumps( + { + "result": "accepted_rolled_back", + "operation": "proposal_responses.complete", + "active_lease_at_attempt": {k: str(v) for k, v in live.items()}, + "reservation_id": fixture["reservation_id"], + } + ) + ) + return raise AssertionError("stale checkpoint write accepted") diff --git a/services/evolver/scripts/validate_campaign_takeover.py b/services/evolver/scripts/validate_campaign_takeover.py index 86f7689c..54629b28 100644 --- a/services/evolver/scripts/validate_campaign_takeover.py +++ b/services/evolver/scripts/validate_campaign_takeover.py @@ -16,6 +16,7 @@ import subprocess import sys import time +import traceback from datetime import UTC, datetime from pathlib import Path from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit @@ -44,7 +45,7 @@ def container_database_url(url): options = [(k, v) for k, v in parse_qsl(parsed.query) if k != "hostaddr"] return urlunsplit( parsed._replace( - netloc=credentials + "host.docker.internal:" + str(parsed.port), + netloc=credentials + "host.docker.internal:" + str(parsed.port or 5432), query=urlencode(options), ) ) @@ -316,9 +317,11 @@ def health(): time.sleep(0.1) raise TimeoutError("health") + stage = "create_audit_database" try: with psycopg.connect(admin, autocommit=True) as conn: conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name))) + stage = "migrate_audit_database" with (logs / "migration.log").open("w") as out: subprocess.run( [ @@ -335,6 +338,7 @@ def health(): timeout=120, check=True, ) + stage = "seed_synthetic_baseline" with (logs / "seed.log").open("w") as out: subprocess.run( [python, str(probe_dir / "seed.py"), str(logs / "fixture.json")], @@ -348,8 +352,10 @@ def health(): fixture.update(json.loads((logs / "fixture.json").read_text())) (exchange / "fixture.json").write_text(json.dumps(fixture)) with (logs / "app.log").open("w") as out: + stage = "initial_app_health" child = start(out, "before") report["initial_health"] = health() + stage = "first_offline_calls" deadline = time.monotonic() + 60 while time.monotonic() < deadline: if (exchange / "calls").exists() and len( @@ -368,6 +374,48 @@ def health(): before["ledger"]["reserved_usd"] > 0 and before["ledger"]["max_cost_usd"] > before["ledger"]["reserved_usd"] * 2 ) + stage = "live_write_positive_control" + with psycopg.connect(raw, row_factory=dict_row) as control_conn: + control_conn.execute("SET TRANSACTION READ ONLY") + control_conn.execute("SET LOCAL statement_timeout='5s'") + reservation = control_conn.execute( + "SELECT reservation_id FROM evolution_loop_proposal_responses WHERE loop_id=%s AND response IS NULL ORDER BY batch LIMIT 1", + (fixture["loop_id"],), + ).fetchone() + (exchange / "stale_scope.json").write_text( + json.dumps( + { + **fixture, + "old_lease_token": str(before["lease_token"]), + "new_lease_token": str(before["lease_token"]), + "reservation_id": str(reservation["reservation_id"]), + } + ) + ) + control_run = subprocess.run( + [ + "docker", + "exec", + container_name, + ".venv/bin/python", + "/probe/stale_write.py", + "--positive-control", + ], + capture_output=True, + text=True, + timeout=20, + ) + (logs / "positive-control.log").write_text(control_run.stdout + control_run.stderr) + if control_run.returncode != 0: + raise RuntimeError("live checkpoint positive control failed") + report["live_write_positive_control"] = json.loads(control_run.stdout.strip()) + assert report["live_write_positive_control"]["result"] == "accepted_rolled_back" + after_control = observe() + assert ( + after_control["ledger"] == before["ledger"] + and after_control["stored_response_count"] == 0 + ) + stage = "pause_old_application" kill_container("STOP") report["sigstop_completed_at"] = datetime.now(UTC).isoformat() old_child = child @@ -378,6 +426,7 @@ def health(): before = observe() assert before["proposal_count"] == 0 report["before"] = before + stage = "observe_natural_expiry" deadline = time.monotonic() + 45 while not observe()["expired"]: if time.monotonic() > deadline: @@ -388,16 +437,20 @@ def health(): expired["expired"] is True and expired["observed_at"] > expired["lease_expires_at"] ) report["expired_lease_observation"] = expired + stage = "acquire_observation_lock" lock = psycopg.connect(raw) lock.execute("SET LOCAL statement_timeout='5s'") lock.execute("LOCK TABLE evolution_loop_proposal_responses IN ACCESS EXCLUSIVE MODE") + report["observation_lock_acquired_at"] = datetime.now(UTC).isoformat() container_name = "inalpha-e2-response-" + uuid4().hex with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] report["second_application_started_at"] = datetime.now(UTC).isoformat() + stage = "takeover_app_health" child = start(out, "after") report["restart_health"] = health() + stage = "observe_replacement_claim" deadline = time.monotonic() + 30 with psycopg.connect(raw, row_factory=dict_row) as claim_conn: while time.monotonic() < deadline: @@ -426,14 +479,6 @@ def health(): and claim["observed_at"] > expired["observed_at"] ) report["second_application_claim"] = claim - with psycopg.connect(raw, row_factory=dict_row) as stale_conn: - stale_conn.execute("SET TRANSACTION READ ONLY") - stale_conn.execute("SET LOCAL statement_timeout='5s'") - # The response table is locked, so read the independent reservation ledger. - reservation = stale_conn.execute( - "SELECT reservation_id FROM evolution_loop_cost_reservations WHERE loop_id=%s ORDER BY reservation_id LIMIT 1", - (fixture["loop_id"],), - ).fetchone() (exchange / "stale_scope.json").write_text( json.dumps( { @@ -444,6 +489,7 @@ def health(): } ) ) + stage = "stale_write_rejection" stale_run = subprocess.run( ["docker", "exec", container_name, ".venv/bin/python", "/probe/stale_write.py"], capture_output=True, @@ -458,6 +504,8 @@ def health(): lock.rollback() lock.close() lock = None + report["observation_lock_released_at"] = datetime.now(UTC).isoformat() + stage = "unknown_response_stop" deadline = time.monotonic() + 120 while time.monotonic() < deadline: after = observe() @@ -495,6 +543,7 @@ def health(): report["old_application_resumed_and_fixture_released_at"] = datetime.now( UTC ).isoformat() + stage = "resume_old_application_invariance" deadline = time.monotonic() + 20 while time.monotonic() < deadline: final = observe() @@ -523,6 +572,16 @@ def health(): report["result"] = "passed" except Exception as exc: report["error_type"] = type(exc).__name__ + report["failure_stage"] = stage + try: + failure_path = logs / "failure-traceback.log" + fd = os.open(failure_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as output: + traceback.print_exc(file=output) + report["private_failure_traceback"] = "failure-traceback.log" + except OSError: + report["private_failure_traceback"] = "not_written" + finally: signal.signal(signal.SIGTERM, signal.SIG_IGN) report["cleanup_errors"] = [] diff --git a/services/evolver/tests/test_campaign_takeover_validator.py b/services/evolver/tests/test_campaign_takeover_validator.py index 38d009c5..5b6fb9ae 100644 --- a/services/evolver/tests/test_campaign_takeover_validator.py +++ b/services/evolver/tests/test_campaign_takeover_validator.py @@ -79,3 +79,19 @@ def test_cleanup_tolerates_already_exited_container(monkeypatch): "exit_code": 0, "errors": [], } + + +def test_portless_local_database_uses_postgres_default(): + assert ( + validator.container_database_url("postgresql+psycopg://fixture@localhost/a") + == "postgresql+psycopg://fixture@host.docker.internal:5432/a" + ) + + +def test_reused_baseline_fixture_contract_remains_available(): + """Keep the intentionally reused dev fixture discoverable when tests are refactored.""" + import inspect + + from .test_loop_handoff import ready_loop + + assert inspect.iscoroutinefunction(ready_loop) From 66d214e09a1cebe6535e27a26292edce3cf7b0fe Mon Sep 17 00:00:00 2001 From: Miro Date: Wed, 14 Oct 2026 16:57:27 +0800 Subject: [PATCH 4/5] docs(e2): record positive-control and observation-lock evidence --- docs/04-current-state.md | 2 +- ...ntained-takeover-validator-2026-10-14.json | 78 ++++++++++++------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/docs/04-current-state.md b/docs/04-current-state.md index 7ccee27c..937e313d 100644 --- a/docs/04-current-state.md +++ b/docs/04-current-state.md @@ -39,7 +39,7 @@ Inalpha 是 AI agent 编排与多 Python kernel 的量化实验框架。D-12 与 | 在途响应未知时退出 | 已有费用/响应回归 | 同镜像重启及再次重启均停在`LOOP_PROPOSAL_RESPONSE_UNCONFIRMED`；调用总数2，未知预留0.0340608保持，额度1充足 | 未取得真实供应商对账与生产未知费用恢复证据 | | 其他故障、交接与一次性holdout | 已有相关状态机与自动化 | 部分源码进程/受控数据库证据；完整矩阵待补 | 生产恢复、冠军与长期正向链路未完成 | -已将双进程探索整理为 [固定镜像接手工具](../services/evolver/scripts/validate_campaign_takeover.py)，新增镜像/连接边界与清理超时、已退出竞争7项回归。最终同镜像运行在新租约仍为`replaying`且未过期时，直接旧token调用`proposal_responses.complete()`收到`LOOP_AUTHORIZATION_UNAVAILABLE`；预留0.0340608、非空响应0和阶段保持不变。两容器退出0，env删除及独立Docker清理确认；7项测试、Ruff/格式、一致性19/0/12通过。见 [维护工具回执](validation/e2-maintained-takeover-validator-2026-10-14.json)。这是实际存储调用拒绝证据，不是旧SDK回调收包、完整外围授权栈或生产fencing通过；其他故障场景尚未整理为维护套件。 +已将双进程探索整理为 [固定镜像接手工具](../services/evolver/scripts/validate_campaign_takeover.py)，新增镜像/连接边界与清理超时、已退出竞争和夹具契约9项回归。最终同镜像运行先以相同reservation/参数在有效租约下完成并回滚写入作为对照，再在新租约仍为`replaying`且未过期时，直接旧token调用`proposal_responses.complete()`收到`LOOP_AUTHORIZATION_UNAVAILABLE`；预留0.0340608、非空响应0和阶段保持不变。两容器退出0，env删除及独立Docker清理确认；9项测试、Ruff/格式、一致性19/0/12通过。见 [维护工具回执](validation/e2-maintained-takeover-validator-2026-10-14.json)。这是实际存储调用拒绝证据，不是旧SDK回调收包、完整外围授权栈或生产fencing通过；其他故障场景尚未整理为维护套件。 同镜像双进程补充：旧应用SIGSTOP后租约自然过期，新应用接手并拒绝未知响应重发；恢复旧应用、释放离线响应后连续20秒观察费用预留、零非空响应、零提案及阶段均未变化。未直接采样旧SDK成功收包或旧complete()调用，不能区分心跳取消与写入fencing；仍不视为整个旧worker矩阵完成。见 [暂停worker接手回执](validation/e2-paused-worker-takeover-2026-10-14.json)。 diff --git a/docs/validation/e2-maintained-takeover-validator-2026-10-14.json b/docs/validation/e2-maintained-takeover-validator-2026-10-14.json index 67b56687..d8f80e41 100644 --- a/docs/validation/e2-maintained-takeover-validator-2026-10-14.json +++ b/docs/validation/e2-maintained-takeover-validator-2026-10-14.json @@ -1,5 +1,5 @@ { - "database": "inalpha_container_audit_a891ed653b0a4fcc9e8dddd31de5e225", + "database": "inalpha_container_audit_0bd5a5582eb04c2b9815529a559b73ce", "result": "passed", "scope": "full actual Evolver application lifespan and dispatchers; provider SDK HTTP transport replaced, synthetic local signer/Dashboard/Data; actual fixed business image, local AMD64 emulation, not real owner/production", "initial_health": { @@ -7,18 +7,29 @@ "service": "inalpha-evolver", "event_evolution": "enabled" }, - "sigstop_completed_at": "2026-10-14T08:36:33.618504+00:00", + "live_write_positive_control": { + "result": "accepted_rolled_back", + "operation": "proposal_responses.complete", + "active_lease_at_attempt": { + "status": "replaying", + "lease_token": "f02e141a-8508-40f9-ab23-06f8bf0f5d1f", + "lease_expires_at": "2026-10-14 08:53:35.576676+00:00", + "observed_at": "2026-10-14 08:53:11.073594+00:00" + }, + "reservation_id": "4e42807f-7024-421b-82a2-73e2ea0c256e" + }, + "sigstop_completed_at": "2026-10-14T08:53:11.455161+00:00", "fault": "SIGSTOP old application; wait natural lease expiry; second application claims", "before": { "status": "replaying", "active_generation": 1, - "lease_token": "01b2e32e-e00e-4328-bf34-9e8e2ae8f361", - "lease_expires_at": "2026-10-14 08:37:01.562946+00:00", + "lease_token": "f02e141a-8508-40f9-ab23-06f8bf0f5d1f", + "lease_expires_at": "2026-10-14 08:53:35.576676+00:00", "locked_candidate_id": null, "forward_sandbox_id": null, "holdout_consumed_at": null, "failure_code": null, - "observed_at": "2026-10-14 08:36:33.623580+00:00", + "observed_at": "2026-10-14 08:53:11.457961+00:00", "proposal_count": 0, "stored_response_count": 0, "response_count": 2, @@ -34,13 +45,13 @@ "expired_lease_observation": { "status": "replaying", "active_generation": 1, - "lease_token": "01b2e32e-e00e-4328-bf34-9e8e2ae8f361", - "lease_expires_at": "2026-10-14 08:37:01.562946+00:00", + "lease_token": "f02e141a-8508-40f9-ab23-06f8bf0f5d1f", + "lease_expires_at": "2026-10-14 08:53:35.576676+00:00", "locked_candidate_id": null, "forward_sandbox_id": null, "holdout_consumed_at": null, "failure_code": null, - "observed_at": "2026-10-14 08:37:01.890314+00:00", + "observed_at": "2026-10-14 08:53:36.049514+00:00", "proposal_count": 0, "stored_response_count": 0, "response_count": 2, @@ -53,7 +64,8 @@ }, "expired": true }, - "second_application_started_at": "2026-10-14T08:37:01.911612+00:00", + "observation_lock_acquired_at": "2026-10-14T08:53:36.076120+00:00", + "second_application_started_at": "2026-10-14T08:53:36.076340+00:00", "restart_health": { "status": "ok", "service": "inalpha-evolver", @@ -61,10 +73,10 @@ }, "second_application_claim": { "status": "replaying", - "lease_token": "7475e1e6-49cf-4730-9734-40fa0a96a0f7", - "lease_expires_at": "2026-10-14 08:37:45.614037+00:00", - "updated_at": "2026-10-14 08:37:15.614044+00:00", - "observed_at": "2026-10-14 08:37:15.948605+00:00" + "lease_token": "85851308-ee2a-42d8-a55a-fbe1a48bca37", + "lease_expires_at": "2026-10-14 08:54:22.027023+00:00", + "updated_at": "2026-10-14 08:53:52.027031+00:00", + "observed_at": "2026-10-14 08:53:52.541338+00:00" }, "direct_stale_write": { "result": "rejected", @@ -72,12 +84,13 @@ "operation": "proposal_responses.complete", "active_lease_at_attempt": { "status": "replaying", - "lease_token": "7475e1e6-49cf-4730-9734-40fa0a96a0f7", - "lease_expires_at": "2026-10-14 08:37:45.614037+00:00", - "observed_at": "2026-10-14 08:37:19.070242+00:00" + "lease_token": "85851308-ee2a-42d8-a55a-fbe1a48bca37", + "lease_expires_at": "2026-10-14 08:54:22.027023+00:00", + "observed_at": "2026-10-14 08:53:56.905621+00:00" }, "scope": "direct same-image storage write attempt, not an old SDK callback" }, + "observation_lock_released_at": "2026-10-14T08:53:57.229046+00:00", "after": { "status": "failed", "active_generation": 1, @@ -87,7 +100,7 @@ "forward_sandbox_id": null, "holdout_consumed_at": null, "failure_code": "LOOP_PROPOSAL_RESPONSE_UNCONFIRMED", - "observed_at": "2026-10-14 08:37:19.795621+00:00", + "observed_at": "2026-10-14 08:53:57.810103+00:00", "proposal_count": 0, "stored_response_count": 0, "response_count": 2, @@ -112,7 +125,7 @@ "local-fixture-grant", "credential" ], - "old_application_resumed_and_fixture_released_at": "2026-10-14T08:37:19.917544+00:00", + "old_application_resumed_and_fixture_released_at": "2026-10-14T08:53:57.935548+00:00", "after_old_worker_return": { "status": "failed", "active_generation": 1, @@ -122,7 +135,7 @@ "forward_sandbox_id": null, "holdout_consumed_at": null, "failure_code": "LOOP_PROPOSAL_RESPONSE_UNCONFIRMED", - "observed_at": "2026-10-14 08:37:39.842318+00:00", + "observed_at": "2026-10-14 08:54:06.227804+00:00", "proposal_count": 0, "stored_response_count": 0, "response_count": 2, @@ -153,24 +166,18 @@ "actual_image_id": "sha256:5aadcaf79089ece40137a11ad39085c3f190b2eff645219e9c9324e19e5be7df", "runtime": "local Docker AMD64 emulation on ARM host; CPUs2/memory4GB; no production container restarted", "entrypoint_shim_sha256": "e025efe7315ee9c80196bad678673aba78b10b4a80a13698dd264be182202b90", - "harness_commit": "f2b768fe", "business_sha": "a275f8be7679578e4aef62cffb3ff5c6f13a988e", "new_production_model_calls": 0, "production_e2_searches_started": 0, - "harness_files_sha256": { - "services/evolver/scripts/validate_campaign_takeover.py": "94feedae0d1320e5902ec6e02ef69a5241ef0664869e58cb517cd54111850abd", - "services/evolver/scripts/takeover_fixture/app.py": "e025efe7315ee9c80196bad678673aba78b10b4a80a13698dd264be182202b90", - "services/evolver/scripts/takeover_fixture/seed.py": "7c8fe7002efdb4d702298b8472b365b5f4683106c25f472e86fabab8347e59c6", - "services/evolver/scripts/takeover_fixture/stale_write.py": "58d6aca522353f310fee342f4ec8f732def09c53a92ec148f64851f8058bb845" - }, "acceptance_limits": [ "Direct complete() invocation runs inside the actual business image while a different current lease is live. This proves the observed storage-call rejection, not successful receipt by an old SDK callback.", "Full Evolver lifespan and dispatchers with offline peripheral HTTP/provider fixtures, local AMD64 emulation on an ARM host. Not full peripheral service stack, normal-owner authorization, production recovery, strategy improvement or supplier billing.", "Audit databases retained; only owned UUID containers and private temporary environment files removed.", - "Correct fresh migration chain through 0069; no legacy audit database upgrade or force stamp." + "Correct fresh migration chain through 0069; no legacy audit database upgrade or force stamp.", + "An ACCESS EXCLUSIVE lock on the isolated proposal-response table holds recovery while the replacement claim is observed. Acquire/release UTC anchors are recorded; application-side blocking was not separately sampled. The lock is released before unknown-response termination and resumed-worker invariance checks. These timings are instrumented, not natural recovery latency." ], "local_validation": { - "boundary_and_cleanup_tests": 7, + "boundary_and_cleanup_tests": 9, "ruff": "passed", "ruff_format": "passed", "consistency": { @@ -204,6 +211,19 @@ "database": "inalpha_container_audit_1bcfcca48d54466db4df125f25b20687", "result": "passed", "scope": "Cleanup revision probe before supported event-type fixture alignment." + }, + { + "database": "inalpha_container_audit_a891ed653b0a4fcc9e8dddd31de5e225", + "result": "passed", + "scope": "Previous maintained probe without positive write control or lock time anchors; retained historical evidence." } - ] + ], + "harness_commit": "1fae2a18", + "harness_files_sha256": { + "services/evolver/scripts/validate_campaign_takeover.py": "120662d3f65d6811888acc40208218b1ef75ed469298530a23d9915ecbbd53a9", + "services/evolver/scripts/takeover_fixture/app.py": "e025efe7315ee9c80196bad678673aba78b10b4a80a13698dd264be182202b90", + "services/evolver/scripts/takeover_fixture/seed.py": "2c655702d2809a57c7ead5e63d2827aa86b1474d967377f764b3ba3c2a8426c4", + "services/evolver/scripts/takeover_fixture/stale_write.py": "753ef7d648fddd4899bb8cef112b4d87e5628b7a6b99089a11d48e2b53603c1c" + }, + "positive_control_scope": "The same reservation and arguments were accepted under the original live lease and rolled back; after natural expiry and replacement claim the old token was rejected. This isolates stale authority from invalid reservation/phase operands; it is not an old SDK callback." } From 8b2f4746dfae5cff04b479e7c5a26d2c3a7e91d7 Mon Sep 17 00:00:00 2001 From: Miro Date: Wed, 14 Oct 2026 16:59:55 +0800 Subject: [PATCH 5/5] docs(evolver): clarify positive-control execution order --- services/evolver/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/evolver/README.md b/services/evolver/README.md index 86291434..fcd866ac 100644 --- a/services/evolver/README.md +++ b/services/evolver/README.md @@ -262,6 +262,6 @@ DATABASE_URL="" .venv/bin/python scripts/validate_campaign --image "ghcr.io/mirror29/inalpha-evolver:<40-character-business-sha>" ``` -需要本地 PostgreSQL、Docker、Evolver dev 依赖和 migrations 虚拟环境。工具新建并迁移 UUID 审计库，启动两个完整 Evolver 应用；暂停旧进程并等待真实租约过期，先以当前有效 token 对同一 reservation 执行并回滚 `proposal_responses.complete()` 作为正向对照；再观察接手的新 token，并尝试旧 token 的相同写入。旧请求必须被拒绝，未知费用预留及阶段不得改变。外围 Data、签发与模型 HTTP 是合成夹具，不购买真实模型调用；不是正常 owner 或生产授权验收，也不证明旧 SDK 回调成功收包。 +需要本地 PostgreSQL、Docker、Evolver dev 依赖和 migrations 虚拟环境。工具新建并迁移 UUID 审计库，启动两个完整 Evolver 应用；先以当前有效 token 对同一 reservation 执行并回滚 `proposal_responses.complete()` 作为正向对照；再暂停旧进程，等待真实租约过期，观察接手的新 token，并尝试旧 token 的相同写入。旧请求必须被拒绝，未知费用预留及阶段不得改变。外围 Data、签发与模型 HTTP 是合成夹具，不购买真实模型调用；不是正常 owner 或生产授权验收，也不证明旧 SDK 回调成功收包。 回执和私有日志写入仓库 `.tmp/e2-takeover-validation/`；审计库保留，只清理本次 UUID 容器及临时 env。工具明确记录清理错误，不把 Docker 查询失败当作容器已删除。临时隔离响应表锁用于观察领取状态，回执记录取得/释放时间；失败阶段写入回执，traceback 只保存在私有日志中。合成 baseline 故意复用 checkout 的 `tests.test_loop_handoff.ready_loop`（需要 dev 依赖），由路径定位和 CI 契约测试防止无声漂移，不是可安装的生产 seed 接口。单次检查仍不能替代完整故障矩阵、真实五代搜索和长期 Forward/holdout；剩余清单见 [当前状态](../../docs/04-current-state.md#未完成--下一步)。