From b378341a30492af346da969f3fe72eccec6cd0a9 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 20 Sep 2026 05:22:13 +0800 Subject: [PATCH 1/6] ci: add opt-in paired race cache seed measurement --- .github/workflows/check-action-file.yaml | 3 + .github/workflows/race-seed-canary.yaml | 94 +++++++ .github/workflows/race-seed-sample.yaml | 119 ++++++++ actions/seed-go-caches/action.yaml | 4 + actions/seed-go-caches/seed.py | 22 +- actions/seed-go-caches/test_seed.py | 35 +++ docs/race-seed-canary.md | 75 +++++ scripts/race_seed_canary.py | 334 +++++++++++++++++++++++ scripts/race_seed_plan.py | 40 +++ scripts/test_race_seed_canary.py | 108 ++++++++ 10 files changed, 830 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/race-seed-canary.yaml create mode 100644 .github/workflows/race-seed-sample.yaml create mode 100644 docs/race-seed-canary.md create mode 100644 scripts/race_seed_canary.py create mode 100644 scripts/race_seed_plan.py create mode 100644 scripts/test_race_seed_canary.py diff --git a/.github/workflows/check-action-file.yaml b/.github/workflows/check-action-file.yaml index fd11799..bce21bd 100644 --- a/.github/workflows/check-action-file.yaml +++ b/.github/workflows/check-action-file.yaml @@ -17,6 +17,7 @@ on: - 'scripts/select_coverage_artifacts.py' - 'scripts/test_select_coverage_artifacts.py' - 'scripts/test_merge_trigger_tke_subject.py' + - 'scripts/*race_seed*.py' permissions: contents: read @@ -79,6 +80,8 @@ jobs: bash actions/restore-sca-analysis-cache/environment-hash.test.sh - name: Test Go cache import contracts run: python3 -m unittest discover -s actions/seed-go-caches -p 'test_*.py' -v + - name: Test race canary evidence contracts + run: python3 -m unittest discover -s scripts -p 'test_race_seed*.py' -v - name: Guard deliberate workflow disablements env: ACTIONLINT: ${{ steps.install-actionlint.outputs.executable }} diff --git a/.github/workflows/race-seed-canary.yaml b/.github/workflows/race-seed-canary.yaml new file mode 100644 index 0000000..4056ee9 --- /dev/null +++ b/.github/workflows/race-seed-canary.yaml @@ -0,0 +1,94 @@ +name: Race seed canary + +on: + workflow_dispatch: + inputs: + matrixone_sha: + description: Full SHA reachable from matrixorigin/matrixone main + required: true + type: string + image: + description: Builder repository@sha256 digest (never a mutable tag) + required: true + type: string + default: matrixorigin/matrixone@sha256:8137d222d3a3b29d119173639cea98931168ab7d886bf882894391f7804d5d88 + repetitions: + description: One smoke pair per cache state, or three measured pairs + type: choice + options: ['1', '3'] + default: '1' + +permissions: + contents: read + +concurrency: + group: race-seed-canary + cancel-in-progress: false + +jobs: + plan: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-22.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - id: plan + env: + GH_TOKEN: ${{ github.token }} + SOURCE_SHA: ${{ inputs.matrixone_sha }} + SEED_IMAGE: ${{ inputs.image }} + REPETITIONS: ${{ inputs.repetitions }} + run: python3 scripts/race_seed_plan.py + + warm-snapshot: + needs: plan + uses: ./.github/workflows/race-seed-sample.yaml + with: + matrixone_sha: ${{ inputs.matrixone_sha }} + image: ${{ inputs.image }} + name: warm-preparation + cache_state: prepare + seed: false + secrets: inherit + + samples: + needs: [plan, warm-snapshot] + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + uses: ./.github/workflows/race-seed-sample.yaml + with: + matrixone_sha: ${{ inputs.matrixone_sha }} + image: ${{ inputs.image }} + name: ${{ matrix.name }} + cache_state: ${{ matrix.cache_state }} + seed: ${{ matrix.seed }} + secrets: inherit + + compare: + needs: [plan, warm-snapshot, samples] + if: ${{ always() && needs.plan.result == 'success' }} + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: race-* + path: results + - env: + CANARY_REPETITIONS: ${{ inputs.repetitions }} + run: python3 scripts/race_seed_canary.py --summarize results + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: comparison + path: results/comparison.json + if-no-files-found: warn diff --git a/.github/workflows/race-seed-sample.yaml b/.github/workflows/race-seed-sample.yaml new file mode 100644 index 0000000..2f947d1 --- /dev/null +++ b/.github/workflows/race-seed-sample.yaml @@ -0,0 +1,119 @@ +name: Race seed measurement sample + +on: + workflow_call: + inputs: + matrixone_sha: + required: true + type: string + image: + required: true + type: string + name: + required: true + type: string + cache_state: + required: true + type: string + seed: + required: true + type: boolean + secrets: + S3ENDPOINT: + required: true + S3REGION: + required: true + S3APIKEY: + required: true + S3APISECRET: + required: true + S3BUCKET: + required: true + +permissions: + contents: read + +jobs: + sample: + # Fresh hosted runners only: no production runner caches are cleared. + runs-on: ${{ vars.RACE_CANARY_RUNNER_LABEL || 'ubuntu-22.04' }} + environment: ci + timeout-minutes: 120 + steps: + - name: Require an ephemeral hosted runner before any checkout + id: runner_guard + env: + RUNNER_KIND: ${{ runner.environment }} + run: test "$RUNNER_KIND" = github-hosted + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: matrixorigin/matrixone + ref: ${{ inputs.matrixone_sha }} + path: subject + persist-credentials: false + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: subject/go.mod + cache: false + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4 + with: + distribution: adopt + java-version: '8' + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + if: inputs.cache_state == 'warm' + with: + name: warm-snapshot + path: warm-snapshot + - name: Measure complete race suite + env: + SEED_IMAGE: ${{ inputs.image }} + SEED_FLAVOR: race + ENABLE_SEED: ${{ inputs.seed }} + CACHE_STATE: ${{ inputs.cache_state }} + CANARY_RUNNER_LABEL: ${{ vars.RACE_CANARY_RUNNER_LABEL || 'ubuntu-22.04' }} + CANARY_UT_PARALLEL: ${{ vars.UT_PARALLEL || 6 }} + CANARY_UT_TIMEOUT: ${{ vars.UT_TIMEOUT || 40 }} + GOCACHE: /home/runner/.cache/mo-race-canary + GOMODCACHE: /home/runner/go/pkg/mod + UT_WORKDIR: /home/runner/_work/matrixone/matrixone + endpoint: ${{ secrets.S3ENDPOINT }} + region: ${{ secrets.S3REGION }} + apikey: ${{ secrets.S3APIKEY }} + apisecret: ${{ secrets.S3APISECRET }} + bucket: ${{ secrets.S3BUCKET }} + run: | + set -euo pipefail + args=() + if [[ "$ENABLE_SEED" == true ]]; then args+=(--seed); fi + case "$CACHE_STATE" in + cold) ;; + warm) args+=(--snapshot "$GITHUB_WORKSPACE/warm-snapshot");; + prepare) args+=(--export "$GITHUB_WORKSPACE/warm-snapshot");; + *) exit 1;; + esac + python3 scripts/race_seed_canary.py "${args[@]}" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: ${{ always() && steps.runner_guard.outcome == 'success' }} + with: + name: ${{ inputs.name }} + path: canary-report/ + retention-days: 7 + if-no-files-found: error + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: ${{ always() && steps.runner_guard.outcome == 'success' }} + with: + name: diagnostics-${{ inputs.name }} + path: /home/runner/_work/matrixone/matrixone/ut-report/ + retention-days: 7 + if-no-files-found: warn + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: success() && inputs.cache_state == 'prepare' + with: + name: warm-snapshot + path: warm-snapshot/ + compression-level: 0 + retention-days: 1 + if-no-files-found: error diff --git a/actions/seed-go-caches/action.yaml b/actions/seed-go-caches/action.yaml index 5c44285..a8b0664 100644 --- a/actions/seed-go-caches/action.yaml +++ b/actions/seed-go-caches/action.yaml @@ -1,6 +1,9 @@ name: Seed Go caches description: Best-effort additive import from the trusted CI builder, preserving existing caches inputs: + image: + description: Optional trusted repository at an immutable sha256 digest for paired measurements + default: '' flavor: description: Producer warm flavor (race or coverage) required: true @@ -13,6 +16,7 @@ runs: - name: Import trusted Go cache entries shell: bash env: + SEED_IMAGE: ${{ inputs.image }} SEED_FLAVOR: ${{ inputs.flavor }} SEED_GENERATION: ${{ inputs.generation }} run: python3 "$GITHUB_ACTION_PATH/seed.py" diff --git a/actions/seed-go-caches/seed.py b/actions/seed-go-caches/seed.py index 747d7ff..2049b46 100644 --- a/actions/seed-go-caches/seed.py +++ b/actions/seed-go-caches/seed.py @@ -31,6 +31,20 @@ MARKER = ".matrixone-seed.json" +def image_candidates(pinned, hosted=False): + """A canary may pin a digest, but may not change the trusted repository.""" + repositories = ["registry.cn-shanghai.aliyuncs.com/matrixorigin/matrixone", + "matrixorigin/matrixone"] + if pinned: + if not any(re.fullmatch(re.escape(repo) + r"@sha256:[0-9a-f]{64}", pinned) + for repo in repositories): + raise ValueError("seed image must be a trusted repository at a sha256 digest") + return [pinned] + if hosted: + repositories.reverse() + return [repo + ":ci-builder" for repo in repositories] + + def directory(value): path = Path(value) if not path.is_absolute() or path == Path("/"): @@ -318,6 +332,8 @@ def payload(self, cache, source, root, budget, build=False): return data def seed(self, stack): + pinned = self.env.get("SEED_IMAGE", "") + images = image_candidates(pinned, self.env.get("RUNNER_ENVIRONMENT") == "github-hosted") values = json.loads(self.command([ "go", "env", "-json", "GOCACHE", "GOMODCACHE", "GOVERSION", "GOOS", "GOARCH", "GOAMD64", "GOEXPERIMENT", "GOCACHEPROG", "GOMOD"])) @@ -343,6 +359,8 @@ def seed(self, stack): key = {"schema": SCHEMA, "generation": self.generation, "consumer": values, "flavor": self.flavor, "profile": PROFILE, "contracts": CONTRACTS, "checkout": CHECKOUT} + if pinned: + key["image"] = pinned marker = cache / MARKER if marker.is_symlink(): raise ValueError("symlink completion record") @@ -368,10 +386,6 @@ def seed(self, stack): if not space_available([(docker_root, 0), (cache, 0), (modules, 0)]): self.report["state"] = "insufficient-space" return - images = ["registry.cn-shanghai.aliyuncs.com/matrixorigin/matrixone:ci-builder", - "matrixorigin/matrixone:ci-builder"] - if self.env.get("RUNNER_ENVIRONMENT") == "github-hosted": - images.reverse() image = None for candidate in images: try: diff --git a/actions/seed-go-caches/test_seed.py b/actions/seed-go-caches/test_seed.py index 6efc6de..364858c 100644 --- a/actions/seed-go-caches/test_seed.py +++ b/actions/seed-go-caches/test_seed.py @@ -23,6 +23,21 @@ CONTAINER = "c" * 64 +class ImageSelectionTests(unittest.TestCase): + def test_default_locality_and_immutable_override(self): + self.assertTrue(seed.image_candidates('', True)[0].startswith('matrixorigin/')) + self.assertTrue(seed.image_candidates('', False)[0].startswith('registry.cn-shanghai.')) + image = 'matrixorigin/matrixone@sha256:' + '1' * 64 + self.assertEqual(seed.image_candidates(image), [image]) + + def test_untrusted_or_mutable_override_rejected(self): + for image in ('evil/matrixone@sha256:' + '1' * 64, + 'matrixorigin/matrixone:ci-builder', + 'matrixorigin/matrixone@sha256:abc', '--help'): + with self.subTest(image=image), self.assertRaises(ValueError): + seed.image_candidates(image) + + def tar_bytes(entries): """Entries are (name, bytes[, mode]) or explicit TarInfo objects.""" output = io.BytesIO() @@ -202,6 +217,26 @@ def test_second_matching_invocation_avoids_all_docker_calls(self): self.assertEqual(report["imported_files"], 0) self.assertEqual(report["module_state"], "previous-import") + def test_pinned_digest_participates_in_marker_identity(self): + image = 'matrixorigin/matrixone@sha256:' + '1' * 64 + first = self.make() + first.env['SEED_IMAGE'] = image + self.assertEqual(self.run_seed(first)['state'], 'seeded') + same = self.make() + same.env['SEED_IMAGE'] = image + self.assertEqual(self.run_seed(same)['state'], 'already-seeded') + self.assertEqual(same.calls, []) + changed = self.make() + changed.env['SEED_IMAGE'] = 'matrixorigin/matrixone@sha256:' + '2' * 64 + self.assertEqual(self.run_seed(changed)['state'], 'seeded') + self.assertIn(('pull', changed.env['SEED_IMAGE']), changed.calls) + + def test_invalid_pin_fails_before_any_docker_call(self): + instance = self.make() + instance.env['SEED_IMAGE'] = 'untrusted/image:latest' + self.assertEqual(self.run_seed(instance)['state'], 'failed') + self.assertEqual(instance.calls, []) + def executable_payload(self): directory = tarfile.TarInfo("go-build/" + EXECUTABLE) directory.type = tarfile.DIRTYPE diff --git a/docs/race-seed-canary.md b/docs/race-seed-canary.md new file mode 100644 index 0000000..4d26d19 --- /dev/null +++ b/docs/race-seed-canary.md @@ -0,0 +1,75 @@ +# Race cache seed measurement + +The canary compares the complete race suite with seed disabled/enabled, on a +fixed MatrixOne main commit and builder digest. Production defaults stay off. +The manually dispatched workflow runs only from CI main. All harness code comes +from its immutable workflow commit. MatrixOne must be an ancestor of main, +because the suite uses the existing CI test credentials. + +Each arm gets a fresh GitHub-hosted Linux x64 runner. Cold arms start without +Go caches or the builder image. Warm arms restore the **same immutable artifact** +produced by one successful full race run without seeding. This snapshot has no +seed completion marker: warm means an existing compiler/module cache, not a +previous seed import. The already-seeded marker fast path is a separate future +experiment. Repetitions alternate AB/BA order and run one arm at a time. + +The comparison rejects different source/harness/image identities, snapshot +hashes, runner image versions, CPU/memory/toolchains or UT settings. OS page +cache and network conditions remain noise; pair repetition measures that noise. +This initial canary supports ephemeral GitHub-hosted runners only. Use a matching +larger hosted runner via RACE_CANARY_RUNNER_LABEL if the standard runner fails +the importer space gate. It does not claim measurements for self-hosted pools. + +Measured wall time starts before seed acquisition and ends after make clean, +config, full make ut and importer-owned cleanup. It includes seeder failures. +Checkout/toolchain setup, warm snapshot production/transfer/restoration, report +parsing and artifact upload are separately visible workflow preparation costs, +not part of the paired UT interval. Docker image layers remain daemon-owned, +as in the production importer; they are charged to disk occupancy and reclaimed +by ephemeral runner destruction, not by an invented image-prune step. +Host CPU counters, disk counters, memory availability, cgroup counters and disk +free space are sampled throughout the measured interval. Raw samples and UT +logs are retained. Imported entries are not reported as compiler cache hits. + +A successful workflow is a **valid experiment**, not rollout approval. Both +arms must complete the same nonempty test execution multiset with identical +pass/skip/package outcomes. B must report seeded, producer race ok, imported +files > 0 and complete cleanup; any skip/fallback/partial seed is invalid. Warm +cache identity is checked before running. Cold/warm are reported independently; +one smoke pair is insufficient to establish a stable gain. No coverage result +is inferred. Keep rollout off until repeated net gains and acceptable memory, +disk and CPU costs have been reviewed. + +## Initial verified image + +2026-09-20: builder job 105925505882 in MatrixOne run 35443952690 published +`matrixorigin/matrixone@sha256:8137d222d3a3b29d119173639cea98931168ab7d886bf882894391f7804d5d88` +from source `3ac87c30625fc082391e5384a4c94e50a2916cf6`. +The registry metadata layer was downloaded independently and its SHA256 +verified. Manifest schema 2 / host-ut-v1, canonical checkout/modules, Go 1.26.4, +linux/amd64/v1, both contract IDs, race=ok and coverage=ok match the consumer. +This proves publication/compatibility, not cache-hit rate or end-to-end speedup. + +## Invocation + +Configure the five existing S3 UT credentials in this repository's CI +environment (S3ENDPOINT, S3REGION, S3APIKEY, S3APISECRET, S3BUCKET); they are not +copied from MatrixOne and are never printed. Missing credentials reject the run. +The currently configured MatrixOne UT pool is `amd64-mo-shanghai-8c16g`; +this hosted-only first measurement does not establish performance on that pool. +Rollout to that pool additionally requires a matching ephemeral runner experiment. + +After this workflow is merged, dispatch Race seed canary on main with a full +40-character MatrixOne SHA and the immutable image reference above. Start with +repetitions=1 (four measured arms plus one warm preparation run); repetitions=3 +gives twelve measured arms and the same one preparation run. Review the report +artifact and job summaries before requesting a separate rollout change. + +## Design scope + +R2 measurement contract: isolate compiler caches, preserve full race execution, +keep fixed inputs, and fail closed on incomplete evidence. No suite selection, +timeouts, scheduling or production activation changes. The importer gets one +optional digest input restricted to its two existing trusted repositories; +ordinary callers retain their current behavior. Tests cover input rejection, +empty/truncated execution evidence, mismatched outcomes and mismatched pairing. diff --git a/scripts/race_seed_canary.py b/scripts/race_seed_canary.py new file mode 100644 index 0000000..6c0cafa --- /dev/null +++ b/scripts/race_seed_canary.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""Fixed-input race measurement. No rollout decision is automated here.""" + +import argparse +from collections import Counter +import hashlib +import json +import os +from pathlib import Path +import re +import resource +import shutil +import signal +import subprocess +import sys +import tarfile +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = Path('/home/runner/_work/matrixone/matrixone') +CACHE = Path('/home/runner/.cache/mo-race-canary') +MODULES = Path('/home/runner/go/pkg/mod') + + +def dump(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + '\n') + + +def command(args, cwd=None): + return subprocess.check_output(args, cwd=cwd, text=True, timeout=120).strip() + + +def sha256(path): + digest = hashlib.sha256() + with path.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def execution(path): + """Compare execution multisets, not just a headline passing-test count.""" + runs, ends, outcomes = Counter(), Counter(), Counter() + packages, completed = Counter(), Counter() + with path.open() as stream: + for line in stream: + if not line.startswith('{'): + continue + event = json.loads(line) # truncated JSON must not look like success + package, test = event.get('Package', ''), event.get('Test', '') + action = event.get('Action') + if action == 'start': + packages[package] += 1 + if action == 'run' and test: + runs[(package, test)] += 1 + if action in ('pass', 'skip', 'fail'): + outcomes[(package, test, action)] += 1 + if test: + ends[(package, test)] += 1 + else: + completed[package] += 1 + if not runs or runs != ends or any(k[2] == 'fail' for k in outcomes): + raise ValueError('empty, failed or incomplete test execution') + if not packages or packages != completed: + raise ValueError('missing package start or completion') + if any(p not in packages for p, test in runs): + raise ValueError('test without package start') + rows = sorted([*key, count] for key, count in outcomes.items()) + return {'test_runs': sum(runs.values()), + 'outcome_sha256': hashlib.sha256(json.dumps(rows).encode()).hexdigest(), + 'counts': dict(Counter({kind: sum(n for (p, t, a), n in outcomes.items() + if a == kind and t) + for kind in ('pass', 'skip', 'fail')})), + 'outcomes': rows} + + +def fingerprint(): + cpu = Path('/proc/cpuinfo').read_text() + mem = Path('/proc/meminfo').read_text() + return { + 'source_sha': command(['git', 'rev-parse', 'HEAD'], SOURCE), + 'ci_sha': command(['git', 'rev-parse', 'HEAD'], ROOT), + 'image': os.environ['SEED_IMAGE'], + 'runner_image': [os.environ.get(k, '') for k in ('ImageOS', 'ImageVersion')], + 'runner_label': os.environ['CANARY_RUNNER_LABEL'], + 'cpu_model': sorted(set(re.findall(r'^model name\s*:\s*(.*)', cpu, re.M))), + 'cpu_count': os.cpu_count(), + 'mem_total': re.search(r'^MemTotal:.*', mem, re.M)[0], + 'go': json.loads(command(['go', 'env', '-json', 'GOVERSION', 'GOOS', 'GOARCH', + 'GOAMD64', 'GOEXPERIMENT', 'CGO_ENABLED', 'GOFLAGS', + 'GOMOD', 'GOMODCACHE', 'GOPROXY'], SOURCE)), + 'compiler': command(['cc', '--version']), + 'cmake': command(['cmake', '--version']), + 'kernel': command(['uname', '-srmo']), + 'ut_parallel': os.environ['CANARY_UT_PARALLEL'], + 'ut_timeout': os.environ['CANARY_UT_TIMEOUT'], + } + + +def sample(): + result = {'monotonic': time.monotonic(), 'disk_free': shutil.disk_usage(SOURCE).free} + for name, file in { + 'cpu': '/proc/stat', 'memory': '/proc/meminfo', 'disk': '/proc/diskstats', + 'pressure_io': '/proc/pressure/io', 'pressure_cpu': '/proc/pressure/cpu', + 'cgroup_cpu': '/sys/fs/cgroup/cpu.stat', + 'cgroup_memory': '/sys/fs/cgroup/memory.current', + 'cgroup_memory_events': '/sys/fs/cgroup/memory.events', + 'cgroup_io': '/sys/fs/cgroup/io.stat', + }.items(): + try: + result[name] = Path(file).read_text() + except FileNotFoundError: + result[name] = None + return result + + +def measured_command(args, log, timeout): + """Bound the process group, preserve nonzero status and capture CPU cost.""" + started = time.monotonic() + before = resource.getrusage(resource.RUSAGE_CHILDREN) + with log.open('wb') as output: + child = subprocess.Popen(args, cwd=SOURCE, stdout=output, + stderr=subprocess.STDOUT, start_new_session=True) + try: + code = child.wait(timeout=timeout) + except BaseException: + os.killpg(child.pid, signal.SIGTERM) + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=10) + raise + after = resource.getrusage(resource.RUSAGE_CHILDREN) + result = {'seconds': time.monotonic() - started, 'exit_code': code, + 'user_seconds': after.ru_utime - before.ru_utime, + 'system_seconds': after.ru_stime - before.ru_stime} + return result + + +def resource_summary(path): + with path.open() as stream: + samples = [json.loads(line) for line in stream] + if len(samples) < 2: + raise ValueError('missing resource samples') + def cpu(row): + return list(map(int, row['cpu'].splitlines()[0].split()[1:9])) + first, last = cpu(samples[0]), cpu(samples[-1]) + delta = [b - a for a, b in zip(first, last)] + ticks = sum(delta) + busy = [] + for a, b in zip(samples, samples[1:]): + d = [y - x for x, y in zip(cpu(a), cpu(b))] + if sum(d) > 0: + busy.append(100 * (1 - (d[3] + d[4]) / sum(d))) + mem = [int(re.search(r'^MemAvailable:\s+(\d+)', r['memory'], re.M)[1]) * 1024 + for r in samples] + return {'host_cpu_busy_percent': 100 * (1 - (delta[3] + delta[4]) / ticks) if ticks else 0, + 'host_cpu_peak_interval_percent': max(busy, default=0), + 'host_cpu_seconds': (ticks - delta[3] - delta[4]) / os.sysconf('SC_CLK_TCK'), + 'min_memory_available_bytes': min(mem), + 'min_disk_free_bytes': min(r['disk_free'] for r in samples), + 'sample_interval_seconds': 5, + 'note': 'Host-wide sampled counters; daemon CPU included, short peaks may be missed'} + + +def prepare(snapshot): + if os.environ.get('RUNNER_ENVIRONMENT') != 'github-hosted': + raise ValueError('canary requires a fresh GitHub-hosted runner') + for path in (SOURCE, CACHE, MODULES): + if path.exists() or path.is_symlink(): + raise ValueError(f'preexisting path; refusing to alter it: {path}') + SOURCE.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(ROOT / 'subject'), SOURCE) + CACHE.parent.mkdir(parents=True, exist_ok=True) + MODULES.parent.mkdir(parents=True, exist_ok=True) + initial = {'cache_state': 'cold', 'snapshot_sha256': None} + if snapshot: + meta = json.loads((snapshot / 'snapshot.json').read_text()) + archive = snapshot / 'cache.tar' + if sha256(archive) != meta['sha256']: + raise ValueError('snapshot hash mismatch') + staging = snapshot / 'extracted' + staging.mkdir() + with tarfile.open(archive) as stream: + stream.extractall(staging, filter='data') + shutil.move(str(staging / 'go-build'), CACHE) + shutil.move(str(staging / 'mod'), MODULES) + if meta['identity'] != fingerprint(): + raise ValueError('snapshot environment mismatch') + archive.unlink() # only the task-owned downloaded archive + initial = {'cache_state': 'warm', 'snapshot_sha256': meta['sha256']} + else: + CACHE.mkdir() + MODULES.mkdir() + if (CACHE / '.matrixone-seed.json').exists(): + raise ValueError('initial compiler cache must not contain a seed marker') + # Identical test-result invalidation in A/B; compiler cache is preserved. + command(['go', 'clean', '-testcache'], SOURCE) + return initial + + +def run(args): + out = ROOT / 'canary-report' + out.mkdir(exist_ok=True) + report = {'valid': False, 'seed_enabled': args.seed, 'phases': {}} + stop = threading.Event() + + def monitor(): + with (out / 'resources.jsonl').open('w') as log: + while True: + log.write(json.dumps(sample()) + '\n') + log.flush() + if stop.wait(5): + break + + monitor_thread = None + try: + if not all(os.environ.get(k) for k in ('endpoint', 'region', 'apikey', 'apisecret', 'bucket')): + raise ValueError('CI S3 test credentials must be configured; no secret values are logged') + report['initial'] = prepare(args.snapshot) + report['identity'] = fingerprint() + before_images = command(['docker', 'image', 'ls', '-q', '--no-trunc']) + report['initial_images'] = sorted(before_images.splitlines()) + # The pinned builder cannot already be local, even for warm Go caches. + if subprocess.run(['docker', 'image', 'inspect', os.environ['SEED_IMAGE']], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=30).returncode == 0: + raise ValueError('builder image already present') + report['disk_before'] = command(['df', '-Pk', str(SOURCE), str(CACHE), str(MODULES)]) + measured_start = time.monotonic() + monitor_thread = threading.Thread(target=monitor) + monitor_thread.start() + try: + if args.seed: + phase = measured_command([sys.executable, str(ROOT / 'actions/seed-go-caches/seed.py')], + out / 'seed.log', 1140) + report['phases']['seed'] = phase + report['seed'] = json.loads((out / 'seed.log').read_text().splitlines()[-1]) + for name, cmd, timeout in [ + ('clean', ['make', 'clean'], 300), + ('config', ['make', 'config'], 900), + ('ut', ['make', 'ut', 'UT_CONFIGURED=1', 'UT_SHARD=all', + 'UT_PARALLEL=' + os.environ['CANARY_UT_PARALLEL'], + 'UT_TIMEOUT=' + os.environ['CANARY_UT_TIMEOUT']], 4500), + ]: + report['phases'][name] = measured_command(cmd, out / (name + '.log'), timeout) + if report['phases'][name]['exit_code']: + raise ValueError(name + ' failed') + finally: + report['total_seconds'] = time.monotonic() - measured_start + stop.set() + monitor_thread.join(timeout=10) + with (out / 'resources.jsonl').open('a') as log: + log.write(json.dumps(sample()) + '\n') + report['resources'] = resource_summary(out / 'resources.jsonl') + report['disk_after'] = command(['df', '-Pk', str(SOURCE), str(CACHE), str(MODULES)]) + report['docker_disk'] = command(['docker', 'system', 'df']) + reports = list((SOURCE / 'scratch').rglob('*-UT-Report.out')) + if len(reports) != 1: + raise ValueError('expected exactly one complete raw UT report') + report['execution'] = execution(reports[0]) + if args.seed: + seeded = report['seed'] + if (seeded.get('state') != 'seeded' or seeded.get('producer_flavor_status') != 'ok' + or seeded.get('cleanup') != 'complete' or seeded.get('imported_files', 0) <= 0): + raise ValueError('seed skipped, partial, empty or failed: invalid B sample') + report['valid'] = True + if args.export: + args.export.mkdir() + archive = args.export / 'cache.tar' + with tarfile.open(archive, 'w') as stream: + stream.add(CACHE, arcname='go-build') + stream.add(MODULES, arcname='mod') + dump(args.export / 'snapshot.json', {'identity': report['identity'], 'sha256': sha256(archive)}) + except Exception as error: + report.update(valid=False, error=str(error)) + finally: + stop.set() + if monitor_thread: + monitor_thread.join(timeout=10) + dump(out / 'result.json', report) + print(json.dumps({k: v for k, v in report.items() if k != 'execution'}, indent=2), flush=True) + return 0 if report['valid'] else 1 + + +def compare(a, b): + if not a.get('valid') or not b.get('valid'): + raise ValueError('invalid/incomplete arm') + if a['seed_enabled'] or not b['seed_enabled']: + raise ValueError('expected A=off, B=on') + for key in ('identity', 'initial', 'initial_images', 'execution'): + if a[key] != b[key]: + raise ValueError('pair mismatch: ' + key) + return {'A_seconds': a['total_seconds'], 'B_seconds': b['total_seconds'], + 'saved_seconds': a['total_seconds'] - b['total_seconds'], + 'saved_percent': 100 * (1 - b['total_seconds'] / a['total_seconds'])} + + +def summarize(directory): + results = [] + for path in sorted(directory.glob('race-*-A/result.json')): + other = path.parent.with_name(path.parent.name[:-1] + 'B') / 'result.json' + if not other.exists(): + raise ValueError('missing B: ' + str(other)) + result = compare(json.loads(path.read_text()), json.loads(other.read_text())) + results.append({'pair': path.parent.name[:-2], **result}) + expected = 2 * int(os.environ['CANARY_REPETITIONS']) + if len(results) != expected: + raise ValueError(f'expected {expected} cold/warm pairs; got {len(results)}') + dump(directory / 'comparison.json', results) + text = '| Pair | A seconds | B seconds | Saved seconds | Saved % |\n|---|---:|---:|---:|---:|\n' + for r in results: + text += f"| {r['pair']} | {r['A_seconds']:.1f} | {r['B_seconds']:.1f} | {r['saved_seconds']:.1f} | {r['saved_percent']:.1f} |\n" + text += '\nValid paired execution only. Review raw resource samples before any rollout decision.\n' + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as summary: + summary.write(text) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--seed', action='store_true') + parser.add_argument('--snapshot', type=Path) + parser.add_argument('--export', type=Path) + parser.add_argument('--summarize', type=Path) + options = parser.parse_args() + if options.summarize: + summarize(options.summarize) + else: + def cancelled(signum, frame): + raise InterruptedError(f'cancelled by signal {signum}') + signal.signal(signal.SIGTERM, cancelled) + sys.exit(run(options)) diff --git a/scripts/race_seed_plan.py b/scripts/race_seed_plan.py new file mode 100644 index 0000000..5b48538 --- /dev/null +++ b/scripts/race_seed_plan.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Validate immutable inputs before admitting credentialed race jobs.""" +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess + + +def matrix(repetitions): + if repetitions not in (1, 3): + raise ValueError('repetitions must be 1 or 3') + rows = [] + for repeat in range(1, repetitions + 1): + for state in ('cold', 'warm'): + for arm in ('AB' if repeat % 2 else 'BA'): + rows.append(dict(name=f'race-{state}-{repeat}-{arm}', + cache_state=state, seed=arm == 'B')) + return {'include': rows} + + +if __name__ == '__main__': + sha = os.environ['SOURCE_SHA'] + if not re.fullmatch('[0-9a-f]{40}', sha): + raise ValueError('full source SHA required') + spec = importlib.util.spec_from_file_location('seed', Path(__file__).resolve().parents[1] + / 'actions/seed-go-caches/seed.py') + seed = importlib.util.module_from_spec(spec) + spec.loader.exec_module(seed) + if not os.environ['SEED_IMAGE']: + raise ValueError('pinned image is required') + seed.image_candidates(os.environ['SEED_IMAGE']) + status = subprocess.check_output(['gh', 'api', + f'repos/matrixorigin/matrixone/compare/{sha}...main', '--jq', '.status'], + text=True, timeout=60).strip() + if status not in ('ahead', 'identical'): + raise ValueError('source SHA is not reachable from official main') + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write('matrix=' + json.dumps(matrix(int(os.environ['REPETITIONS']))) + '\n') diff --git a/scripts/test_race_seed_canary.py b/scripts/test_race_seed_canary.py new file mode 100644 index 0000000..2fa29c2 --- /dev/null +++ b/scripts/test_race_seed_canary.py @@ -0,0 +1,108 @@ +import copy +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from race_seed_canary import compare, execution, summarize, resource_summary +from race_seed_plan import matrix + + +class EvidenceTests(unittest.TestCase): + def read(self, events): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'raw.json' + path.write_text('\n'.join(json.dumps(e) for e in events)) + return execution(path) + + def events(self): + return [{'Package': 'p', 'Action': 'start'}] + [ + {'Package': 'p', 'Test': 'TestX', 'Action': action} for action in ('run', 'pass')] + [ + {'Package': 'p', 'Action': 'pass'}] + + def test_execution_requires_real_nonempty_complete_tests(self): + for events in ([], [{'Package': 'p', 'Action': 'pass'}], self.events()[:1], + self.events()[:-1]): + with self.subTest(events=events), self.assertRaises(ValueError): + self.read(events) + self.assertEqual(self.read(self.events())['test_runs'], 1) + + def test_failed_or_truncated_output_is_not_success(self): + events = self.events() + events[2]['Action'] = 'fail' + with self.assertRaises(ValueError): + self.read(events) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'raw.json' + path.write_text('{"Action":') + with self.assertRaises(ValueError): + execution(path) + + def test_execution_multiset_detects_test_substitution_and_duplicates(self): + a = self.read(self.events()) + replacement = self.events() + replacement[1]['Test'] = replacement[2]['Test'] = 'TestY' + self.assertNotEqual(a['outcome_sha256'], self.read(replacement)['outcome_sha256']) + self.assertEqual(self.read(self.events() * 2)['test_runs'], 2) + + def test_missing_second_package_terminal_is_rejected(self): + second = [dict(e, Package='q') for e in self.events()[:-1]] + with self.assertRaises(ValueError): + self.read(self.events() + second) + + def test_pair_rejects_each_mismatched_contract(self): + a = dict(valid=True, seed_enabled=False, identity={'sha': 'same'}, + initial={'snapshot_sha256': 'same'}, initial_images=[], + execution=self.read(self.events()), total_seconds=10) + b = dict(copy.deepcopy(a), seed_enabled=True, total_seconds=8) + self.assertEqual(compare(a, b)['saved_seconds'], 2) + for key in ('identity', 'initial', 'initial_images', 'execution', 'valid', 'seed_enabled'): + changed = copy.deepcopy(b) + changed[key] = False if key in ('valid', 'seed_enabled') else 'different' + with self.subTest(key=key), self.assertRaises(ValueError): + compare(a, changed) + + def test_matrix_has_both_states_and_alternates_order(self): + rows = matrix(3)['include'] + self.assertEqual(len(rows), 12) + self.assertEqual([r['name'] for r in rows[:6]], [ + 'race-cold-1-A', 'race-cold-1-B', 'race-warm-1-A', 'race-warm-1-B', + 'race-cold-2-B', 'race-cold-2-A']) + with self.assertRaises(ValueError): + matrix(2) + + def test_downloaded_artifact_layout_and_missing_pair(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for state in ('cold', 'warm'): + for arm in 'AB': + target = root / f'race-{state}-1-{arm}' + target.mkdir() + result = dict(valid=True, seed_enabled=arm == 'B', identity={}, + initial={'cache_state': state}, initial_images=[], + execution=self.read(self.events()), total_seconds=10) + (target / 'result.json').write_text(json.dumps(result)) + with mock.patch.dict(os.environ, CANARY_REPETITIONS='1', + GITHUB_STEP_SUMMARY=str(root / 'summary.md')): + summarize(root) + self.assertEqual(len(json.loads((root / 'comparison.json').read_text())), 2) + (root / 'race-warm-1-B' / 'result.json').unlink() + with self.assertRaises(ValueError): + summarize(root) + + def test_host_cpu_includes_non_child_work_and_disk_minimum(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'samples.jsonl' + rows = [dict(cpu='cpu 0 0 0 100 0 0 0 0\n', memory='MemAvailable: 100 kB\n', disk_free=100), + dict(cpu='cpu 50 0 0 150 0 0 0 0\n', memory='MemAvailable: 50 kB\n', disk_free=20)] + path.write_text('\n'.join(map(json.dumps, rows))) + result = resource_summary(path) + self.assertEqual(result['host_cpu_busy_percent'], 50) + self.assertEqual(result['host_cpu_peak_interval_percent'], 50) + self.assertEqual(result['min_disk_free_bytes'], 20) + + +if __name__ == '__main__': + unittest.main() From 87fcadec3a600ba3c5a2b27c7d35d0cd229f9287 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 20 Sep 2026 05:35:24 +0800 Subject: [PATCH 2/6] fix(canary): distinguish runner cgroup load from host load --- docs/race-seed-canary.md | 31 ++++++++++++++++---- scripts/race_seed_canary.py | 49 +++++++++++++++++++++++++++++++- scripts/test_race_seed_canary.py | 23 ++++++++++++++- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/docs/race-seed-canary.md b/docs/race-seed-canary.md index 4d26d19..2b64b74 100644 --- a/docs/race-seed-canary.md +++ b/docs/race-seed-canary.md @@ -1,5 +1,23 @@ # Race cache seed measurement +**Draft / not ready for the production runner experiment.** Read-only inspection +on 2026-09-20 confirmed job 105924102623 uses `amd64-mo-shanghai-8c16g`, +canonical checkout/module paths, CPU quota `800000/100000` (8 cores), and +memory limit `17179869184` (16 GiB), while its visible cpuset is `0-95`. +Consequently host CPU utilization is only context, not runner utilization. +Reports now separately calculate visible-cgroup CPU seconds/quota utilization, +throttling, sampled memory peak and OOM counter deltas. They do not attribute +an external Docker daemon's CPU to the test container, nor claim hidden ancestor +limits are known. + +Pool isolation remains **unverified**: the organization runners API and the +configured Kubernetes autoscaling-runner-set listing both reject access (403). +Before replacing the hosted guard, obtain a sanitized runner template confirming +per-job lifecycle and volume ownership for checkout, Go caches and Docker storage. +Also resolve a MatrixOne-side caller that inherits existing UT credentials; +do not copy credentials into CI merely to make this draft runnable. No runner +configuration, production defaults or cache contents were changed by inspection. + The canary compares the complete race suite with seed disabled/enabled, on a fixed MatrixOne main commit and builder digest. Production defaults stay off. The manually dispatched workflow runs only from CI main. All harness code comes @@ -11,7 +29,8 @@ Go caches or the builder image. Warm arms restore the **same immutable artifact* produced by one successful full race run without seeding. This snapshot has no seed completion marker: warm means an existing compiler/module cache, not a previous seed import. The already-seeded marker fast path is a separate future -experiment. Repetitions alternate AB/BA order and run one arm at a time. +experiment. The matrix requests alternating AB/BA order and runs one arm at a +time; matrix scheduling is not proof of execution order. Check job timestamps. The comparison rejects different source/harness/image identities, snapshot hashes, runner image versions, CPU/memory/toolchains or UT settings. OS page @@ -52,14 +71,16 @@ This proves publication/compatibility, not cache-hit rate or end-to-end speedup. ## Invocation -Configure the five existing S3 UT credentials in this repository's CI -environment (S3ENDPOINT, S3REGION, S3APIKEY, S3APISECRET, S3BUCKET); they are not -copied from MatrixOne and are never printed. Missing credentials reject the run. +The sample requires the five existing S3 UT credentials (S3ENDPOINT, S3REGION, +S3APIKEY, S3APISECRET, S3BUCKET). The existing CI-repository dispatch does not +establish that credential path; replace it with a reviewed MatrixOne caller +before execution. Missing credentials reject the run; values are never printed. The currently configured MatrixOne UT pool is `amd64-mo-shanghai-8c16g`; this hosted-only first measurement does not establish performance on that pool. Rollout to that pool additionally requires a matching ephemeral runner experiment. -After this workflow is merged, dispatch Race seed canary on main with a full +Only after the isolation and caller blockers above are resolved and the updated +workflow is reviewed and merged, dispatch Race seed canary on main with a full 40-character MatrixOne SHA and the immutable image reference above. Start with repetitions=1 (four measured arms plus one warm preparation run); repetitions=3 gives twelve measured arms and the same one preparation run. Review the report diff --git a/scripts/race_seed_canary.py b/scripts/race_seed_canary.py index 6c0cafa..644a8f5 100644 --- a/scripts/race_seed_canary.py +++ b/scripts/race_seed_canary.py @@ -86,6 +86,9 @@ def fingerprint(): 'runner_label': os.environ['CANARY_RUNNER_LABEL'], 'cpu_model': sorted(set(re.findall(r'^model name\s*:\s*(.*)', cpu, re.M))), 'cpu_count': os.cpu_count(), + 'cgroup_limits': {name: Path('/sys/fs/cgroup', name).read_text().strip() + for name in ('cpu.max', 'memory.max', 'cpuset.cpus.effective') + if Path('/sys/fs/cgroup', name).exists()}, 'mem_total': re.search(r'^MemTotal:.*', mem, re.M)[0], 'go': json.loads(command(['go', 'env', '-json', 'GOVERSION', 'GOOS', 'GOARCH', 'GOAMD64', 'GOEXPERIMENT', 'CGO_ENABLED', 'GOFLAGS', @@ -104,6 +107,8 @@ def sample(): 'cpu': '/proc/stat', 'memory': '/proc/meminfo', 'disk': '/proc/diskstats', 'pressure_io': '/proc/pressure/io', 'pressure_cpu': '/proc/pressure/cpu', 'cgroup_cpu': '/sys/fs/cgroup/cpu.stat', + 'cgroup_cpu_limit': '/sys/fs/cgroup/cpu.max', + 'cgroup_memory_limit': '/sys/fs/cgroup/memory.max', 'cgroup_memory': '/sys/fs/cgroup/memory.current', 'cgroup_memory_events': '/sys/fs/cgroup/memory.events', 'cgroup_io': '/sys/fs/cgroup/io.stat', @@ -139,6 +144,47 @@ def measured_command(args, log, timeout): return result +def cgroup_summary(samples): + """Container-scope evidence; never substitute host counters for missing data. + + Scope is the visible cgroup root, not a claim about hidden ancestors or a + Docker daemon running in another container. + """ + keys = ('cgroup_cpu', 'cgroup_cpu_limit', 'cgroup_memory', + 'cgroup_memory_limit', 'cgroup_memory_events', 'monotonic') + if any(any(row.get(key) is None for key in keys) for row in samples): + return {'available': False, 'reason': 'incomplete cgroup v2 samples'} + def counters(text): + return {key: int(value) for key, value in (line.split() for line in text.splitlines())} + limits = {(r['cgroup_cpu_limit'].strip(), r['cgroup_memory_limit'].strip()) for r in samples} + if len(limits) != 1: + raise ValueError('cgroup limits changed during measurement') + cpu_limit, memory_limit = next(iter(limits)) + quota, period = cpu_limit.split() + if int(period) <= 0 or (quota != 'max' and int(quota) <= 0): + raise ValueError('invalid cgroup CPU quota') + cpus = None if quota == 'max' else int(quota) / int(period) + elapsed = samples[-1]['monotonic'] - samples[0]['monotonic'] + if elapsed <= 0: + raise ValueError('nonpositive measurement interval') + cpu_rows = [counters(r['cgroup_cpu']) for r in samples] + event_rows = [counters(r['cgroup_memory_events']) for r in samples] + def delta(rows, key): + values = [r[key] for r in rows] + if any(b < a for a, b in zip(values, values[1:])): + raise ValueError('cgroup counter reset: ' + key) + return values[-1] - values[0] + used = delta(cpu_rows, 'usage_usec') / 1e6 + return {'available': True, 'scope': 'visible cgroup root; external daemon excluded', + 'cpu_quota_cores': cpus, 'cpu_seconds': used, + 'cpu_quota_utilization_percent': 100 * used / elapsed / cpus if cpus else None, + 'cpu_throttled_seconds': delta(cpu_rows, 'throttled_usec') / 1e6, + 'memory_limit_bytes': None if memory_limit == 'max' else int(memory_limit), + 'sampled_peak_memory_bytes': max(int(r['cgroup_memory']) for r in samples), + 'oom_events': delta(event_rows, 'oom'), + 'oom_kill_events': delta(event_rows, 'oom_kill')} + + def resource_summary(path): with path.open() as stream: samples = [json.loads(line) for line in stream] @@ -156,7 +202,8 @@ def cpu(row): busy.append(100 * (1 - (d[3] + d[4]) / sum(d))) mem = [int(re.search(r'^MemAvailable:\s+(\d+)', r['memory'], re.M)[1]) * 1024 for r in samples] - return {'host_cpu_busy_percent': 100 * (1 - (delta[3] + delta[4]) / ticks) if ticks else 0, + return {'cgroup': cgroup_summary(samples), + 'host_cpu_busy_percent': 100 * (1 - (delta[3] + delta[4]) / ticks) if ticks else 0, 'host_cpu_peak_interval_percent': max(busy, default=0), 'host_cpu_seconds': (ticks - delta[3] - delta[4]) / os.sysconf('SC_CLK_TCK'), 'min_memory_available_bytes': min(mem), diff --git a/scripts/test_race_seed_canary.py b/scripts/test_race_seed_canary.py index 2fa29c2..52a6b89 100644 --- a/scripts/test_race_seed_canary.py +++ b/scripts/test_race_seed_canary.py @@ -6,11 +6,32 @@ import unittest from unittest import mock -from race_seed_canary import compare, execution, summarize, resource_summary +from race_seed_canary import compare, execution, summarize, resource_summary, cgroup_summary from race_seed_plan import matrix class EvidenceTests(unittest.TestCase): + def test_cgroup_metrics_use_quota_not_host_cpu_count(self): + rows = [dict(monotonic=i * 10, cgroup_cpu=f'usage_usec {i * 40000000}\nthrottled_usec {i * 1000000}', + cgroup_cpu_limit='800000 100000', cgroup_memory_limit='17179869184', + cgroup_memory=str(100 + i * 50), cgroup_memory_events=f'oom {i}\noom_kill 0') + for i in range(2)] + result = cgroup_summary(rows) + self.assertEqual(result['cpu_quota_utilization_percent'], 50) + self.assertEqual(result['cpu_seconds'], 40) + self.assertEqual(result['cpu_throttled_seconds'], 1) + self.assertEqual(result['sampled_peak_memory_bytes'], 150) + self.assertEqual(result['oom_events'], 1) + self.assertFalse(cgroup_summary([{}, {}])['available']) + unlimited = [dict(r, cgroup_cpu_limit='max 100000', cgroup_memory_limit='max') for r in rows] + self.assertIsNone(cgroup_summary(unlimited)['cpu_quota_utilization_percent']) + for key, value in [('cgroup_cpu_limit', '400000 100000'), + ('monotonic', 0), ('cgroup_cpu', 'usage_usec -1\nthrottled_usec 0')]: + changed = copy.deepcopy(rows) + changed[1][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + cgroup_summary(changed) + def read(self, events): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'raw.json' From 6da76ff0f5acfd7cdd5d3727fc788bea7934a300 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 20 Sep 2026 05:45:10 +0800 Subject: [PATCH 3/6] docs(canary): record verified Shanghai Docker prerequisite --- docs/race-seed-canary.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/race-seed-canary.md b/docs/race-seed-canary.md index 2b64b74..39e3bfe 100644 --- a/docs/race-seed-canary.md +++ b/docs/race-seed-canary.md @@ -10,13 +10,27 @@ throttling, sampled memory peak and OOM counter deltas. They do not attribute an external Docker daemon's CPU to the test container, nor claim hidden ancestor limits are known. -Pool isolation remains **unverified**: the organization runners API and the -configured Kubernetes autoscaling-runner-set listing both reject access (403). -Before replacing the hosted guard, obtain a sanitized runner template confirming -per-job lifecycle and volume ownership for checkout, Go caches and Docker storage. -Also resolve a MatrixOne-side caller that inherits existing UT credentials; -do not copy credentials into CI merely to make this draft runnable. No runner -configuration, production defaults or cache contents were changed by inspection. +Subsequent read-only inspection through the `idc` environment resolved the +earlier access blocker. The pool template and a running pod confirm one runner +container, requests/limits of 8 CPU / 16 GiB, an `EphemeralRunner` owner and an +`emptyDir` mounted at `/home/runner/_work`. There are no cache PVC/hostPath mounts, +Docker socket mounts or Docker sidecars in the observed pod. + +**The actual execution blocker is Docker availability.** A read-only probe in +that runner found `/usr/bin/docker`, but `docker info` exited 1 because +`/var/run/docker.sock` does not exist. Neither `DOCKER_HOST` nor `DOCKER_CONTEXT` +was set. The current importer requires `docker info`, pull, create and cp; an +available builder image therefore does not make seeding usable on this pool. +This observation covers the inspected template/pod, not all possible runner +configurations. Enabling seed here without addressing acquisition would produce +a failed import/fallback, not a valid B arm or evidence of speedup. + +Before replacing the hosted guard, choose and review either daemonless image +extraction or a runner infrastructure change providing an isolated Docker daemon. +The latter changes the resource envelope and is not authorized by a measurement +PR alone. Also resolve a MatrixOne-side caller that inherits existing UT +credentials; do not copy credentials into CI merely to make this draft runnable. +No runner configuration, production defaults or cache contents were changed. The canary compares the complete race suite with seed disabled/enabled, on a fixed MatrixOne main commit and builder digest. Production defaults stay off. @@ -79,7 +93,7 @@ The currently configured MatrixOne UT pool is `amd64-mo-shanghai-8c16g`; this hosted-only first measurement does not establish performance on that pool. Rollout to that pool additionally requires a matching ephemeral runner experiment. -Only after the isolation and caller blockers above are resolved and the updated +Only after the image-acquisition and caller blockers above are resolved and the updated workflow is reviewed and merged, dispatch Race seed canary on main with a full 40-character MatrixOne SHA and the immutable image reference above. Start with repetitions=1 (four measured arms plus one warm preparation run); repetitions=3 From 21058b0c4a0224cf3cec844a82945f023fc7c2c2 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 20 Sep 2026 06:32:33 +0800 Subject: [PATCH 4/6] feat(ci): measure race cache seeding on daemonless Shanghai runners --- .github/actionlint.yaml | 1 + .github/workflows/race-seed-canary.yaml | 30 +++- .github/workflows/race-seed-sample.yaml | 45 ++++-- actions/seed-go-caches/action.yaml | 4 + actions/seed-go-caches/registry.py | 179 +++++++++++++++++++++++ actions/seed-go-caches/seed.py | 135 ++++++++++++------ actions/seed-go-caches/test_registry.py | 144 +++++++++++++++++++ actions/seed-go-caches/test_seed.py | 78 ++++++++++ docs/race-seed-canary.md | 182 ++++++++++-------------- docs/race-seed-daemonless-design.md | 145 +++++++++++++++++++ docs/race-seed-validation.md | 46 ++++++ docs/ut-cache-seeding.md | 6 + scripts/race_seed_canary.py | 46 +++--- scripts/race_seed_plan.py | 8 ++ scripts/test_race_seed_canary.py | 65 +++++++++ 15 files changed, 928 insertions(+), 186 deletions(-) create mode 100644 actions/seed-go-caches/registry.py create mode 100644 actions/seed-go-caches/test_registry.py create mode 100644 docs/race-seed-daemonless-design.md create mode 100644 docs/race-seed-validation.md diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 81b9941..a675a7f 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,5 +1,6 @@ self-hosted-runner: labels: + - amd64-mo-shanghai-8c16g - amd64-mo-guangzhou-2xlarge16 - amd64-mo-guangzhou-2xlarge32 - amd64-mo-guangzhou-medium8 diff --git a/.github/workflows/race-seed-canary.yaml b/.github/workflows/race-seed-canary.yaml index 4056ee9..6f2fe21 100644 --- a/.github/workflows/race-seed-canary.yaml +++ b/.github/workflows/race-seed-canary.yaml @@ -1,8 +1,12 @@ name: Race seed canary on: - workflow_dispatch: + workflow_call: inputs: + ci_sha: + description: Same immutable CI commit as the reusable workflow reference + required: true + type: string matrixone_sha: description: Full SHA reachable from matrixorigin/matrixone main required: true @@ -11,12 +15,21 @@ on: description: Builder repository@sha256 digest (never a mutable tag) required: true type: string - default: matrixorigin/matrixone@sha256:8137d222d3a3b29d119173639cea98931168ab7d886bf882894391f7804d5d88 repetitions: description: One smoke pair per cache state, or three measured pairs - type: choice - options: ['1', '3'] + type: string default: '1' + secrets: + S3ENDPOINT: + required: true + S3REGION: + required: true + S3APIKEY: + required: true + S3APISECRET: + required: true + S3BUCKET: + required: true permissions: contents: read @@ -27,7 +40,7 @@ concurrency: jobs: plan: - if: github.ref == 'refs/heads/main' + if: github.repository == 'matrixorigin/matrixone' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' runs-on: ubuntu-22.04 timeout-minutes: 5 outputs: @@ -35,6 +48,8 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + repository: matrixorigin/CI + ref: ${{ inputs.ci_sha }} persist-credentials: false - id: plan env: @@ -42,12 +57,14 @@ jobs: SOURCE_SHA: ${{ inputs.matrixone_sha }} SEED_IMAGE: ${{ inputs.image }} REPETITIONS: ${{ inputs.repetitions }} + CI_SHA: ${{ inputs.ci_sha }} run: python3 scripts/race_seed_plan.py warm-snapshot: needs: plan uses: ./.github/workflows/race-seed-sample.yaml with: + ci_sha: ${{ inputs.ci_sha }} matrixone_sha: ${{ inputs.matrixone_sha }} image: ${{ inputs.image }} name: warm-preparation @@ -63,6 +80,7 @@ jobs: matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} uses: ./.github/workflows/race-seed-sample.yaml with: + ci_sha: ${{ inputs.ci_sha }} matrixone_sha: ${{ inputs.matrixone_sha }} image: ${{ inputs.image }} name: ${{ matrix.name }} @@ -78,6 +96,8 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + repository: matrixorigin/CI + ref: ${{ inputs.ci_sha }} persist-credentials: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: diff --git a/.github/workflows/race-seed-sample.yaml b/.github/workflows/race-seed-sample.yaml index 2f947d1..862b486 100644 --- a/.github/workflows/race-seed-sample.yaml +++ b/.github/workflows/race-seed-sample.yaml @@ -3,6 +3,9 @@ name: Race seed measurement sample on: workflow_call: inputs: + ci_sha: + required: true + type: string matrixone_sha: required: true type: string @@ -35,33 +38,50 @@ permissions: jobs: sample: - # Fresh hosted runners only: no production runner caches are cleared. - runs-on: ${{ vars.RACE_CANARY_RUNNER_LABEL || 'ubuntu-22.04' }} + # Verified ARC pool: ephemeral pod, emptyDir workspace, no shared cache mount. + runs-on: amd64-mo-shanghai-8c16g environment: ci timeout-minutes: 120 steps: - - name: Require an ephemeral hosted runner before any checkout + - name: Require the audited pool and MatrixOne caller before any checkout id: runner_guard env: RUNNER_KIND: ${{ runner.environment }} - run: test "$RUNNER_KIND" = github-hosted + run: | + test "$RUNNER_KIND" = self-hosted + test "$GITHUB_REPOSITORY" = matrixorigin/matrixone + test "$GITHUB_WORKSPACE" = /home/runner/_work/matrixone/matrixone + case "$RUNNER_NAME" in amd64-mo-shanghai-8c16g-*-runner-*) ;; *) exit 1;; esac - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + repository: matrixorigin/matrixone + ref: ${{ inputs.matrixone_sha }} persist-credentials: false - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - repository: matrixorigin/matrixone - ref: ${{ inputs.matrixone_sha }} - path: subject + repository: matrixorigin/CI + ref: ${{ inputs.ci_sha }} + path: _canary persist-credentials: false - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version-file: subject/go.mod + go-version-file: go.mod cache: false - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4 with: distribution: adopt java-version: '8' + - name: Match the existing UT CMake setup + uses: ./_canary/actions/setup-cmake + - name: Match the Shanghai UT module proxy policy + shell: bash + run: | + set -euo pipefail + proxy='http://goproxy.goproxy.svc.cluster.local' + version="$(awk '$1 == "github.com/spf13/cobra" {print $2; exit}' go.mod)" + if test -n "$version" && curl --disable --fail --silent --connect-timeout 5 --max-time 30 "$proxy/github.com/spf13/cobra/@v/$version.info" >/dev/null; then + echo "GOPROXY=$proxy|https://goproxy.cn|direct" >> "$GITHUB_ENV" + fi - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 if: inputs.cache_state == 'warm' with: @@ -71,9 +91,12 @@ jobs: env: SEED_IMAGE: ${{ inputs.image }} SEED_FLAVOR: race + SEED_TRANSPORT: registry ENABLE_SEED: ${{ inputs.seed }} CACHE_STATE: ${{ inputs.cache_state }} - CANARY_RUNNER_LABEL: ${{ vars.RACE_CANARY_RUNNER_LABEL || 'ubuntu-22.04' }} + CANARY_RUNNER_LABEL: amd64-mo-shanghai-8c16g + CANARY_SOURCE_SHA: ${{ inputs.matrixone_sha }} + CANARY_CI_SHA: ${{ inputs.ci_sha }} CANARY_UT_PARALLEL: ${{ vars.UT_PARALLEL || 6 }} CANARY_UT_TIMEOUT: ${{ vars.UT_TIMEOUT || 40 }} GOCACHE: /home/runner/.cache/mo-race-canary @@ -94,12 +117,12 @@ jobs: prepare) args+=(--export "$GITHUB_WORKSPACE/warm-snapshot");; *) exit 1;; esac - python3 scripts/race_seed_canary.py "${args[@]}" + python3 _canary/scripts/race_seed_canary.py "${args[@]}" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: ${{ always() && steps.runner_guard.outcome == 'success' }} with: name: ${{ inputs.name }} - path: canary-report/ + path: _canary/canary-report/ retention-days: 7 if-no-files-found: error - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/actions/seed-go-caches/action.yaml b/actions/seed-go-caches/action.yaml index a8b0664..0e1adfa 100644 --- a/actions/seed-go-caches/action.yaml +++ b/actions/seed-go-caches/action.yaml @@ -1,6 +1,9 @@ name: Seed Go caches description: Best-effort additive import from the trusted CI builder, preserving existing caches inputs: + transport: + description: Acquisition backend; registry is opt-in and requires a pinned image + default: docker image: description: Optional trusted repository at an immutable sha256 digest for paired measurements default: '' @@ -16,6 +19,7 @@ runs: - name: Import trusted Go cache entries shell: bash env: + SEED_TRANSPORT: ${{ inputs.transport }} SEED_IMAGE: ${{ inputs.image }} SEED_FLAVOR: ${{ inputs.flavor }} SEED_GENERATION: ${{ inputs.generation }} diff --git a/actions/seed-go-caches/registry.py b/actions/seed-go-caches/registry.py new file mode 100644 index 0000000..37690d2 --- /dev/null +++ b/actions/seed-go-caches/registry.py @@ -0,0 +1,179 @@ +"""Narrow, digest-pinned COPY-layer transport for the trusted ci-builder format. + +Not a general OCI rootfs merger: the trusted producer contract isolates the +selected caches from omitted layers. History checks detect layout drift; they +are not proof of arbitrary layer contents. No image code is executed. +""" +import gzip +import hashlib +import json +from pathlib import PurePosixPath +import re +import shutil +import tarfile + +VERSION = '0.22.1' +SHA256 = '0ab7a1d6932a213aed964ce97666c3077fe691c8606413674a8b3e0b9ec4cda0' +URL = f'https://github.com/google/go-containerregistry/releases/download/v{VERSION}/go-containerregistry_Linux_x86_64.tar.gz' +PAYLOAD_LIMIT = 24 * 1024**3 +BLOB_LIMIT = 8 * 1024**3 +TOOL_LIMIT = 32 * 1024**2 +METADATA_LIMIT = 65536 +COPY_PATHS = ( + ('/home/runner/go/pkg/mod', '/go/pkg/mod'), + ('/root/.cache/go-build', '/root/.cache/go-build'), + ('/home/runner/_work/matrixone/matrixone/thirdparties/install', '/mo-prebuilt/thirdparties/install'), + ('/mo-prebuilt/thirdparties.fingerprint', '/mo-prebuilt/thirdparties.fingerprint'), + ('/mo-prebuilt/warm-status', '/mo-prebuilt/warm-status'), + ('/mo-prebuilt/go-cache-manifest.json', '/mo-prebuilt/go-cache-manifest.json'), +) + + +def digest(data): + return 'sha256:' + hashlib.sha256(data).hexdigest() + + +def descriptor(value, limit): + if (not isinstance(value, dict) + or not re.fullmatch(r'sha256:[0-9a-f]{64}', str(value.get('digest', ''))) + or type(value.get('size')) is not int or not 0 < value['size'] <= limit): + raise ValueError('invalid or oversized registry descriptor') + return value + + +def layout(manifest, config): + layers = manifest.get('layers', []) + history = [h.get('created_by') for h in config.get('history', []) if not h.get('empty_layer')] + diff_ids = config.get('rootfs', {}).get('diff_ids', []) + expected = [f'COPY {src} {dst} # buildkit' for src, dst in COPY_PATHS] + if (config.get('os') != 'linux' or config.get('architecture') != 'amd64' + or len(layers) < 6 or len(layers) != len(history) or len(layers) != len(diff_ids) + or history[-6:] != expected): + raise ValueError('unsupported ci-builder COPY-layer layout') + selected = {} + for (_, path), layer, diff_id in zip(COPY_PATHS, layers[-6:], diff_ids[-6:]): + if layer.get('mediaType') not in ('application/vnd.oci.image.layer.v1.tar+gzip', + 'application/vnd.docker.image.rootfs.diff.tar.gzip'): + raise ValueError('unsupported cache layer encoding') + if not re.fullmatch(r'sha256:[0-9a-f]{64}', str(diff_id)): + raise ValueError('invalid uncompressed layer digest') + selected[path] = (descriptor(layer, BLOB_LIMIT), diff_id) + descriptor(selected[COPY_PATHS[-1][1]][0], METADATA_LIMIT) + return selected + + +class DigestReader: + def __init__(self, stream, limit): + self.stream, self.limit = stream, limit + self.hash = hashlib.sha256() + self.size = 0 + + def read(self, size): + data = self.stream.read(min(size, 1024 * 1024)) + self.size += len(data) + if self.size > self.limit: + raise ValueError('uncompressed cache layers exceed budget') + self.hash.update(data) + return data + + def finish(self, expected): + # tar stops at its end marker, but the gzip/diff_id contract covers the + # entire uncompressed stream, including padding and concatenated members. + while self.read(1024 * 1024): + pass + if 'sha256:' + self.hash.hexdigest() != expected: + raise ValueError('uncompressed layer digest mismatch') + + +class RegistryImage: + def __init__(self, seeder, cache): + self.seeder = seeder + self.stage = seeder.temporary(cache) + self.tool = self.stage / 'crane' + self.remaining = PAYLOAD_LIMIT + + def command(self, *args, **kwargs): + return self.seeder.command([str(self.tool), *args], **kwargs) + + def acquire(self, image): + seeder = self.seeder + config = self.stage / 'auth' + config.mkdir() + (config / 'config.json').write_text('{}\n') + seeder.env.update(DOCKER_CONFIG=str(config), XDG_CONFIG_HOME=str(config), + XDG_RUNTIME_DIR=str(config), REGISTRY_AUTH_FILE=str(config / 'absent')) + tool_archive = self.stage / 'tool.tar.gz' + with tool_archive.open('wb') as out: + seeder.command(['curl', '--disable', '--fail', '--silent', '--show-error', '--location', + '--proto', '=https', '--proto-redir', '=https', + '--connect-timeout', '10', '--max-time', '120', URL], + output=out, timeout=125, max_bytes=TOOL_LIMIT) + if hashlib.sha256(tool_archive.read_bytes()).hexdigest() != SHA256: + raise ValueError('crane release checksum mismatch') + with tarfile.open(tool_archive, 'r:gz') as archive: + member = archive.getmember('crane') + if not member.isfile() or member.size > 64 * 1024**2: + raise ValueError('invalid crane executable') + with archive.extractfile(member) as src, self.tool.open('xb') as dst: + shutil.copyfileobj(src, dst, 1024 * 1024) + self.tool.chmod(0o700) + tool_archive.unlink() + self.repository, expected = image.split('@') + raw = self.command('manifest', image, max_bytes=METADATA_LIMIT) + if digest(raw) != expected: + raise ValueError('image manifest digest mismatch') + manifest = json.loads(raw) + config_desc = descriptor(manifest.get('config'), METADATA_LIMIT) + raw = self.command('blob', self.repository + '@' + config_desc['digest'], + max_bytes=config_desc['size']) + if len(raw) != config_desc['size'] or digest(raw) != config_desc['digest']: + raise ValueError('image config digest mismatch') + self.layers = layout(manifest, json.loads(raw)) + return dict(Id=image, Os='linux', Architecture='amd64', Size=PAYLOAD_LIMIT) + + def blob(self, source): + desc, diff_id = self.layers[source] + path = self.stage / 'layer.tar.gz' + with path.open('xb') as out: + self.command('blob', self.repository + '@' + desc['digest'], output=out, + timeout=900, max_bytes=desc['size']) + hashed = hashlib.sha256() + with path.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + hashed.update(block) + if path.stat().st_size != desc['size'] or 'sha256:' + hashed.hexdigest() != desc['digest']: + raise ValueError('compressed layer digest or size mismatch') + return path, diff_id + + def manifest(self): + path, expected = self.blob(COPY_PATHS[-1][1]) + metadata = None + seen = False + with gzip.open(path, 'rb') as compressed: + reader = DigestReader(compressed, METADATA_LIMIT) + with tarfile.open(fileobj=reader, mode='r|') as archive: + for member in archive: + archive.members.clear() + name = PurePosixPath(member.name) + if member.isdir() and name.as_posix() == 'mo-prebuilt': + continue + if (name.as_posix() != 'mo-prebuilt/go-cache-manifest.json' or not member.isfile() + or member.size > 8192 or seen): + raise ValueError('invalid producer metadata layer') + seen = True + metadata = json.load(archive.extractfile(member)) + reader.finish(expected) + path.unlink() + if metadata is None: + raise ValueError('missing producer metadata') + return metadata + + def extract(self, source, destination, root, budget, build, extractor): + path, expected = self.blob(source) + with gzip.open(path, 'rb') as compressed: + reader = DigestReader(compressed, self.remaining) + size = extractor(reader, destination, root, budget, build, prefix=source.lstrip('/')) + reader.finish(expected) + self.remaining -= reader.size + path.unlink() + return size diff --git a/actions/seed-go-caches/seed.py b/actions/seed-go-caches/seed.py index 2049b46..77cb51f 100644 --- a/actions/seed-go-caches/seed.py +++ b/actions/seed-go-caches/seed.py @@ -15,6 +15,7 @@ import tempfile import time import uuid +import zlib SCHEMA = 2 PROFILE = "host-ut-v1" @@ -67,17 +68,32 @@ def space_available(requirements): for path, size in devices.values()) -def extract(archive, destination, root, budget, build=False): +def extract(archive, destination, root, budget, build=False, prefix=None): """No tar extraction APIs: allow only directories/regular files under root.""" total = 0 - with tarfile.open(archive, "r|*") as stream: + found = False + prefix_parts = PurePosixPath(prefix).parts if prefix else None + opener = ({"fileobj": archive, "mode": "r|"} if hasattr(archive, 'read') + else {"name": archive, "mode": "r|*"}) + with tarfile.open(**opener) as stream: for member in stream: + stream.members.clear() name = PurePosixPath(member.name) if name.is_absolute() or ".." in name.parts or not name.parts: raise ValueError("unsafe archive path") - if name.parts[0] != root: - raise ValueError("unexpected archive root") - relative = name.parts[1:] + if prefix_parts: + if any(part.startswith('.wh.') for part in name.parts): + raise ValueError('cache COPY layer contains whiteout') + if name.parts[:len(prefix_parts)] != prefix_parts: + if member.isdir() and prefix_parts[:len(name.parts)] == name.parts: + continue + raise ValueError('unexpected cache COPY layer entry') + relative = name.parts[len(prefix_parts):] + else: + if name.parts[0] != root: + raise ValueError("unexpected archive root") + relative = name.parts[1:] + found = True if not (member.isdir() or member.isfile()): raise ValueError("archive links/special files are not allowed") if not relative: @@ -108,6 +124,8 @@ def extract(archive, destination, root, budget, build=False): with stream.extractfile(member) as source, target.open("xb") as output: shutil.copyfileobj(source, output, 1024 * 1024) target.chmod(0o755 if member.mode & 0o111 else 0o644) + if prefix_parts and not found: + raise ValueError('missing cache COPY subtree') if build: for shard in destination.iterdir(): for entry in shard.iterdir(): @@ -183,6 +201,7 @@ def __init__(self, flavor, generation): self.cleanup_remaining = 45.0 self.cleanup_each = 15.0 self.pending_marker = None + self.registry = None self.report = {"state": "failed", "flavor": flavor, "module_state": "not-attempted", "imported_bytes": 0, "producer_go_version": "unknown"} @@ -322,18 +341,25 @@ def payload(self, cache, source, root, budget, build=False): archive = stage / "payload.tar" data = stage / "data" data.mkdir() - with archive.open("wb") as output: - self.docker("cp", f"{self.container}:{source}", "-", - output=output, timeout=300) - extracted = extract(archive, data, root, budget, build) + if self.registry: + extracted = self.registry.extract(source, data, root, budget, build, extract) + else: + with archive.open("wb") as output: + self.docker("cp", f"{self.container}:{source}", "-", + output=output, timeout=300) + extracted = extract(archive, data, root, budget, build) + archive.unlink() if build and extracted == 0: raise ValueError("empty build cache payload") - archive.unlink() return data def seed(self, stack): pinned = self.env.get("SEED_IMAGE", "") images = image_candidates(pinned, self.env.get("RUNNER_ENVIRONMENT") == "github-hosted") + transport = self.env.get("SEED_TRANSPORT", "docker") + if transport not in ("docker", "registry") or (transport == "registry" and not pinned): + raise ValueError("registry transport requires a trusted immutable image") + self.report["transport"] = transport values = json.loads(self.command([ "go", "env", "-json", "GOCACHE", "GOMODCACHE", "GOVERSION", "GOOS", "GOARCH", "GOAMD64", "GOEXPERIMENT", "GOCACHEPROG", "GOMOD"])) @@ -361,6 +387,8 @@ def seed(self, stack): "profile": PROFILE, "contracts": CONTRACTS, "checkout": CHECKOUT} if pinned: key["image"] = pinned + if transport == "registry": + key["transport"] = transport marker = cache / MARKER if marker.is_symlink(): raise ValueError("symlink completion record") @@ -376,24 +404,34 @@ def seed(self, stack): imported_bytes=0, imported_files=0, module_state="previous-import", acquisition_seconds=0, build_import_seconds=0, module_import_seconds=0) return - # An empty config prevents reuse of runner registry credentials. - config = self.temporary(prefix="mo-docker-config-") - self.env["DOCKER_CONFIG"] = str(config) - docker_root = Path(self.docker("info", "--format", "{{.DockerRootDir}}").decode().strip()) - if not docker_root.is_absolute() or not docker_root.is_dir(): - self.report["state"] = "storage-unavailable" - return - if not space_available([(docker_root, 0), (cache, 0), (modules, 0)]): - self.report["state"] = "insufficient-space" - return - image = None - for candidate in images: - try: - self.docker("pull", candidate, timeout=300) - image = json.loads(self.docker("image", "inspect", candidate))[0] - break - except (subprocess.CalledProcessError, subprocess.TimeoutExpired): - continue + if transport == "registry": + from registry import RegistryImage, PAYLOAD_LIMIT, BLOB_LIMIT + # One compressed blob plus extracted caches; no whole rootfs export. + if not space_available([(cache, PAYLOAD_LIMIT + BLOB_LIMIT), (modules, PAYLOAD_LIMIT)]): + self.report["state"] = "insufficient-space" + return + self.registry = RegistryImage(self, cache) + docker_root = self.registry.stage + image = self.registry.acquire(pinned) + else: + # An empty config prevents reuse of runner registry credentials. + config = self.temporary(prefix="mo-docker-config-") + self.env["DOCKER_CONFIG"] = str(config) + docker_root = Path(self.docker("info", "--format", "{{.DockerRootDir}}").decode().strip()) + if not docker_root.is_absolute() or not docker_root.is_dir(): + self.report["state"] = "storage-unavailable" + return + if not space_available([(docker_root, 0), (cache, 0), (modules, 0)]): + self.report["state"] = "insufficient-space" + return + image = None + for candidate in images: + try: + self.docker("pull", candidate, timeout=300) + image = json.loads(self.docker("image", "inspect", candidate))[0] + break + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + continue if image is None: self.report["state"] = "unavailable" return @@ -413,23 +451,19 @@ def seed(self, stack): return # Establish an unpredictable owned name BEFORE creating the resource: # timeout/cancellation may lose stdout after the daemon created it. - self.container = "mo-go-seed-" + uuid.uuid4().hex - identity = self.docker("create", "--name", self.container, image["Id"]).decode().strip() - if not re.fullmatch(r"[0-9a-f]{12,64}", identity): - raise ValueError("unexpected container identity") + if not self.registry: + self.container = "mo-go-seed-" + uuid.uuid4().hex + identity = self.docker("create", "--name", self.container, image["Id"]).decode().strip() + if not re.fullmatch(r"[0-9a-f]{12,64}", identity): + raise ValueError("unexpected container identity") try: - # Reject legacy/incompatible producers BEFORE large cache payloads. + # Both transports validate the small producer manifest BEFORE + # acquiring large cache payloads or publishing any cache entries. with tempfile.TemporaryFile() as metadata: - self.docker("cp", f"{self.container}:/mo-prebuilt/go-cache-manifest.json", "-", - output=metadata, timeout=15, max_bytes=65536) - if metadata.tell() > 65536: - raise ValueError("oversize metadata archive") - metadata.seek(0) - with tarfile.open(fileobj=metadata, mode="r:") as archive: - member = archive.getmember("go-cache-manifest.json") - if not member.isfile() or member.size > 8192: - raise ValueError("invalid manifest member") - manifest = json.load(archive.extractfile(member)) + if self.registry: + manifest = self.registry.manifest() + else: + manifest = self.docker_manifest(metadata) expected = {"schema": SCHEMA, "profile": PROFILE, "checkout": CHECKOUT, "go_env": {field: values[field] for field in GO_FIELDS}, **CONTRACTS} if not isinstance(manifest, dict) or any(manifest.get(k) != v for k, v in expected.items()): @@ -481,6 +515,18 @@ def seed(self, stack): completed_at=int(time.time())) self.pending_marker = cache + def docker_manifest(self, metadata): + self.docker("cp", f"{self.container}:/mo-prebuilt/go-cache-manifest.json", "-", + output=metadata, timeout=15, max_bytes=65536) + if metadata.tell() > 65536: + raise ValueError("oversize metadata archive") + metadata.seek(0) + with tarfile.open(fileobj=metadata, mode="r:") as archive: + member = archive.getmember("go-cache-manifest.json") + if not member.isfile() or member.size > 8192: + raise ValueError("invalid manifest member") + return json.load(archive.extractfile(member)) + def commit_marker(self): # File creation, publication and cleanup form one short ownership # transition. Signals are remembered; a cancelled commit is removed. @@ -523,7 +569,8 @@ def run(self): try: signal.alarm(1080) self.seed(stack) - except (OSError, ValueError, tarfile.TarError, subprocess.SubprocessError, + except (OSError, ValueError, KeyError, TypeError, AttributeError, EOFError, + tarfile.TarError, zlib.error, subprocess.SubprocessError, TimeoutError) as error: self.report.update(state="failed", error=str(error)) finally: diff --git a/actions/seed-go-caches/test_registry.py b/actions/seed-go-caches/test_registry.py new file mode 100644 index 0000000..6b5f1dd --- /dev/null +++ b/actions/seed-go-caches/test_registry.py @@ -0,0 +1,144 @@ +"""COPY-layer transport contracts; no network or daemon required.""" +import copy +import gzip +import hashlib +import io +import json +from pathlib import Path +import tarfile +import tempfile +from unittest import mock +import unittest + +import registry +from test_seed import tar_bytes, MISSING, seed + + +def image_fixture(): + blobs = {} + layers, diff_ids = [], [] + for _, path in registry.COPY_PATHS: + raw = tar_bytes([(path.lstrip('/'), b'{}')]) + compressed = gzip.compress(raw) + desc = dict(digest=registry.digest(compressed), size=len(compressed), + mediaType='application/vnd.oci.image.layer.v1.tar+gzip') + blobs[desc['digest']] = compressed + layers.append(desc) + diff_ids.append(registry.digest(raw)) + config = dict(os='linux', architecture='amd64', rootfs={'diff_ids': diff_ids}, + history=[{'created_by': f'COPY {src} {dst} # buildkit'} for src, dst in registry.COPY_PATHS]) + raw_config = json.dumps(config).encode() + config_desc = dict(digest=registry.digest(raw_config), size=len(raw_config)) + blobs[config_desc['digest']] = raw_config + manifest = dict(schemaVersion=2, config=config_desc, layers=layers) + raw_manifest = json.dumps(manifest).encode() + reference = 'matrixorigin/matrixone@' + registry.digest(raw_manifest) + return manifest, config, raw_manifest, blobs, reference + + +class RegistryTests(unittest.TestCase): + def test_layout_rejects_drift_platform_compression_count_and_size(self): + manifest, config, *_ = image_fixture() + self.assertEqual(len(registry.layout(manifest, config)), 6) + for field, value in [('architecture', 'arm64'), ('history', []), ('rootfs', {})]: + with self.subTest(field=field), self.assertRaises(ValueError): + registry.layout(manifest, dict(config, **{field: value})) + for field, value in [('mediaType', 'zstd'), ('size', registry.BLOB_LIMIT + 1), ('digest', 'bad')]: + changed = copy.deepcopy(manifest) + changed['layers'][0][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + registry.layout(changed, config) + changed = copy.deepcopy(config) + changed['history'][-1]['created_by'] = 'RUN mutate cached files' + with self.assertRaises(ValueError): + registry.layout(manifest, changed) + + def test_digest_reader_hashes_trailing_bytes_and_enforces_budget(self): + raw = tar_bytes([]) + b'trailing bytes' + reader = registry.DigestReader(io.BytesIO(raw), len(raw)) + with tarfile.open(fileobj=reader, mode='r|') as archive: + self.assertEqual(list(archive), []) + reader.finish(registry.digest(raw)) + self.assertEqual(reader.size, len(raw)) + with self.assertRaises(ValueError): + registry.DigestReader(io.BytesIO(raw), 1).finish(registry.digest(raw)) + with self.assertRaises(ValueError): + registry.DigestReader(io.BytesIO(raw), len(raw)).finish('sha256:' + '0' * 64) + + def test_strict_direct_layer_extract_paths_whiteout_links_missing_and_pax(self): + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / 'data' + target.mkdir() + relative = 'example.test/' + 'a' * 120 + '/go.mod' + raw = tar_bytes([('go/pkg/mod/' + relative, b'module m')]) + self.assertEqual(seed.extract(io.BytesIO(raw), target, 'mod', 100, + prefix='go/pkg/mod'), 8) + self.assertEqual((target / relative).read_bytes(), b'module m') + link = tarfile.TarInfo('go/pkg/mod/link') + link.type, link.linkname = tarfile.SYMTYPE, '/etc/passwd' + for entries in ([], [('go/pkg/mod/.wh.deleted', b'')], [('unrelated', b'x')], + [('../escape', b'x')], [link]): + with self.subTest(entries=entries), self.assertRaises(ValueError): + seed.extract(io.BytesIO(tar_bytes(entries)), target, 'mod', 100, prefix='go/pkg/mod') + empty = tarfile.TarInfo('go/pkg/mod') + empty.type = tarfile.DIRTYPE + self.assertEqual(seed.extract(io.BytesIO(tar_bytes([empty])), target, 'mod', 100, + prefix='go/pkg/mod'), 0) + + def test_bootstrap_auth_and_integrity_metadata_before_large_payload(self): + with tempfile.TemporaryDirectory() as tmp: + stage = Path(tmp) + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode='w:gz') as tar: + member = tarfile.TarInfo('crane') + member.size = 4 + tar.addfile(member, io.BytesIO(b'tool')) + tool_bytes = raw.getvalue() + manifest, config, raw_manifest, blobs, reference = image_fixture() + calls = [] + instance = mock.Mock() + instance.env = {'REGISTRY_AUTH_FILE': '/unowned/auth', 'XDG_CONFIG_HOME': '/unowned/podman'} + instance.temporary.return_value = stage + def command(args, **kwargs): + calls.append((args, kwargs)) + self.assertEqual(json.loads((Path(instance.env['DOCKER_CONFIG']) / 'config.json').read_text()), {}) + if args[0] == 'curl': + data = tool_bytes + elif args[1] == 'manifest': + data = raw_manifest + else: + data = blobs[args[-1].split('@')[1]] + if 'output' in kwargs: + kwargs['output'].write(data) + else: + return data + instance.command.side_effect = command + with mock.patch.object(registry, 'SHA256', hashlib.sha256(tool_bytes).hexdigest()): + image = registry.RegistryImage(instance, stage) + image.acquire(reference) + self.assertEqual(image.manifest(), {}) + self.assertEqual(len(calls), 4) # tool, manifest, config, small metadata layer only + self.assertEqual(calls[-1][1]['max_bytes'], manifest['layers'][-1]['size']) + self.assertFalse(Path(instance.env['REGISTRY_AUTH_FILE']).exists()) + self.assertFalse((stage / 'layer.tar.gz').exists()) + # A digest failure never reaches the extractor or publication. + blobs[manifest['layers'][1]['digest']] = b'corrupt' + with self.assertRaisesRegex(ValueError, 'compressed layer'): + image.blob('/root/.cache/go-build') + + def test_checksum_failure_prevents_executable_install(self): + with tempfile.TemporaryDirectory() as tmp: + stage = Path(tmp) + instance = mock.Mock() + instance.env = {} + instance.temporary.return_value = stage + instance.command.side_effect = lambda args, **kwargs: kwargs['output'].write(b'bad download') + image = registry.RegistryImage(instance, stage) + with self.assertRaisesRegex(ValueError, 'checksum'): + image.acquire('matrixorigin/matrixone@sha256:' + '1' * 64) + self.assertFalse(image.tool.exists()) + self.assertEqual(instance.command.call_count, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/actions/seed-go-caches/test_seed.py b/actions/seed-go-caches/test_seed.py index 364858c..afef468 100644 --- a/actions/seed-go-caches/test_seed.py +++ b/actions/seed-go-caches/test_seed.py @@ -139,6 +139,84 @@ def docker(self, *args, **kwargs): class SeederTests(unittest.TestCase): + def test_registry_diff_id_and_truncated_gzip_reject_before_publication(self): + import registry + import gzip + for damage in ('diff-id', 'truncated', 'deflate'): + instance = self.make() + instance.env.update(SEED_TRANSPORT='registry', SEED_IMAGE='matrixorigin/matrixone@sha256:' + '1' * 64) + raw = tar_bytes([('root/.cache/go-build/' + MISSING, b'unpublished')]) + def blob(image, source): + compressed = gzip.compress(raw) + if damage == 'truncated': + compressed = compressed[:-4] + elif damage == 'deflate': + compressed = compressed[:10] + b'\x07' + compressed[11:] + path = image.stage / 'layer.tar.gz' + path.write_bytes(compressed) + return path, 'sha256:' + '0' * 64 if damage == 'diff-id' else registry.digest(raw) + with self.subTest(damage=damage), \ + mock.patch.object(registry.RegistryImage, 'acquire', return_value=instance.image), \ + mock.patch.object(registry.RegistryImage, 'manifest', return_value=instance.manifest), \ + mock.patch.object(registry.RegistryImage, 'blob', blob): + report = self.run_seed(instance) + self.assertEqual(report['state'], 'failed') + self.assertEqual(report['cleanup'], 'complete') + self.assertFalse((instance.cache / MISSING).exists()) + self.assert_no_marker(instance) + + def test_registry_partial_acquisition_cleans_without_completion(self): + import registry + instance = self.make() + instance.env.update(SEED_TRANSPORT='registry', SEED_IMAGE='matrixorigin/matrixone@sha256:' + '1' * 64) + def acquire(image, reference): + (image.stage / 'partial').write_bytes(b'partial') + raise TimeoutError('bounded export timeout') + with mock.patch.object(registry.RegistryImage, 'acquire', acquire): + report = self.run_seed(instance) + self.assertEqual(report['state'], 'failed') + self.assertEqual(report['cleanup'], 'complete') + self.assert_no_marker(instance) + self.assertEqual(instance.calls, []) + + def test_registry_import_preserves_cache_and_cleans_without_docker(self): + import registry + import gzip + instance = self.make() + instance.env.update(SEED_TRANSPORT='registry', SEED_IMAGE='matrixorigin/matrixone@sha256:' + '1' * 64) + def acquire(image, reference): + return dict(Id=reference, Os='linux', Architecture='amd64', Size=registry.PAYLOAD_LIMIT) + fetched = [] + def blob(image, source): + fetched.append(source) + entries = { + '/mo-prebuilt/go-cache-manifest.json': [('mo-prebuilt/go-cache-manifest.json', json.dumps(instance.manifest).encode())], + '/root/.cache/go-build': [('root/.cache/go-build/' + EXISTING, b'collision'), + ('root/.cache/go-build/' + MISSING, b'new')], + '/go/pkg/mod': [('go/pkg/mod/example.test/m@v1/m.go', b'package m')], + }[source] + raw = tar_bytes(entries) + path = image.stage / 'layer.tar.gz' + path.write_bytes(gzip.compress(raw)) + return path, registry.digest(raw) + with mock.patch.object(registry.RegistryImage, 'acquire', acquire), \ + mock.patch.object(registry.RegistryImage, 'blob', blob): + report = self.run_seed(instance) + self.assertIn('/go/pkg/mod', fetched) + fetched.clear() + # A later generation still imports build entries additively, but + # must not acquire the already-populated module layer at all. + second = self.make(generation='2') + second.env.update(instance.env) + second_report = self.run_seed(second) + self.assertEqual(second_report['state'], 'seeded') + self.assertNotIn('/go/pkg/mod', fetched) + self.assertEqual(report['state'], 'seeded') + self.assertEqual(report['cleanup'], 'complete') + self.assertEqual((instance.cache / EXISTING).read_bytes(), b'local probe') + self.assertEqual((instance.cache / MISSING).read_bytes(), b'new') + self.assertEqual(instance.calls, []) + def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) diff --git a/docs/race-seed-canary.md b/docs/race-seed-canary.md index 39e3bfe..a1d2c49 100644 --- a/docs/race-seed-canary.md +++ b/docs/race-seed-canary.md @@ -1,110 +1,76 @@ # Race cache seed measurement -**Draft / not ready for the production runner experiment.** Read-only inspection -on 2026-09-20 confirmed job 105924102623 uses `amd64-mo-shanghai-8c16g`, -canonical checkout/module paths, CPU quota `800000/100000` (8 cores), and -memory limit `17179869184` (16 GiB), while its visible cpuset is `0-95`. -Consequently host CPU utilization is only context, not runner utilization. -Reports now separately calculate visible-cgroup CPU seconds/quota utilization, -throttling, sampled memory peak and OOM counter deltas. They do not attribute -an external Docker daemon's CPU to the test container, nor claim hidden ancestor -limits are known. - -Subsequent read-only inspection through the `idc` environment resolved the -earlier access blocker. The pool template and a running pod confirm one runner -container, requests/limits of 8 CPU / 16 GiB, an `EphemeralRunner` owner and an -`emptyDir` mounted at `/home/runner/_work`. There are no cache PVC/hostPath mounts, -Docker socket mounts or Docker sidecars in the observed pod. - -**The actual execution blocker is Docker availability.** A read-only probe in -that runner found `/usr/bin/docker`, but `docker info` exited 1 because -`/var/run/docker.sock` does not exist. Neither `DOCKER_HOST` nor `DOCKER_CONTEXT` -was set. The current importer requires `docker info`, pull, create and cp; an -available builder image therefore does not make seeding usable on this pool. -This observation covers the inspected template/pod, not all possible runner -configurations. Enabling seed here without addressing acquisition would produce -a failed import/fallback, not a valid B arm or evidence of speedup. - -Before replacing the hosted guard, choose and review either daemonless image -extraction or a runner infrastructure change providing an isolated Docker daemon. -The latter changes the resource envelope and is not authorized by a measurement -PR alone. Also resolve a MatrixOne-side caller that inherits existing UT -credentials; do not copy credentials into CI merely to make this draft runnable. -No runner configuration, production defaults or cache contents were changed. - -The canary compares the complete race suite with seed disabled/enabled, on a -fixed MatrixOne main commit and builder digest. Production defaults stay off. -The manually dispatched workflow runs only from CI main. All harness code comes -from its immutable workflow commit. MatrixOne must be an ancestor of main, -because the suite uses the existing CI test credentials. - -Each arm gets a fresh GitHub-hosted Linux x64 runner. Cold arms start without -Go caches or the builder image. Warm arms restore the **same immutable artifact** -produced by one successful full race run without seeding. This snapshot has no -seed completion marker: warm means an existing compiler/module cache, not a -previous seed import. The already-seeded marker fast path is a separate future -experiment. The matrix requests alternating AB/BA order and runs one arm at a -time; matrix scheduling is not proof of execution order. Check job timestamps. - -The comparison rejects different source/harness/image identities, snapshot -hashes, runner image versions, CPU/memory/toolchains or UT settings. OS page -cache and network conditions remain noise; pair repetition measures that noise. -This initial canary supports ephemeral GitHub-hosted runners only. Use a matching -larger hosted runner via RACE_CANARY_RUNNER_LABEL if the standard runner fails -the importer space gate. It does not claim measurements for self-hosted pools. - -Measured wall time starts before seed acquisition and ends after make clean, -config, full make ut and importer-owned cleanup. It includes seeder failures. -Checkout/toolchain setup, warm snapshot production/transfer/restoration, report -parsing and artifact upload are separately visible workflow preparation costs, -not part of the paired UT interval. Docker image layers remain daemon-owned, -as in the production importer; they are charged to disk occupancy and reclaimed -by ephemeral runner destruction, not by an invented image-prune step. -Host CPU counters, disk counters, memory availability, cgroup counters and disk -free space are sampled throughout the measured interval. Raw samples and UT -logs are retained. Imported entries are not reported as compiler cache hits. - -A successful workflow is a **valid experiment**, not rollout approval. Both -arms must complete the same nonempty test execution multiset with identical -pass/skip/package outcomes. B must report seeded, producer race ok, imported -files > 0 and complete cleanup; any skip/fallback/partial seed is invalid. Warm -cache identity is checked before running. Cold/warm are reported independently; -one smoke pair is insufficient to establish a stable gain. No coverage result -is inferred. Keep rollout off until repeated net gains and acceptable memory, -disk and CPU costs have been reviewed. - -## Initial verified image - -2026-09-20: builder job 105925505882 in MatrixOne run 35443952690 published -`matrixorigin/matrixone@sha256:8137d222d3a3b29d119173639cea98931168ab7d886bf882894391f7804d5d88` -from source `3ac87c30625fc082391e5384a4c94e50a2916cf6`. -The registry metadata layer was downloaded independently and its SHA256 -verified. Manifest schema 2 / host-ut-v1, canonical checkout/modules, Go 1.26.4, -linux/amd64/v1, both contract IDs, race=ok and coverage=ok match the consumer. -This proves publication/compatibility, not cache-hit rate or end-to-end speedup. - -## Invocation - -The sample requires the five existing S3 UT credentials (S3ENDPOINT, S3REGION, -S3APIKEY, S3APISECRET, S3BUCKET). The existing CI-repository dispatch does not -establish that credential path; replace it with a reviewed MatrixOne caller -before execution. Missing credentials reject the run; values are never printed. -The currently configured MatrixOne UT pool is `amd64-mo-shanghai-8c16g`; -this hosted-only first measurement does not establish performance on that pool. -Rollout to that pool additionally requires a matching ephemeral runner experiment. - -Only after the image-acquisition and caller blockers above are resolved and the updated -workflow is reviewed and merged, dispatch Race seed canary on main with a full -40-character MatrixOne SHA and the immutable image reference above. Start with -repetitions=1 (four measured arms plus one warm preparation run); repetitions=3 -gives twelve measured arms and the same one preparation run. Review the report -artifact and job summaries before requesting a separate rollout change. - -## Design scope - -R2 measurement contract: isolate compiler caches, preserve full race execution, -keep fixed inputs, and fail closed on incomplete evidence. No suite selection, -timeouts, scheduling or production activation changes. The importer gets one -optional digest input restricted to its two existing trusted repositories; -ordinary callers retain their current behavior. Tests cover input rejection, -empty/truncated execution evidence, mismatched outcomes and mismatched pairing. +CI #456 supplies the reusable measurement workflow; a thin MatrixOne manual +workflow supplies caller context and existing CI-environment UT credentials. +No existing PR check, production default, coverage workflow or runner deployment +is changed. Implementation design: [daemonless design](race-seed-daemonless-design.md). + +## Why daemonless acquisition + +Read-only inspection on 2026-09-20 established the Shanghai 8c16g pool uses +ephemeral ARC pods with an emptyDir workspace, 8 CPU and 16 GiB. The observed +container has no Docker socket/sidecar/remote Docker configuration; docker info +fails. Registry transport therefore reads dedicated COPY layers using checksum-pinned +crane v0.22.1 without executing the image. An explicit empty credential configuration +prevents runner registry-auth reuse. Existing Docker transport stays unchanged. + +The importer validates producer schema 2 / host-ut-v1 and Go/platform/path/flavor +contracts, imports regular cache files additively and cleans all owned blob, +tool and payload staging before publishing completion. Registry mode requires +a trusted immutable image and the known producer layer layout. Compressed blob +digests and complete uncompressed diff_ids are checked before publication. Only +metadata and build-cache layers are always acquired; module cache is fetched only +when empty. Base/native-image layers and whole-filesystem export are unnecessary. +Even these remaining transfer costs can outweigh compilation savings. + +## Paired contract + +- One source SHA reachable from official MatrixOne main and one immutable, + reviewed CI harness SHA. The trusted main-branch caller pins both the reusable + workflow and harness input to the same CI commit. CI squash merges do not + preserve head ancestry, so admission checks checkout identity, not CI ancestry. +- Both arms use `amd64-mo-shanghai-8c16g`, canonical checkout and module paths, + the existing Go/Java/CMake and Shanghai proxy policy, no actions Go cache restore, + full unsharded race UT, parallelism 6 unless the existing repository variable + overrides it, and the same UT timeout. +- A disables seeding; B uses registry seeding. No daemon or shared image cache. + The first measured operation includes B's crane bootstrap, image transfer, + validation, direct streaming extraction, import and cleanup, followed by + clean/config/full UT. Per-import phase times include their cache blob downloads. +- Cold arms refuse preexisting populated Go caches. Warm arms restore the exact + same artifact from one successful seed-off full UT. The snapshot contains no + seed marker. Both arms invalidate test-result cache, preserving compile cache. +- CPU quota, cgroup CPU seconds/utilization/throttling, sampled memory and OOM + deltas, host counters, disk occupancy and UT logs are retained. Host counters + are context only: visible cpuset can be 96 CPUs while quota is 8 cores. +- Exact nonempty test and package outcome multisets must match. Empty, failed, + truncated, skipped-import, partial-import or mismatched arms are invalid. + +The matrix requests AB/BA alternation with max-parallel 1; actual scheduler order +must be checked from timestamps. Page-cache state, colocated workloads and network +conditions remain noise. First run: one warm preparation plus four measured arms. +Three repetitions: one preparation plus twelve arms. No automatic rollout decision. + +Checkout/toolchain setup, snapshot production/transfer/restore, parsing and +artifact upload are outside paired UT time and visible separately in job duration. +Importer-owned cleanup is inside the measurement. No image-prune cost is deferred +to pod destruction in registry mode. Hard-kill leftovers belong only to the +ephemeral pod. Sampled peaks can miss short spikes; OOM counters supplement them. + +## Publication and operation + +Verified builder run 35443952690 / job 105925505882 produced source +`3ac87c30625fc082391e5384a4c94e50a2916cf6`, manifest +`sha256:8137d222d3a3b29d119173639cea98931168ab7d886bf882894391f7804d5d88`, +published to Docker Hub and the Shanghai ACR mirror. Registry metadata was checked +independently: Go 1.26.4, linux/amd64/v1, race=ok and coverage=ok. Publication and +import success are not compiler-hit evidence or end-to-end speedup evidence. + +Merge order (human authorization required): CI #456 first, then the MatrixOne +manual caller pinned to the reviewed CI commit. Dispatch only on MatrixOne main +with repetitions=1. Review complete cold/warm outcomes and resource deltas before +increasing repetitions. Do not copy S3 credentials to the CI repository. + +Keep ordinary seed off unless repeated measurements show stable net wall-time +benefit without correctness, memory, CPU or disk regression. No benefit or a +slowdown means no rollout. Coverage requires its own later experiment. diff --git a/docs/race-seed-daemonless-design.md b/docs/race-seed-daemonless-design.md new file mode 100644 index 0000000..0311a3e --- /dev/null +++ b/docs/race-seed-daemonless-design.md @@ -0,0 +1,145 @@ +# Daemonless race canary — design revision 2 + +## Revision 2 decision (supersedes whole-filesystem export below) + +The Shanghai probe disproved the practicality of full export: 900s timeout, +18.4GB partial tar, cache payloads already present, still exporting unrelated +base-system files. No marker published, cleanup complete. Do not raise budgets. + +Select only the producer's dedicated COPY layers, using the same pinned crane +binary for manifest/config/blob and the same anonymous credential isolation. +Admit only a single linux/amd64 image manifest with gzip layers, history count +matching layers and diff_ids, and the exact final six nonempty history entries +from Dockerfile.ci-builder: module-cache COPY, build-cache COPY, native-install +COPY, fingerprint COPY, warm-status COPY, manifest COPY. Any layout drift rejects +the experiment, never guesses or attempts general OCI merging. This is a narrow +producer transport contract; schema-2 files still provide semantic compatibility. + +Verify pinned manifest bytes by SHA256 and config blob bytes by its descriptor. +History is an admission guard, not authority to execute commands. Select final +COPY payloads only, ignoring inherited base caches (never depending on them). +No later layer may modify these paths under the admitted suffix. Reject all +whiteouts and unexpected entries; allow only parent directories and the selected +subtree's existing regular-file/directory contract. No symlinks or image execution. + +Download the small producer-manifest layer first and validate its semantic +contract before large transfers. Download build layer, verify compressed digest +and declared byte size, then stream gzip directly into the existing strict +extractor's owned staging. Hash the entire uncompressed stream through EOF and +verify config rootfs.diff_id before publication. Repeat for modules only if empty. +This removes base/native downloads, decompressed export and filtered tar copies. + +Bounds remain 1080s work +45s cleanup. Metadata <=64KiB compressed/decompressed; +each cache blob <=8GiB compressed; aggregate cache-layer uncompressed bytes <=24GiB. +Admission budgets current-cache stage + one compressed blob + module destination +on their actual filesystems, with reserve. Streaming hash and bounded reads avoid +full-layer memory capture; tar headers are discarded as processed. Seeder owns +all blob/stage resources; checksum/layout/timeout/cancel/partial failure never +publishes that payload or a success marker. Existing additive/empty-only commit +semantics and Docker transport stay unchanged. + +Focused proof adds layout/digest/size/whiteout/path/partial-gzip/diff-id rejection, +metadata-before-large-transfer, module-transfer omission for populated caches, +and direct strict extraction/publisher cleanup. Live final backend must acquire +and import the real pinned image in the no-Docker Shanghai probe. Record actual +timing and resource outcomes; no full-UT speedup claim from this capability test. + +The earlier revision below records the rejected approach and unchanged caller, +ownership, rollout and evidence decisions. Revision 2 is reviewed before code. +GPT-6 medium design review: PASS, 2026-09-20. Explicit assumption: isolation of +omitted layers relies on the trusted producer Dockerfile, not history strings +alone. Missing subtree differs from an explicit empty module directory; full +uncompressed-stream digest verification precedes publication. + +Owner: CI #456, continuing the UT cache work in CI #455 / MatrixOne #29109. +Scope: image acquisition and a manual paired experiment, not production rollout. +Trigger: acquisition crosses a registry trust/resource boundary and the canary +crosses repositories. Design review precedes implementation of this revision. + +## Evidence and invariant + +The Shanghai 8c16g ARC template has one unprivileged runner container, ephemeral +runner ownership and an emptyDir workspace. A live read-only probe confirmed no +Docker socket or remote Docker configuration; docker info fails. Image publication +does not establish consumer usability or any speedup. + +Invariant: opt-in registry acquisition imports only regular cache entries from +the two already trusted repositories at an immutable digest, validates the same +producer contract, preserves existing entries, and publishes completion only +after owned cleanup. It never executes the image or needs cluster privileges. +Production remains default-off and existing Docker transport remains the default. + +## Alternatives and decision + +1. Status quo Docker transport cannot work on the observed runner. +2. Add a privileged DinD sidecar: changes infrastructure, resources and isolation; + rejected for this measurement task. +3. Implement OCI authentication/layer/whiteout logic ourselves: larger security + and maintenance surface; rejected. +4. Use upstream crane's merged filesystem export: selected. It implements OCI + layer/whiteout semantics without a daemon. Cost: downloads the full image and + retains one bounded uncompressed tar; measured acquisition includes tool setup. + +Reference: google/go-containerregistry v0.22.1, crane export and OCI Image Spec. +Linux amd64 release archive SHA256 is pinned to +0ab7a1d6932a213aed964ce97666c3077fe691c8606413674a8b3e0b9ec4cda0. +No mutable executable download, image execution or shell evaluation of inputs. +Design review: GPT-6 medium PASS (2026-09-20), with the refinement below: +write an explicit `{}` to task-owned DOCKER_CONFIG/config.json before every +crane invocation so the default keychain cannot fall back to Podman credentials. + +## Ownership, transitions and bounds + +Registry mode is explicit and requires the existing trusted digest input. One +Seeder owns all staging directories; existing inode-checked subprocess cleanup +and 1080s work +45s cleanup deadline apply on success, error and SIGTERM. +Bootstrap archive <=32 MiB, metadata <=64 KiB, export <=24 GiB; same-filesystem +free-space admission reserves export plus extracted data plus 4 GiB. Crane uses +an empty Docker config and task-owned XDG runtime/config dirs (no registry auth +reuse); subprocess tree is bounded/reaped by the existing command runner. +Export tar is read without extraction. Only manifest and the two known cache +subtrees are selected; links, traversal and special files within selected paths +are rejected by the existing strict extractor. All other rootfs paths are ignored, +not written. Export reaches EOF before successful import; incomplete downloads +never publish a marker. Original Docker path is unchanged for existing callers. +Cache publication remains additive; module cache replaced only when empty. +Peak disk admission includes export and extracted payloads, with no filtered-tar +copy: reuse the strict extractor with an explicit source prefix over the seekable +export. Release staging between build/module import, release export at cleanup. +No shared Docker/image/cache pruning. Kernel hard kill relies on ephemeral pod +teardown; no persistent deployment is created. + +## Experiment integration + +Run in MatrixOne caller context to inherit existing CI-environment S3 secrets. +A thin manual workflow calls this reusable workflow at an immutable CI commit. +The trusted main caller pins workflow and checkout to the same reviewed SHA; +checkout identity is enforced, but CI ancestry is not required because this +repository squash-merges PRs. This avoids invalidating a reviewed immutable pin. +Harness checkout explicitly pins its own workflow revision; source SHA is an +ancestor of official MatrixOne main. Samples use the observed Shanghai pool, +canonical source/module paths and job-local compiler cache. Never clear an +unknown populated path. Warm preparation exports a common artifact; each warm +arm restores identical bytes, and both invalidate test-result cache. Record +source/harness/image identity, cgroup quotas, toolchain, cache hash, CPU/memory, +disk and full test outcome multiset. A/B sequential, one warm preparation plus +four measured arms for the first run; no coverage or rollout inference. +Avoid contaminating initial caches with tool compilation: bootstrap a verified +binary inside measured seed acquisition only. Toolchain/source preparation is +identical in A/B; no GitHub Go cache restore. Job durations expose setup overhead. + +## Verification and rollout + +Deterministic tests: trust/pin validation, download/export limits, traversal/links, +manifest mismatch, partial export, unchanged existing cache, cleanup/cancellation, +real nonempty paired test evidence and workflow source/caller identity. Reuse the +existing importer tests; run Linux tests and actionlint. Independently validate +the pinned release checksum and live registry metadata, then exercise actual +image import in an owned Linux environment without a Docker socket. Final review +uses GPT-6 medium. Full A/B only runs through authorized manual dispatch after +entry workflows are available; do not merge PRs without authorization or wait for +unrelated CI. Missing full A/B means no measured speedup claim or seed rollout. + +Rollback: keep seed off; remove opt-in workflow/backend without cache migration. +Known cost risk: full filesystem export may erase compile savings; that is a valid +negative result. No promised percentage improvement. Any incomplete arm is invalid. diff --git a/docs/race-seed-validation.md b/docs/race-seed-validation.md new file mode 100644 index 0000000..b5216e2 --- /dev/null +++ b/docs/race-seed-validation.md @@ -0,0 +1,46 @@ +# Race seed validation record + +Scope: CI #456 against main `bff80f0fed2739725cd7cd256d6920875e871fc8`, +daemonless adaptation against prior head `6da76ff`, plus the MatrixOne manual +caller. Production seed remains off. No kernel/SQL behavior changes, so kernel +UT/BVT is not a substitute for these importer and workflow checks. + +Design-first review: GPT-6 medium approved daemonless design revision 1 before +implementation, requiring explicit empty Docker config rather than merely an +empty directory. Final caller pins will use the exact reviewed CI commit. + +| Closure | Risk / invariant | Evidence | +|---|---|---| +| Registry acquisition / strict import | R3 trust boundary; trusted digest, pinned binary, no auth reuse/image execution | checksum/auth/pin/path/PAX/platform tests; real acquisition probe | +| Temporary files / subprocesses | R3 ownership, cancellation and capacity | existing repeated-signal/bounded-reaper tests plus partial registry acquisition cleanup | +| Runner preparation / snapshot restore | R2 preserve cache, correct roots across filesystems | cold/populated guard, forced EXDEV warm restore, exact bytes/layout | +| Caller / immutable source / comparison | R2 credentials and measurement identity | official-main source rejection, exact harness identity, outcome multiset tests, workflow lint | +| Production defaults | R1 compatibility | unchanged Docker default and opt-in seed; prior importer regression suite | + +| Audit | Owner / termination / bound | +|---|---| +| Q1 | Seeder registers staging inode at creation, deletes only owned paths, commits marker after cleanup. Pod teardown owns hard-kill leftovers. | +| Q2 | Existing command process-group reaper; 1080s total work, 45s shared cleanup; no unbounded network subprocess. | +| Q3 | 32 MiB tool archive, 64 KiB metadata, 8 GiB compressed cache blob, 24 GiB aggregate uncompressed cache layers, free-space admission; no full export or global tar header index. | + +Linux Python 3.12: 43 importer/cleanup/registry tests and 11 canary tests pass. +Regression includes safe populated-cache rejection and cross-filesystem warm +restore using real shutil.move fallback. actionlint v1.7.12 checks both reusable +workflows, the validation workflow and the MatrixOne caller. Diff whitespace clean. + +Live capability validation uses a task-owned, deadline-bounded Shanghai pod with +the observed runner image digest and 8 CPU / 16 GiB, no Docker socket and no +service-account token. It does not register as a GitHub runner or run full UT. +The Go 1.26.4 toolchain archive is independently checksum-verified. Producer +source is `3ac87c30625fc082391e5384a4c94e50a2916cf6`; image manifest is +`sha256:8137d222d3a3b29d119173639cea98931168ab7d886bf882894391f7804d5d88`. +The rejected full-export backend timed out after 900s (902.205s including setup +and cleanup), with 18.4GB partial output, zero imported bytes and complete owned +cleanup. The diagnostic-only retained hardlink was explicitly removed. No OOM +was recorded, but cgroup memory.max reclaim events occurred. This negative result +led to reviewed design revision 2: direct cache-layer acquisition/extraction. +Final-backend live outcome is recorded before delivery; no passing claim is made +here for an in-progress probe. + +Missing by design before merge/dispatch: complete cold/warm A/B race UT and +net-speedup evidence. Successful import or local tests cannot justify rollout. diff --git a/docs/ut-cache-seeding.md b/docs/ut-cache-seeding.md index d2bd084..8a0c7fb 100644 --- a/docs/ut-cache-seeding.md +++ b/docs/ut-cache-seeding.md @@ -1,5 +1,11 @@ # Correct, measurable UT cache seeding +For the opt-in daemonless Shanghai race experiment, see +[race-seed-canary.md](race-seed-canary.md). `transport: registry` requires a +trusted digest-pinned `image`; it uses checksum-pinned crane COPY-layer reads rather than +a Docker daemon. Existing action callers still default to `transport: docker`. +No production workflow enables the new transport or seeding automatically. + ## Problem and scope The race and coverage UT workflows both consider a nonempty GOCACHE warm. diff --git a/scripts/race_seed_canary.py b/scripts/race_seed_canary.py index 644a8f5..b0ca0e3 100644 --- a/scripts/race_seed_canary.py +++ b/scripts/race_seed_canary.py @@ -82,6 +82,7 @@ def fingerprint(): 'source_sha': command(['git', 'rev-parse', 'HEAD'], SOURCE), 'ci_sha': command(['git', 'rev-parse', 'HEAD'], ROOT), 'image': os.environ['SEED_IMAGE'], + 'transport': os.environ.get('SEED_TRANSPORT', ''), 'runner_image': [os.environ.get(k, '') for k in ('ImageOS', 'ImageVersion')], 'runner_label': os.environ['CANARY_RUNNER_LABEL'], 'cpu_model': sorted(set(re.findall(r'^model name\s*:\s*(.*)', cpu, re.M))), @@ -213,13 +214,19 @@ def cpu(row): def prepare(snapshot): - if os.environ.get('RUNNER_ENVIRONMENT') != 'github-hosted': - raise ValueError('canary requires a fresh GitHub-hosted runner') - for path in (SOURCE, CACHE, MODULES): - if path.exists() or path.is_symlink(): + if (os.environ.get('RUNNER_ENVIRONMENT') != 'self-hosted' + or not re.fullmatch(r'amd64-mo-shanghai-8c16g-.+-runner-.+', os.environ.get('RUNNER_NAME', '')) + or Path(os.environ.get('GITHUB_WORKSPACE', '/')) != SOURCE): + raise ValueError('canary requires the audited Shanghai ARC pool and canonical checkout') + if (command(['git', 'rev-parse', 'HEAD'], SOURCE) != os.environ['CANARY_SOURCE_SHA'] + or command(['git', 'rev-parse', 'HEAD'], ROOT) != os.environ['CANARY_CI_SHA']): + raise ValueError('checkout identity mismatch') + for path in (CACHE, MODULES): + if path.is_symlink() or (path.exists() and (not path.is_dir() or any(path.iterdir()))): raise ValueError(f'preexisting path; refusing to alter it: {path}') - SOURCE.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(ROOT / 'subject'), SOURCE) + for parent in path.parents: + if parent.is_symlink(): + raise ValueError('symlink cache parent') CACHE.parent.mkdir(parents=True, exist_ok=True) MODULES.parent.mkdir(parents=True, exist_ok=True) initial = {'cache_state': 'cold', 'snapshot_sha256': None} @@ -232,15 +239,19 @@ def prepare(snapshot): staging.mkdir() with tarfile.open(archive) as stream: stream.extractall(staging, filter='data') - shutil.move(str(staging / 'go-build'), CACHE) - shutil.move(str(staging / 'mod'), MODULES) + for name, destination in (('go-build', CACHE), ('mod', MODULES)): + if destination.exists(): + destination.rmdir() # only empty; never remove populated cache contents + # ARC emptyDir workspace and container-root cache may be different + # filesystems. move copies on EXDEV and avoids nested cache roots. + shutil.move(str(staging / name), str(destination)) if meta['identity'] != fingerprint(): raise ValueError('snapshot environment mismatch') archive.unlink() # only the task-owned downloaded archive initial = {'cache_state': 'warm', 'snapshot_sha256': meta['sha256']} else: - CACHE.mkdir() - MODULES.mkdir() + CACHE.mkdir(exist_ok=True) + MODULES.mkdir(exist_ok=True) if (CACHE / '.matrixone-seed.json').exists(): raise ValueError('initial compiler cache must not contain a seed marker') # Identical test-result invalidation in A/B; compiler cache is preserved. @@ -268,13 +279,11 @@ def monitor(): raise ValueError('CI S3 test credentials must be configured; no secret values are logged') report['initial'] = prepare(args.snapshot) report['identity'] = fingerprint() - before_images = command(['docker', 'image', 'ls', '-q', '--no-trunc']) - report['initial_images'] = sorted(before_images.splitlines()) - # The pinned builder cannot already be local, even for warm Go caches. - if subprocess.run(['docker', 'image', 'inspect', os.environ['SEED_IMAGE']], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=30).returncode == 0: - raise ValueError('builder image already present') + # Registry export has no shared local image store. Both arms start + # without task-owned tool/export staging; B pays all acquisition costs. + report['initial_images'] = [] + if os.environ.get('SEED_TRANSPORT') != 'registry': + raise ValueError('Shanghai canary requires registry transport') report['disk_before'] = command(['df', '-Pk', str(SOURCE), str(CACHE), str(MODULES)]) measured_start = time.monotonic() monitor_thread = threading.Thread(target=monitor) @@ -302,8 +311,9 @@ def monitor(): with (out / 'resources.jsonl').open('a') as log: log.write(json.dumps(sample()) + '\n') report['resources'] = resource_summary(out / 'resources.jsonl') + if not report['resources']['cgroup']['available']: + raise ValueError('missing runner cgroup resource evidence') report['disk_after'] = command(['df', '-Pk', str(SOURCE), str(CACHE), str(MODULES)]) - report['docker_disk'] = command(['docker', 'system', 'df']) reports = list((SOURCE / 'scratch').rglob('*-UT-Report.out')) if len(reports) != 1: raise ValueError('expected exactly one complete raw UT report') diff --git a/scripts/race_seed_plan.py b/scripts/race_seed_plan.py index 5b48538..8e81a72 100644 --- a/scripts/race_seed_plan.py +++ b/scripts/race_seed_plan.py @@ -36,5 +36,13 @@ def matrix(repetitions): text=True, timeout=60).strip() if status not in ('ahead', 'identical'): raise ValueError('source SHA is not reachable from official main') + ci_sha = os.environ['CI_SHA'] + if not re.fullmatch('[0-9a-f]{40}', ci_sha): + raise ValueError('full CI SHA required') + # The trusted main-branch caller pins BOTH workflow and checkout to this + # reviewed commit. CI squash merges do not preserve reviewed-head ancestry. + actual = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True, timeout=10).strip() + if actual != ci_sha: + raise ValueError('harness checkout identity mismatch') with open(os.environ['GITHUB_OUTPUT'], 'a') as output: output.write('matrix=' + json.dumps(matrix(int(os.environ['REPETITIONS']))) + '\n') diff --git a/scripts/test_race_seed_canary.py b/scripts/test_race_seed_canary.py index 52a6b89..c66c801 100644 --- a/scripts/test_race_seed_canary.py +++ b/scripts/test_race_seed_canary.py @@ -1,16 +1,81 @@ import copy +import errno +import io import json import os +import runpy from pathlib import Path import tempfile +import tarfile import unittest from unittest import mock +import race_seed_canary as canary from race_seed_canary import compare, execution, summarize, resource_summary, cgroup_summary from race_seed_plan import matrix class EvidenceTests(unittest.TestCase): + def test_plan_checks_trusted_source_and_exact_harness_without_squash_ancestry(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'output' + env = dict(SOURCE_SHA='1' * 40, CI_SHA='2' * 40, + SEED_IMAGE='matrixorigin/matrixone@sha256:' + '3' * 64, + REPETITIONS='1', GITHUB_OUTPUT=str(output)) + script = str(Path(__file__).with_name('race_seed_plan.py')) + with mock.patch.dict(os.environ, env), \ + mock.patch('subprocess.check_output', side_effect=['ahead', '2' * 40]) as command: + runpy.run_path(script, run_name='__main__') + self.assertEqual(len(json.loads(output.read_text().split('=', 1)[1])['include']), 4) + self.assertEqual(command.call_count, 2) + with mock.patch.dict(os.environ, env), \ + mock.patch('subprocess.check_output', side_effect=['ahead', '4' * 40]): + with self.assertRaisesRegex(ValueError, 'checkout identity'): + runpy.run_path(script, run_name='__main__') + with mock.patch.dict(os.environ, env), \ + mock.patch('subprocess.check_output', return_value='diverged'): + with self.assertRaisesRegex(ValueError, 'official main'): + runpy.run_path(script, run_name='__main__') + + def test_audited_runner_preparation_preserves_preexisting_cache(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + source, cache, modules = (root / n for n in ('source', 'cache', 'modules')) + source.mkdir() + environment = dict(RUNNER_ENVIRONMENT='self-hosted', + RUNNER_NAME='amd64-mo-shanghai-8c16g-abc-runner-def', + GITHUB_WORKSPACE=str(source), CANARY_SOURCE_SHA='s', CANARY_CI_SHA='c') + def command(args, cwd=None): + return ('s' if cwd == source else 'c') if args[0] == 'git' else '' + with mock.patch.multiple(canary, SOURCE=source, CACHE=cache, MODULES=modules), \ + mock.patch.object(canary, 'command', side_effect=command), \ + mock.patch.dict(os.environ, environment): + self.assertEqual(canary.prepare(None)['cache_state'], 'cold') + (cache / 'existing').write_text('retain') + with self.assertRaisesRegex(ValueError, 'preexisting'): + canary.prepare(None) + self.assertEqual((cache / 'existing').read_text(), 'retain') + (cache / 'existing').unlink() + snapshot = root / 'snapshot' + snapshot.mkdir() + archive = snapshot / 'cache.tar' + with tarfile.open(archive, 'w') as stream: + for name in ('go-build/cache-hit', 'mod/example.test/m.go'): + member = tarfile.TarInfo(name) + member.size = 4 + stream.addfile(member, io.BytesIO(b'data')) + (snapshot / 'snapshot.json').write_text(json.dumps({ + 'sha256': canary.sha256(archive), 'identity': {'fixture': 'same'}})) + with mock.patch.object(canary, 'fingerprint', return_value={'fixture': 'same'}), \ + mock.patch('shutil.os.rename', side_effect=OSError(errno.EXDEV, 'cross-device')): + self.assertEqual(canary.prepare(snapshot)['cache_state'], 'warm') + self.assertEqual((cache / 'cache-hit').read_bytes(), b'data') + self.assertEqual((modules / 'example.test/m.go').read_bytes(), b'data') + self.assertFalse((cache / 'go-build').exists()) + with mock.patch.dict(os.environ, RUNNER_NAME='unknown-runner'): + with self.assertRaisesRegex(ValueError, 'audited'): + canary.prepare(None) + def test_cgroup_metrics_use_quota_not_host_cpu_count(self): rows = [dict(monotonic=i * 10, cgroup_cpu=f'usage_usec {i * 40000000}\nthrottled_usec {i * 1000000}', cgroup_cpu_limit='800000 100000', cgroup_memory_limit='17179869184', From 23eb3cc728e27db9d3e680bf78f69a44911a0bf5 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 20 Sep 2026 10:50:35 +0800 Subject: [PATCH 5/6] docs(ci): record daemonless race seed proof --- docs/race-seed-validation.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/race-seed-validation.md b/docs/race-seed-validation.md index b5216e2..3603221 100644 --- a/docs/race-seed-validation.md +++ b/docs/race-seed-validation.md @@ -5,9 +5,11 @@ daemonless adaptation against prior head `6da76ff`, plus the MatrixOne manual caller. Production seed remains off. No kernel/SQL behavior changes, so kernel UT/BVT is not a substitute for these importer and workflow checks. -Design-first review: GPT-6 medium approved daemonless design revision 1 before -implementation, requiring explicit empty Docker config rather than merely an -empty directory. Final caller pins will use the exact reviewed CI commit. +Design-first review: GPT-6 medium approved both daemonless design revisions +before implementation. The review required an explicit empty Docker config, +strict trusted COPY-layer admission, full compressed and uncompressed digest +verification, and a regression for malformed DEFLATE input. The MatrixOne +caller pins the exact reviewed CI commit. | Closure | Risk / invariant | Evidence | |---|---|---| @@ -39,8 +41,17 @@ and cleanup), with 18.4GB partial output, zero imported bytes and complete owned cleanup. The diagnostic-only retained hardlink was explicitly removed. No OOM was recorded, but cgroup memory.max reclaim events occurred. This negative result led to reviewed design revision 2: direct cache-layer acquisition/extraction. -Final-backend live outcome is recorded before delivery; no passing claim is made -here for an in-progress probe. +The final direct-layer backend completed on the same runner class with +`state=seeded`, `cleanup=complete`, producer status `ok`, and module state +`seeded`. It imported 64,500 files / 15,819,961,352 bytes: acquisition took +2.912s, build-cache download/verification/import took 825.284s, module import +took 201.134s, cleanup took 1.047s, and end-to-end seeding took 1,029.392s. +A forced `go test -race -count=1` smoke then executed its test body and passed +in 1.012s. There was no OOM or OOM kill, but memory reached approximately +16.4GB and generated 64 cgroup `memory.max` events. The task-owned pod was +deleted after collection. These results prove capability, not net performance: +the 1,080s work budget has only about 51s of margin and the 16GiB runner has +little memory headroom. Missing by design before merge/dispatch: complete cold/warm A/B race UT and net-speedup evidence. Successful import or local tests cannot justify rollout. From 20a53a824c4526942ca5e570fb4f0c865f8c6492 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 20 Sep 2026 11:13:08 +0800 Subject: [PATCH 6/6] fix(ci): require module cache contract in race seed comparisons --- scripts/race_seed_canary.py | 17 +++++++++++++---- scripts/test_race_seed_canary.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/scripts/race_seed_canary.py b/scripts/race_seed_canary.py index b0ca0e3..598d583 100644 --- a/scripts/race_seed_canary.py +++ b/scripts/race_seed_canary.py @@ -319,10 +319,7 @@ def monitor(): raise ValueError('expected exactly one complete raw UT report') report['execution'] = execution(reports[0]) if args.seed: - seeded = report['seed'] - if (seeded.get('state') != 'seeded' or seeded.get('producer_flavor_status') != 'ok' - or seeded.get('cleanup') != 'complete' or seeded.get('imported_files', 0) <= 0): - raise ValueError('seed skipped, partial, empty or failed: invalid B sample') + validate_seed(report) report['valid'] = True if args.export: args.export.mkdir() @@ -342,6 +339,17 @@ def monitor(): return 0 if report['valid'] else 1 +def validate_seed(report): + seeded = report.get('seed', {}) + if (seeded.get('state') != 'seeded' or seeded.get('producer_flavor_status') != 'ok' + or seeded.get('cleanup') != 'complete' or seeded.get('imported_files', 0) <= 0): + raise ValueError('seed skipped, partial, empty or failed: invalid B sample') + cache_state = report.get('initial', {}).get('cache_state') + expected = {'cold': 'seeded', 'warm': 'preserved-populated'}.get(cache_state) + if expected is None or seeded.get('module_state') != expected: + raise ValueError(f'invalid {cache_state} B module_state: expected {expected}') + + def compare(a, b): if not a.get('valid') or not b.get('valid'): raise ValueError('invalid/incomplete arm') @@ -350,6 +358,7 @@ def compare(a, b): for key in ('identity', 'initial', 'initial_images', 'execution'): if a[key] != b[key]: raise ValueError('pair mismatch: ' + key) + validate_seed(b) return {'A_seconds': a['total_seconds'], 'B_seconds': b['total_seconds'], 'saved_seconds': a['total_seconds'] - b['total_seconds'], 'saved_percent': 100 * (1 - b['total_seconds'] / a['total_seconds'])} diff --git a/scripts/test_race_seed_canary.py b/scripts/test_race_seed_canary.py index c66c801..8e07e60 100644 --- a/scripts/test_race_seed_canary.py +++ b/scripts/test_race_seed_canary.py @@ -16,6 +16,30 @@ class EvidenceTests(unittest.TestCase): + def seed_report(self, state): + return dict(state='seeded', producer_flavor_status='ok', cleanup='complete', + imported_files=1, module_state=('seeded' if state == 'cold' else 'preserved-populated')) + + def test_module_state_required_even_when_build_seed_and_ut_succeed(self): + for state in ('cold', 'warm'): + a = dict(valid=True, seed_enabled=False, identity={}, initial={'cache_state': state}, + initial_images=[], execution=self.read(self.events()), total_seconds=10) + b = dict(copy.deepcopy(a), seed_enabled=True, seed=self.seed_report(state)) + self.assertEqual(compare(a, b)['saved_seconds'], 0) + for module_state in (None, 'insufficient-space', 'not-attempted', + 'preserved-mountpoint', 'previous-import', + 'preserved-populated' if state == 'cold' else 'seeded'): + changed = copy.deepcopy(b) + if module_state is None: + changed['seed'].pop('module_state') + else: + changed['seed']['module_state'] = module_state + with self.subTest(state=state, module_state=module_state): + with self.assertRaisesRegex(ValueError, 'module_state'): + canary.validate_seed(changed) + with self.assertRaisesRegex(ValueError, 'module_state'): + compare(a, changed) + def test_plan_checks_trusted_source_and_exact_harness_without_squash_ancestry(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / 'output' @@ -140,9 +164,9 @@ def test_missing_second_package_terminal_is_rejected(self): def test_pair_rejects_each_mismatched_contract(self): a = dict(valid=True, seed_enabled=False, identity={'sha': 'same'}, - initial={'snapshot_sha256': 'same'}, initial_images=[], + initial={'snapshot_sha256': 'same', 'cache_state': 'warm'}, initial_images=[], execution=self.read(self.events()), total_seconds=10) - b = dict(copy.deepcopy(a), seed_enabled=True, total_seconds=8) + b = dict(copy.deepcopy(a), seed_enabled=True, total_seconds=8, seed=self.seed_report('warm')) self.assertEqual(compare(a, b)['saved_seconds'], 2) for key in ('identity', 'initial', 'initial_images', 'execution', 'valid', 'seed_enabled'): changed = copy.deepcopy(b) @@ -169,6 +193,8 @@ def test_downloaded_artifact_layout_and_missing_pair(self): result = dict(valid=True, seed_enabled=arm == 'B', identity={}, initial={'cache_state': state}, initial_images=[], execution=self.read(self.events()), total_seconds=10) + if arm == 'B': + result['seed'] = self.seed_report(state) (target / 'result.json').write_text(json.dumps(result)) with mock.patch.dict(os.environ, CANARY_REPETITIONS='1', GITHUB_STEP_SUMMARY=str(root / 'summary.md')):