diff --git a/docs/design/sirius-public-campaign.md b/docs/design/sirius-public-campaign.md index 2dcd3b283f60e..d061f56ec1982 100644 --- a/docs/design/sirius-public-campaign.md +++ b/docs/design/sirius-public-campaign.md @@ -1,6 +1,6 @@ # Public Sirius acceptance campaign -Design version: 3. Owners: MatrixOne public SQL, compile/CN evidence and +Design version: 4. Owners: MatrixOne public SQL, compile/CN evidence and `optools` campaign tooling. Tracks MO #28968 and parent #28966. The user approved this design before implementation on 2026-10-09 and selected @@ -11,6 +11,155 @@ native source for the guarded release SDK. The user approved this implementation revision on 2026-10-11 after selecting one complete D PR and separate allocation profiling without a native observability prerequisite. E/F and Docker changes are excluded. +## Approved execution-profile optimization (2026-10-13) + +The owner approved the focused performance plan before implementation. This +increment makes the owned campaign's Go CPU budget explicit and records pinned +allocation granularity; production backend and service defaults do not change. +Both `serve` and `freeze` require `--gomaxprocs` in Go's runtime range +1 through 2147483647 and accept +`--host-block-bytes` of 65536 or 1048576, defaulting to 65536. New provenance +records `config.gomaxprocs` and `config.native_host_block_bytes`. `run` validates +the frozen profile before starting services and launches native and embedded +routes with the same frozen budget, independently of its ambient environment. +Historical captures retain their actual settings and controller identity; +new runs require refrozen provenance. Read-only verification preserves the +historical v2 configuration shape when both profile fields are absent; it neither +adds defaults nor modifies archived provenance. If either field is present, both +must be valid. New campaign construction and executor admission require both +fields, and evidence with different profile shapes cannot be combined. +Runner/result schema version 2 is unchanged. + +The current-host qualification profile is 16 Go CPUs and 64 KiB host blocks. +Compare code revisions at identical settings; profile/kernel observations are +separate from uninstrumented timing. Canonical SF1/SF10 correctness, streams +1/2/4, bounded input/results, cleanup and GPU evidence remain mandatory. Repeat +pinned-pool controls at the new block size, retain the 5% full-suite regression +and 10% numeric gates, and do not infer acceptance from the targeted diagnostics. +Q15 shared-CTE reuse, Q13 join changes and default cutover require separate work. + +Sirius #32 merged as `87aaaa24137d20e5022e41d2b67c867b608883c7` and #33 +merged as `0cc6e484e0ee9dae3aa41919598116625d1abec8`. Sirius #34 merged as +`06400cf48073e51955067bfd75c5c4defe955a72`; the consumer pins this fixture +correction on top of logical input/result initialization and eligible Decimal64 grouped +SUM/AVG/COUNT. Checked global merge, final precision and AVG rounding retain +their existing implementation. Fresh guarded release qualification compares +against merged #31 on the same MO main base and frozen 16-CPU/64-KiB profile; +historical captures below retain their actual identities. Final optimization +acceptance requires the complete matched timing, resource, numeric and lifecycle +gates recorded below, plus CI and reviewer approval. + +The fresh optimization release capture uses MO `8fafa3ae3e`, Sirius `0cc6e484` +and guarded SDK `1de3ab8b`; its matched pre-optimization baseline uses MO +`8539955f9f`, Sirius `b953bb5a` and SDK `f65b21f4`. Both use MO main `3e50e1b32d`, +the same canonical SQL/datasets/provider, 16 Go CPUs and 64 KiB host blocks. +All 744 timing/correctness executions and 264 separate resource executions pass; +six warmed SF1/SF10 stream 1/2/4 profiles show zero unexplained growth. + +| GPU median | Matched baseline | Optimized release | +| --- | ---: | ---: | +| SF1 full suite | 1.686514 s | 1.395820 s | +| SF10 full suite | 15.091929 s | 12.556484 s | +| SF1 Q20 | 0.116503 s | 0.066711 s | +| SF10 Q20 | 1.235098 s | 0.857208 s | + +Both full-suite metrics pass the 5% regression limit; Q20 improves 42.7% and +30.6%, passing the 15% target at both scales. Q19 improves 37% at SF10. Q15 is +12% slower in this capture; shared-CTE reuse remains separate work. +All ten numeric controls pass the 10% limit on 42 retained samples per case. +One bounded balanced follow-up resolves a bimodal scalar result without +discarding the initial samples or changing inputs, timing boundaries or limits. + +Fresh merged-source GPU checks pass 39 cases/35,107 assertions; production +numeric ABI checks pass ten cases/1,964 assertions. The 96 MiB result tests +close ordered/unordered output, full-window cancellation/deadline and borrowed +lifetime. Native control normal/ASan+UBSan/TSan checks pass 93 cases/877 +assertions using unchanged merged production and the accounting-only fixture +correction in Sirius #34. Four raw sanitizer statuses remain 99, reconciled +against the exact Native A CUB and same-profile no-GPU-work pinned-pool +controls plus all six zero-growth profiles. The conversion CUDA source is +unchanged; the C++ pool-constructor change is separately identified. + +The delivery also merges main `175a645a85`: its set-operation remapping and +row-error-only changes do not affect successful canonical TPC-H paths. The +new error branch passes its targeted frontend regression. A six-line +test-only gossip allocator repair closes the inherited main compile error and +preserves the shared ephemeral-port exclusion and failure propagation. +Measured captures retain their actual identities; no source or binary is +relabeled. Public writer/compiler normal/race cleanup, owning function/compiler, +actual GPU bridge and same-process cuVS checks, all 86 tooling tests and full +SCA pass. Fail-closed artifact acceptance reports local completion, with CI and +reviewer approval remaining. Generated evidence is published separately from +this source change in the contributor-fork evidence branch described below. + +The deep delivery review rejects CPU budgets beyond Go's signed 32-bit range: +the runtime silently ignores those values, so accepting them would misreport +the execution profile. Boundary fixtures fail before this guard and pass after +it. The validated 16-CPU profile is unchanged. Merged #34 is tree-identical to +the tested fixture correction and changes no production, ABI, CMake or provider +input relative to measured `0cc6e484`. Its source-pin and SDK receipts are +refreshed separately; existing captures retain their original identities. +The refreshed release SDK is `878c0ec8` at merged `06400cf4`. Its guarded MO +consumer build, standard merged control suite (93 cases/877 assertions), owning +parser/function/compiler suites, all 22 native preparations and real GPU +bridge/cancellation/cuVS coexistence checks pass. Main `e5111936b9`'s +qualified-identifier parser consumers pass separately. The unchanged complete +archive passes the tightened acceptance validator; raw runs are not relabeled. + +## Optimization evidence preservation + +The matched optimization archive and merged #34 qualification are published at +immutable commit `56a5c86936fbda06d4c41dbf2ccbcb717e82901d`, +on [evidence branch `evidence/sirius-perf-30021-c5b77a2594`](https://github.com/aunjgr/matrixone/tree/evidence/sirius-perf-30021-c5b77a2594). +The [qualification ledger](https://github.com/aunjgr/matrixone/blob/7a241f4138dcc01872636e2e55f3fadeaf828310/docs/design/evidence/sirius-perf-30021/qualification.json) +records the original baseline/candidate identities, all captured artifact +checksums, and the source/build/provider equivalence through merged `06400cf4`. +The bundle retains the 744-run candidate and matched baseline, 264 resource runs, +all 42 numeric samples per case, raw sanitizer/control logs, and merged-SDK +build, control, 22-query preparation and actual GPU bridge evidence. + +Product validation identifies MO `4e330e4219` separately from the earlier +measured `8fafa3ae3e` capture. Review fix `c5b77a2594` changes archive verification +and its tests/documentation; Go/CGo/native product code is unchanged. Its 89 +tooling tests pass, the original checksum-bound v2 archive verifies unchanged, +and new execution still requires an explicit valid profile. Neither historical +nor optimized measurements are relabeled as a new benchmark run. CI and +reviewer approval remain external gates. + +Current delivery qualification for MO `195457c2fd` and main `8586fa0871` is +published separately at immutable evidence commit +`7a241f4138dcc01872636e2e55f3fadeaf828310`. Owning function/EXPLAIN suites, +string/collation ownership controls under race, all 22 native preparations, +GPU bridge/cancellation/cuVS checks, the source/SDK guarded service build and +full SCA pass. The current EXPLAIN BVT passes 115/115 twice with teardown. +These delivery checks preserve the original performance-capture identities; +they do not claim a new benchmark or passing remote coverage result. The +remote fetch verifies all three archive hashes and all 133 recorded file +checksums. The downloaded benchmark archive passes acceptance with +`local_acceptance_complete=true`; the evidence-access review thread is resolved. + +From the source checkout, export and verify the immutable bundle: + +```sh +git fetch https://github.com/aunjgr/matrixone.git \ + refs/heads/evidence/sirius-perf-30021-c5b77a2594 +task_optimization_dir=$(mktemp -d) +git show 56a5c86936fbda06d4c41dbf2ccbcb717e82901d:docs/design/evidence/sirius-perf-30021/sirius-perf-30021.tar.gz \ + > "$task_optimization_dir/qualification.tar.gz" +printf '%s %s\n' \ + dad13b3c3fc6bf7a2aba545fbb173e0aea7e457e0b65080b66ec8fceb43583cc \ + "$task_optimization_dir/qualification.tar.gz" | sha256sum -c - +tar -xzf "$task_optimization_dir/qualification.tar.gz" -C "$task_optimization_dir" +task_optimization_root="$task_optimization_dir/sirius-perf-30021/acceptance" +python3 optools/sirius_acceptance.py \ + --timing "$task_optimization_root/timing" \ + --resource "$task_optimization_root/resource" \ + --allocations "$task_optimization_root/allocations.json" \ + --lifecycle "$task_optimization_root/lifecycle.json" \ + --sanitizer "$task_optimization_root/sanitizer.json" \ + --baselines "$task_optimization_root/numeric-baselines.json" +``` + ## Contract and choice For every admitted campaign cell, the immutable dataset, canonical SQL, public @@ -204,7 +353,11 @@ the generator; a matching row count alone is insufficient. Use `sirius_campaign.py freeze` with the source root, packaged service, runner, dataset directory and both load reports to capture actual source/binary/SDK, -hardware and toolchain identities. Credentials remain in private `runner.json`. +hardware and toolchain identities. Supply `--gomaxprocs 16` to both `serve` and +`freeze` for the current host. Their `--host-block-bytes` defaults to 65536; +use 1048576 explicitly for the matched block-size control. `run` replays the +frozen CPU and block settings, and does not accept ambient overrides. +Credentials remain in private `runner.json`. `run` consumes that frozen provenance and an empty output directory; add `--resource` for the separate allocation campaign. Run serially, without concurrent builds, GPU tests or dataset writes. The controller owns service @@ -318,8 +471,108 @@ not restore admission or publish complete evidence. Small live-writer tests exercise quota growth during finalization/export and preserve the original reduced-quota counterexamples. +## Bounded input batching integration + +Sirius #31 merged as `b953bb5a87788e17557929987208b3a0068d933b`. +This follow-up pins that merged revision and rebuilds the guarded release SDK. +Native input now collects toward the existing 32 MiB expanded target, flushing +on EOS, source bounds or producer credit pressure. Both readiness and direct +claims enforce the policy; immutable size/string/NULL summaries remain charged +to the 64 MiB input window. Public ABI and exact numeric semantics are preserved. + +The user approved the batching-plus-MO-integration plan with EOS/pressure-only +flushing on 2026-10-12. Premerge development validation uses MO base `64b7a43852`, +Sirius `955b6881` versus `585536bf`, the same frozen provider and immutable +datasets, two streams, one excluded warmup and five measured repetitions. Q1 GPU +medians improve from 0.775129 to 0.185689 seconds at SF1 and from 9.296148 to +2.217461 seconds at SF10; median input units fall from 600 to 12 and 7,264 to 117. +All 22 SF1 public comparisons pass exactly. These targeted development results +retain their identities and do not replace the original D release records. + +Merged-source release validation requires the complete 744-execution timing and +correctness schedule plus the separate 264-execution resource schedule, existing +numeric/lifecycle/sanitizer gates and a fresh matched full-suite regression limit +of 5%. Generate captures outside Git and preserve their actual source/binary/SDK +identities. E and numeric execution changes remain separate; no sidecar repository +retirement is required. + +The merged-source release capture uses MO `81db83e921` on base `080365f013`, +Sirius `b953bb5a` and guarded SDK +`4f2235b64347a2642dbad1328de2f031cfbfc401ce114ed55bafcb0e68802ee6`. +All 744 timing/correctness executions and the separate 264 resource executions +pass, with six zero-growth SF1/SF10 stream 1/2/4 allocation profiles. A fresh +744-execution old-native campaign uses the same MO base and semantic inputs. + +| GPU median full suite | Old pin `955b6881` | Merged pin `b953bb5a` | +| --- | ---: | ---: | +| SF1 | 4.216747 s | 2.537705 s | +| SF10 | 49.884309 s | 28.475628 s | + +Median suites and sums of query medians pass the 5% regression limit. Q1 medians +fall from 0.876390 to 0.230886 seconds at SF1 and from 9.446339 to 2.359481 seconds +at SF10; median input units fall from 670 to 12 and 7,258 to 117. GPU full-suite +execution remains slower than native MO on this RTX 3070. + +Final validation closes all ten lifecycle cases, fresh production numeric ABI +and 96 MiB result ownership checks, public runner/compiler normal and race +cleanup checks, and actual GPU data/cancellation plus cuVS/Sirius coexistence. +Native control normal/ASan+UBSan+LSan/TSan runs pass 91 cases/750 assertions; +their actual `585536bf` execution revision is retained and qualified by complete +native-tree equality with merged `b953bb5a`. Fresh GPU memory checking passes +30 cases/34,509 assertions. Four raw sanitizer statuses remain 99, reconciled +only against the exact Native A CUB and no-GPU-work cuDF pinned-pool controls +and all six current steady profiles. Fresh matched numeric cases each retain +21 samples; the maximum ratio is 1.021216 against the 1.1 limit. + +Linked native test builds refreshed a receipt-covered static archive. The real +SDK target and guarded consumer build regenerate validation SDK `f65b21f4`, +with identical native source, compilers and explicit link arguments. Reports +retain this actual consumer execution context separately from the preserved +measured service/runtime bundle and SDK `4f2235b6`; no capture is relabeled. +The artifact acceptance command reports local completion, with required CI and +reviewer approval remaining. Generated captures and raw repetitions stay outside +Git; the original D archive below retains its original measurements. + +The delivery sync includes authoritative main `11914526ba` (MEDIUMINT, +prepared-parameter, SHOW GRANTS, window and Parquet fixes). Canonical SQL, +native bridge/lowering, reader ownership, public campaign evidence and +collectors are unchanged. The new type/protocol branches concern MEDIUMINT or +prepared parameters; the immutable campaign uses ordinary column domains and +text-protocol queries. All eight affected SQL owning suites and all 22 actual +native preparations pass after this sync. The distributed-IVF protocol fixture +now explicitly selects its required centroid route through `gpu_mode=0`, +preserving all supported/old/unknown/cancellation assertions. Its earlier +GPU-default failure reproduced on the old-native baseline. This test correction +changes no production dispatch or captured measurement. + +The conflict-resolution sync includes authoritative main `0a4db0e6a6` at +`87803799f0`. Main's updated IndexSearchScan protocol fixture already selects +CPU centroid workers and uses its current protocol version; the integration +has no remaining Go-file delta against that base. The full compiler suite and +all 22 actual native preparations pass after the sync. These checks retain +their new execution identity; the measured release capture above is unchanged. + ## Release validation record +### CI process-pool readiness correction (2026-10-14) + +The standard race UT retry at `778c61e4fc` failed in +`TestEmbeddedPrebuiltExecutionUsesBoundedProcessPool`. Its driver applied a +three-second worker-readiness timeout while precompilation was still running. +A controlled four-second compilation hold reproduces driver status 100; the +corrected fixture passes the same hold using FIFO readiness and the existing +fifteen-second outer harness deadline. All concurrency, identity, checkpoint +ordering and cleanup assertions remain. All 286 tooling cases pass normally +and under race, and the incremental tooling static check passes. + +Main `cd1eaa7a8a` and its subsequent ROLLUP correction `c3c4581595` are merged +without conflicts. Current SQL/function/parser/EXPLAIN/planner/metadata/exporter/ +compiler/frontend owning evidence covers 10,327 roots and 37,727 named passes. +All 22 actual native preparations and GPU data/cancellation/cuVS controls pass +with unchanged native pin `06400cf4` and SDK fingerprint `878c0ec8`. The final +planner group is rerun after the ROLLUP merge; the other owning inputs remain +unchanged. Retained timing/resource captures keep their original identities. + The generated development and release captures are preserved in contributor-fork tag `evidence/sirius-d-4230ce3d14`, at commit `4230ce3d143e77a8b4be83feb47bd26858a8d459`. The diff --git a/optools/sirius_acceptance_test.py b/optools/sirius_acceptance_test.py index cf1b537f7305f..e08f892be67b4 100644 --- a/optools/sirius_acceptance_test.py +++ b/optools/sirius_acceptance_test.py @@ -228,6 +228,27 @@ def records(manifest, offset): class AcceptanceTest(unittest.TestCase): + def test_historical_v2_archive_preserves_missing_execution_profile(self): + inputs = evidence() + identities = [ + inputs[0]["provenance"], + inputs[2]["provenance"], + *(item["provenance"] for item in inputs[4]), + *(item["provenance"] for item in inputs[5:]), + ] + for identity in identities: + identity["config"].pop("gomaxprocs", None) + identity["config"].pop("native_host_block_bytes", None) + original = copy.deepcopy(inputs) + self.assertTrue(acceptance.validate(*inputs)["local_acceptance_complete"]) + self.assertEqual(original, inputs, "verification must not invent settings") + + inputs = evidence() + for key in ("gomaxprocs", "native_host_block_bytes"): + del inputs[0]["provenance"]["config"][key] + with self.assertRaisesRegex(campaign.CampaignError, "semantic inputs differ"): + acceptance.validate(*inputs) + def test_cli_binds_consumed_inputs_to_archive_checksums(self): inputs = evidence() with tempfile.TemporaryDirectory() as directory: diff --git a/optools/sirius_campaign.py b/optools/sirius_campaign.py index 935896ce5ca40..3e703b036e053 100644 --- a/optools/sirius_campaign.py +++ b/optools/sirius_campaign.py @@ -854,7 +854,19 @@ def write_timing_tables(output, campaign, records, summary): write_private(output / "timings.md", "\n".join(lines) + "\n") -def native_configuration(workspace): +def execution_environment(gomaxprocs): + if type(gomaxprocs) is not int or not 0 < gomaxprocs <= v2.MAX_GOMAXPROCS: + raise v2.CampaignError( + "campaign Go CPU budget must be between 1 and 2147483647" + ) + environment = os.environ.copy() + environment["GOMAXPROCS"] = str(gomaxprocs) + return environment + + +def native_configuration(workspace, host_block_bytes=65536): + if type(host_block_bytes) is not int or host_block_bytes not in (65536, 1048576): + raise v2.CampaignError("unsupported campaign host block size") return f"""sirius: space: gpu: @@ -863,6 +875,7 @@ def native_configuration(workspace): host: - numa_id: -1 memory_capacity: {16 << 30} + block_size: {host_block_bytes} telemetry: output_directory: {json.dumps(str(workspace/'native-telemetry'))} """ @@ -1089,6 +1102,14 @@ def run_complete(arguments, *, provenance=None): ] if base_runner.get("session") != expected_session: raise v2.CampaignError("live campaign session differs from frozen provenance") + campaign = v2.build_campaign( + provenance, + resource=arguments.resource, + flight_profiles=[ + {"id": route, "coverage": [], "status": "N/A", "reason": "not-provisioned"} + for route in ("flight-mo", "flight-tae") + ], + ) sirius = tomllib.loads((workspace / "cn-2.toml").read_text())["cn"]["sirius"] if any( sirius.get(key) != value @@ -1100,21 +1121,12 @@ def run_complete(arguments, *, provenance=None): "cleanup-timeout": "30s", }.items() ) or Path(sirius["native-config-path"]).read_text() != native_configuration( - workspace + workspace, provenance["config"]["native_host_block_bytes"] ): raise v2.CampaignError( "live campaign configuration differs from frozen profile" ) - campaign = v2.build_campaign( - provenance, - resource=arguments.resource, - flight_profiles=[ - {"id": route, "coverage": [], "status": "N/A", "reason": "not-provisioned"} - for route in ("flight-mo", "flight-tae") - ], - ) - environment = os.environ.copy() - environment["GOMAXPROCS"] = "4" + environment = execution_environment(provenance["config"]["gomaxprocs"]) run_workspace = workspace / f"campaign-{uuid.uuid4().hex}" run_workspace.mkdir(mode=0o700) (run_workspace / "scratch").mkdir(mode=0o700) @@ -1521,13 +1533,12 @@ def excluded(specs, phase_name, base): def start_cluster(arguments): + environment = execution_environment(arguments.gomaxprocs) workspace = arguments.workspace.resolve() workspace.mkdir(mode=0o700, parents=True) native = workspace / "sirius.yaml" - write_private(native, native_configuration(workspace)) + write_private(native, native_configuration(workspace, arguments.host_block_bytes)) configs = make_configs(workspace, native, 2, arguments.port_base) - environment = os.environ.copy() - environment["GOMAXPROCS"] = "4" services = [] previous_subreaper = v2.legacy._enter_exclusive_subreaper() try: @@ -1578,6 +1589,8 @@ def freeze_provenance(arguments): """Capture actual binaries, guarded SDK, source data and local toolchain.""" import sirius_sdk + execution_environment(arguments.gomaxprocs) + native_configuration(arguments.source_root, arguments.host_block_bytes) root = arguments.source_root.resolve() packaged = json.loads((root / "lib/sirius-provenance.json").read_text()) if packaged["binary_sha256"] != sha256(arguments.service): @@ -1668,6 +1681,8 @@ def command(*args): "gdb": command("gdb", "--version").splitlines()[0], }, "config": { + "gomaxprocs": arguments.gomaxprocs, + "native_host_block_bytes": arguments.host_block_bytes, "gpu_memory_bytes": 6 << 30, "native_host_memory_bytes": 16 << 30, "input_window_bytes": 64 << 20, @@ -1702,6 +1717,10 @@ def main(argv=None): serve.add_argument("--service", type=Path, required=True) serve.add_argument("--source-root", type=Path, required=True) serve.add_argument("--port-base", type=int, default=36001) + serve.add_argument("--gomaxprocs", type=int, required=True) + serve.add_argument( + "--host-block-bytes", type=int, choices=(65536, 1048576), default=65536 + ) run = sub.add_parser("run") run.add_argument("--workspace", type=Path, required=True) run.add_argument("--service", type=Path, required=True) @@ -1724,6 +1743,10 @@ def main(argv=None): freeze.add_argument("--datasets", type=Path, required=True) freeze.add_argument("--loaded", type=Path, nargs=2, required=True) freeze.add_argument("--output", type=Path, required=True) + freeze.add_argument("--gomaxprocs", type=int, required=True) + freeze.add_argument( + "--host-block-bytes", type=int, choices=(65536, 1048576), default=65536 + ) args = parser.parse_args(argv) def interrupted(*_): diff --git a/optools/sirius_campaign_test.py b/optools/sirius_campaign_test.py index ea0fa3b720c05..a614b7afdb0cf 100644 --- a/optools/sirius_campaign_test.py +++ b/optools/sirius_campaign_test.py @@ -36,6 +36,70 @@ class ControllerTest(unittest.TestCase): + def test_execution_environment_replays_frozen_cpu_budget(self): + with patch.dict(os.environ, {"GOMAXPROCS": "99"}): + for budget in (16, (1 << 31) - 1): + environment = controller.execution_environment(budget) + self.assertEqual(environment["GOMAXPROCS"], str(budget)) + self.assertEqual(os.environ["GOMAXPROCS"], "99") + for value in (0, -1, True, 1.5, "16", None, 1 << 31, 10**100): + with self.subTest(value=value), self.assertRaises(v2.CampaignError): + controller.execution_environment(value) + + def test_native_configuration_records_supported_block_size(self): + workspace = Path("/tmp/campaign-profile-fixture") + self.assertIn("block_size: 65536", controller.native_configuration(workspace)) + self.assertIn( + "block_size: 1048576", controller.native_configuration(workspace, 1048576) + ) + for value in (0, -1, True, 65536.0, 32768, 2097152): + with self.subTest(value=value), self.assertRaises(v2.CampaignError): + controller.native_configuration(workspace, value) + + def test_frozen_block_drift_rejects_before_service_launch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + service, runner = root / "service", root / "runner" + service.write_text("fixture service") + runner.write_text("fixture runner") + identity = provenance() + identity["service_binary_sha256"] = controller.sha256(service) + identity["runner_binary_sha256"] = controller.sha256(runner) + identity["controller_sha256"] = controller.controller_sha256() + (workspace / "runner.json").write_text( + json.dumps( + { + "session": [ + "set session mo_sirius_campaign_evidence=1", + "set session div_precision_increment=4", + "set time_zone='+00:00'", + ] + } + ) + ) + native = workspace / "sirius.yaml" + native.write_text(controller.native_configuration(workspace, 1048576)) + (workspace / "cn-2.toml").write_text( + '[cn.sirius]\nenabled=true\nbackend="embedded"\ninput-mode="mo"\n' + 'max-waiting-queries=16\ncleanup-timeout="30s"\n' + + "native-config-path=" + + json.dumps(str(native)) + + "\n" + ) + args = argparse.Namespace( + workspace=workspace, + output=root / "output", + service=service, + runner=runner, + resource=False, + ) + with patch.object(controller, "Service") as launch: + with self.assertRaisesRegex(v2.CampaignError, "configuration differs"): + controller.run_complete(args, provenance=identity) + launch.assert_not_called() + def test_service_generation_failure_retains_spawn_cleanup(self): with tempfile.TemporaryDirectory() as directory: child = MagicMock(pid=123) diff --git a/optools/sirius_campaign_v2.py b/optools/sirius_campaign_v2.py index 8ed040709bcac..e932cbf945fd0 100644 --- a/optools/sirius_campaign_v2.py +++ b/optools/sirius_campaign_v2.py @@ -44,6 +44,10 @@ ROW_BYTES = 1 << 20 OUTPUT_BYTES = 1 << 20 STDERR_BYTES = 64 << 10 +# The Go runtime parses GOMAXPROCS as a signed 32-bit integer. Larger values +# are ignored, so accepting one would record a CPU budget the service never uses. +MAX_GOMAXPROCS = (1 << 31) - 1 +EXECUTION_PROFILE_FIELDS = frozenset(("gomaxprocs", "native_host_block_bytes")) ROUTES = ( {"id": "mo-native", "backend": "native", "scan_mode": "mo"}, {"id": "embedded-mo", "backend": "embedded", "scan_mode": "mo"}, @@ -91,7 +95,24 @@ def query_uuid(value): return value -def build_campaign(provenance, *, resource=False, flight_profiles=()): +def validate_execution_profile(config): + if not isinstance(config, dict) or not EXECUTION_PROFILE_FIELDS <= config.keys(): + raise CampaignError("campaign execution requires an explicit CPU and host profile") + if ( + type(config["gomaxprocs"]) is not int + or not 0 < config["gomaxprocs"] <= MAX_GOMAXPROCS + ): + raise CampaignError("invalid campaign Go CPU budget") + if ( + type(config["native_host_block_bytes"]) is not int + or config["native_host_block_bytes"] not in (65536, 1048576) + ): + raise CampaignError("invalid campaign host block size") + + +def build_campaign( + provenance, *, resource=False, flight_profiles=(), require_execution_profile=True +): required = ( "matrixone_revision", "sirius_revision", @@ -161,6 +182,11 @@ def build_campaign(provenance, *, resource=False, flight_profiles=()): ), ): value = provenance[field] + if field == "config" and isinstance(value, dict): + # V2 archives predating explicit profiles retain their original + # shape. Partial or invalid profiles never become historical data. + if require_execution_profile or EXECUTION_PROFILE_FIELDS & value.keys(): + allowed |= EXECUTION_PROFILE_FIELDS if not isinstance(value, dict) or set(value) != allowed: raise CampaignError("invalid bounded provenance fields") if len(canonical(value)) > 4096: @@ -173,6 +199,9 @@ def build_campaign(provenance, *, resource=False, flight_profiles=()): "time_zone": "+00:00", }: raise CampaignError("campaign session semantics differ") + config = provenance["config"] + if require_execution_profile or EXECUTION_PROFILE_FIELDS & config.keys(): + validate_execution_profile(config) expected = { "gpu_memory_bytes": 6 << 30, "native_host_memory_bytes": 16 << 30, @@ -1013,6 +1042,7 @@ def execute_campaign(campaign, executor, scratch): identities = set() produced = set() try: + validate_execution_profile(campaign["provenance"]["config"]) for spec in campaign["schedule"]: request = copy.deepcopy(spec) request["spool_budget_bytes"] = min( @@ -1101,6 +1131,7 @@ def summarize(campaign, records): campaign["provenance"], resource=campaign["resource_campaign"], flight_profiles=campaign["flight_profiles"], + require_execution_profile=False, ) if campaign != trusted: raise CampaignError("summary rejects a partial or modified acceptance schedule") diff --git a/optools/sirius_campaign_v2_test.py b/optools/sirius_campaign_v2_test.py index adfe80ef9fd92..b3e349e251b73 100644 --- a/optools/sirius_campaign_v2_test.py +++ b/optools/sirius_campaign_v2_test.py @@ -50,6 +50,8 @@ def provenance(): "gdb": "test", }, "config": { + "gomaxprocs": 4, + "native_host_block_bytes": 65536, "gpu_memory_bytes": 6 << 30, "native_host_memory_bytes": 16 << 30, "input_window_bytes": 64 << 20, @@ -147,6 +149,29 @@ def test_provenance_rejects_secrets_and_changed_limits(self): with self.assertRaises(campaign.CampaignError): campaign.build_campaign(value) + def test_execution_profile_requires_explicit_bounded_settings(self): + for budget, block in ((4, 1048576), (16, 65536), ((1 << 31) - 1, 65536)): + value = provenance() + value["config"].update(gomaxprocs=budget, native_host_block_bytes=block) + frozen = campaign.build_campaign(value)["provenance"]["config"] + self.assertEqual(frozen["gomaxprocs"], budget) + self.assertEqual(frozen["native_host_block_bytes"], block) + for key, values in ( + ("gomaxprocs", (None, True, 0, -1, "16", 16.0, 1 << 31, 10**100)), + ("native_host_block_bytes", (None, True, 0, 32768, 65536.0)), + ): + for setting in values: + value = provenance() + value["config"][key] = setting + with self.subTest(key=key, setting=setting), self.assertRaises( + campaign.CampaignError + ): + campaign.build_campaign(value) + value = provenance() + del value["config"][key] + with self.subTest(missing=key), self.assertRaises(campaign.CampaignError): + campaign.build_campaign(value) + def test_schedule_counts_and_sampling_separation(self): timing = campaign.build_campaign(provenance()) resources = campaign.build_campaign(provenance(), resource=True) @@ -159,6 +184,43 @@ def test_schedule_counts_and_sampling_separation(self): ) self.assertEqual((0, 2, 1, 3), campaign.ORDER_KEYS[2]) + def test_historical_profile_cannot_launch_an_executor(self): + manifest = campaign.build_campaign(provenance()) + for key in ("gomaxprocs", "native_host_block_bytes"): + del manifest["provenance"]["config"][key] + with self.assertRaises(campaign.CampaignError): + campaign.build_campaign(manifest["provenance"]) + calls = [] + + def executor(spec): + calls.append(spec) + return {} + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + root.chmod(0o700) + with self.assertRaises(campaign.CampaignError): + campaign.execute_campaign(manifest, executor, campaign.Scratch(root)) + self.assertEqual([], calls, "historical verification cannot admit execution") + self.assertEqual([], list(root.iterdir())) + + def test_archive_verification_rejects_partial_or_invalid_profiles(self): + for key, values in ( + ("gomaxprocs", (None, True, 0, 1 << 31)), + ("native_host_block_bytes", (None, True, 32768, 65536.0)), + ): + for setting in values: + manifest = campaign.build_campaign(provenance()) + manifest["provenance"]["config"][key] = setting + with self.subTest(key=key, setting=setting), self.assertRaises( + campaign.CampaignError + ): + campaign.summarize(manifest, []) + manifest = campaign.build_campaign(provenance()) + del manifest["provenance"]["config"][key] + with self.subTest(missing=key), self.assertRaises(campaign.CampaignError): + campaign.summarize(manifest, []) + def test_full_value_duplicate_and_empty_comparison(self): for a, b, keys, ok in ( ([], [], [], True), diff --git a/optools/ut_embedded_schedule_test.go b/optools/ut_embedded_schedule_test.go index 9f1b859eff130..cac0eb0833a5e 100644 --- a/optools/ut_embedded_schedule_test.go +++ b/optools/ut_embedded_schedule_test.go @@ -257,14 +257,16 @@ trap cleanup_pool_driver EXIT ( trap release_pool_children EXIT trap 'exit 143' TERM INT - read -r -t 3 first first_pid <&8 || exit 100 - read -r -t 3 second second_pid <&8 || exit 101 + # The harness deadline bounds the whole operation. Precompilation may still + # be running here, so worker readiness must not start a separate short timer. + read -r first first_pid <&8 || exit 100 + read -r second second_pid <&8 || exit 101 [[ "$first $second" == 'a b' || "$first $second" == 'b a' ]] || exit 102 [[ "$first_pid" != "$second_pid" ]] || exit 103 kill -0 "$first_pid" && kill -0 "$second_pid" || exit 104 [[ -e "$CASE_DIR/active-a" && -e "$CASE_DIR/active-b" && ! -e "$CASE_DIR/executed-c" ]] || exit 105 printf 'release\n' >&10 - read -r -t 3 third third_pid <&8 || exit 106 + read -r third third_pid <&8 || exit 106 [[ "$third" == c && "$third_pid" != "$first_pid" && "$third_pid" != "$second_pid" ]] || exit 107 kill -0 "$third_pid" || exit 108 [[ ! -e "$CASE_DIR/active-a" && -e "$CASE_DIR/active-b" && -e "$CASE_DIR/active-c" ]] || exit 109 @@ -275,7 +277,10 @@ UT_EMBEDDED_PACKAGE_PARALLEL=2 run_embedded_tests "$scope" || status=$? driver_status=0 wait "$pool_driver_pid" || driver_status=$? pool_driver_pid="" -[[ "$status" == 0 && "$driver_status" == 0 ]] || exit 90 +[[ "$status" == 0 && "$driver_status" == 0 ]] || { + printf 'embedded execution status=%s pool driver status=%s\n' "$status" "$driver_status" >&2 + exit 90 +} [[ "$(grep -c '^pool=2 ' "$CASE_DIR/pool-events")" == 3 ]] || exit 91 # Indexed execution checkpoints exclude the outer command and compile events. awk ' diff --git a/pkg/sql/plan/function/operator_in.go b/pkg/sql/plan/function/operator_in.go index d423b6cdc95fd..b9b7adffc6e58 100644 --- a/pkg/sql/plan/function/operator_in.go +++ b/pkg/sql/plan/function/operator_in.go @@ -118,7 +118,7 @@ func (op *opOperatorStrIn) init(left, tuple *vector.Vector) { return } op.mp = make(map[string]bool, 1) - op.mp[op.key(v)] = true + op.mp[string(op.keyBytes(v))] = true return } @@ -129,11 +129,11 @@ func (op *opOperatorStrIn) init(left, tuple *vector.Vector) { op.hasNull = true continue } - op.mp[op.key(v)] = true + op.mp[string(op.keyBytes(v))] = true } } -func (op *opOperatorStrIn) key(value []byte) string { +func (op *opOperatorStrIn) keyBytes(value []byte) []byte { if op.padSpace { value = bytes.TrimRight(value, " ") } @@ -144,7 +144,14 @@ func (op *opOperatorStrIn) key(value []byte) string { // can collide with a valid UCA weight string. value = types.CollationKeyOrOriginal(op.typ.Charset, value) } - return string(value) + return value +} + +func (op *opOperatorStrIn) containsKey(value []byte) bool { + // A lookup borrows the normalized bytes only for this expression. Map + // insertion above retains an owned string; probes need no owned copy. + _, ok := op.mp[string(op.keyBytes(value))] + return ok } func (op *opOperatorFixedIn[T]) operatorIn(parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { @@ -219,7 +226,7 @@ func (op *opOperatorStrIn) operatorIn(parameters []*vector.Vector, result vector return err } } else { - _, ok := op.mp[op.key(v)] + ok := op.containsKey(v) if !ok && op.hasNull { if err := rs.Append(false, true); err != nil { return err @@ -248,7 +255,7 @@ func (op *opOperatorStrIn) operatorNotIn(parameters []*vector.Vector, result vec return err } } else { - _, ok := op.mp[op.key(v)] + ok := op.containsKey(v) if !ok && op.hasNull { if err := rs.Append(false, true); err != nil { return err diff --git a/pkg/sql/plan/function/operator_in_test.go b/pkg/sql/plan/function/operator_in_test.go index 702022629798c..c8a0ccd857c27 100644 --- a/pkg/sql/plan/function/operator_in_test.go +++ b/pkg/sql/plan/function/operator_in_test.go @@ -23,6 +23,49 @@ import ( "github.com/stretchr/testify/require" ) +func TestOperatorStrInRetainsOwnedKeys(t *testing.T) { + proc := testutil.NewProcess(t) + for _, typ := range []types.Type{types.T_varchar.ToType(), types.New(types.T_char, 64, 0)} { + t.Run(typ.Oid.String(), func(t *testing.T) { + tuple := vector.NewVec(typ) + defer tuple.Free(proc.Mp()) + value := []byte("membership key longer than inline vector storage ") + require.NoError(t, vector.AppendBytes(tuple, value, false, proc.Mp())) + op := newOpOperatorStrIn() + op.init(nil, tuple) + probe := append([]byte(nil), tuple.GetBytesAt(0)...) + // Reusing the list vector must not alter a retained map key. + tuple.GetBytesAt(0)[0] = 'X' + require.True(t, op.containsKey(probe)) + require.False(t, op.containsKey(tuple.GetBytesAt(0))) + }) + } +} + +func TestOperatorStrInBinaryProbeAllocations(t *testing.T) { + key := "membership key longer than the compiler's short-string buffer" + op := &opOperatorStrIn{typ: types.T_varchar.ToType(), mp: map[string]bool{key: true}} + probe := []byte(key) + require.Zero(t, testing.AllocsPerRun(1000, func() { + if !op.containsKey(probe) { + t.Fatal("known key was not found") + } + })) +} + +func BenchmarkOperatorStrInBinaryProbe(b *testing.B) { + key := "membership key longer than the compiler's short-string buffer" + op := &opOperatorStrIn{typ: types.T_varchar.ToType(), mp: map[string]bool{key: true}} + probe := []byte(key) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !op.containsKey(probe) { + b.Fatal("known key was not found") + } + } +} + func TestOperatorInNullableListUsesThreeValuedLogic(t *testing.T) { proc := testutil.NewProcess(t) diff --git a/third_party/sirius b/third_party/sirius index 955b6881afa6e..06400cf48073e 160000 --- a/third_party/sirius +++ b/third_party/sirius @@ -1 +1 @@ -Subproject commit 955b6881afa6e92d0995f43e3fe9c383d86981c0 +Subproject commit 06400cf48073e51955067bfd75c5c4defe955a72