Skip to content

perf(ci): per-package compiled cache, test shards, single generator pass #1345

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
apresmoi merged 2 commits into from
Oct 5, 2026
Merged

perf(ci): per-package compiled cache, test shards, single generator pass #1345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9f4f0f7
perf(ci): per-package compiled cache, test shards, single generator pass
apresmoi Oct 5, 2026
60bd3d6
fix(ci): keep sharded child runs unsharded and type the new CI tests
apresmoi Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 59 additions & 37 deletions .github/scripts/ci/build-ci.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ import { spawn } from 'node:child_process';
import { lstatSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
import { isAbsolute, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { ciCacheKeys } from './ci-cache-key.mts';
import { compiledCiCacheKeys, type CompiledPackageInput } from './ci-cache-key.mts';

export type CiBuildMode = 'lint' | 'full';
export type CiBuildMode = 'lint' | 'full' | 'packages' | 'generators';
interface CompiledDirectory { path: string; required: readonly string[]; }
export interface CiBuildTask {
id: string;
Expand Down Expand Up @@ -48,19 +48,24 @@ export function ciBuildPlan(root: string, mode: CiBuildMode): readonly CiBuildTa
node('catalog', 'site/build/prepare/prepare-catalog.mts', ['packages']),
node('solar', 'packages/bake/cli/prepare-solar-geometry.mts', ['catalog']),
];
if (mode === 'full') tasks.push(
if (mode === 'full' || mode === 'generators') tasks.push(
// Hashes icon sources with @cssearth/core/node, so it waits for the packages build.
node('icons', 'site/build/prepare/prepare-shell-icons.mts', ['packages']),
node('navigation', 'packages/bake/cli/prepare-navigation.mts', ['solar'], ['--catalog-only']),
node('world', 'site/build/prepare/prepare-spatial-context.ts', ['navigation'], ['src/objects/sun/source/navigation/universe.json', 'src/objects/sun/prepared/world-context.json']),
node('moon-labels', 'site/build/prepare/prepare-moon-labels.mts', ['world']),
node('world-presentation', 'site/build/prepare/prepare-world-presentation.mts', ['world']),
);
// Split callers compile before restoring selected inputs. Keep the full generator closure and its
// internal ordering; only the already-completed package prerequisite leaves this phase.
if (mode === 'packages') return tasks.filter(task => task.id === 'packages');
if (mode === 'generators') return tasks.filter(task => task.id !== 'packages')
.map(task => ({ ...task, after: task.after.filter(id => id !== 'packages') }));
return tasks;
}

/** Receipts list only the small compiled directories, never the installed tree or restored data banks. The cache
* key already names the exact inputs; the listing proves the restore is complete: every file present, none empty, none extra. */
* receipt names each package's exact inputs; the listing proves the restore is complete: every file present, none empty, none extra. */
function outputInventory(directory: string): string[] {
const files: string[] = [];
const visit = (relative: string) => {
Expand All @@ -82,7 +87,7 @@ function outputInventory(directory: string): string[] {
export function compiledOutputsValid(root: string, output: CompiledDirectory, digest: string): boolean {
try {
const directory = resolve(root, output.path), receipt: unknown = JSON.parse(readFileSync(resolve(directory, RECEIPT), 'utf8'));
if (!record(receipt) || receipt.schema !== 2 || receipt.digest !== digest || !Array.isArray(receipt.files)) return false;
if (!record(receipt) || receipt.schema !== 3 || receipt.digest !== digest || !Array.isArray(receipt.files)) return false;
const files = receipt.files;
if (!files.length || !files.every(safePath)) return false;
if (!output.required.every(path => files.includes(path))) return false;
Expand All @@ -94,7 +99,7 @@ export function compiledOutputsValid(root: string, output: CompiledDirectory, di
function recordOutputs(root: string, output: CompiledDirectory, digest: string): void {
const directory = resolve(root, output.path), files = outputInventory(directory);
for (const path of output.required) if (!files.includes(path) || !lstatSync(resolve(directory, path)).isFile()) throw new TypeError(`Build did not produce ${output.path}/${path}`);
writeFileSync(resolve(directory, RECEIPT), JSON.stringify({ schema: 2, digest, files }) + '\n');
writeFileSync(resolve(directory, RECEIPT), JSON.stringify({ schema: 3, digest, files }) + '\n');
}

async function execute(task: CiBuildTask, root: string): Promise<void> {
Expand All @@ -105,12 +110,26 @@ async function execute(task: CiBuildTask, root: string): Promise<void> {
});
}

export async function buildCi({ root = resolve(import.meta.dirname, '../../..'), mode, cacheHit = false, digest,
packageCacheHit = cacheHit, packageDigest = digest, run = execute,
}: { root?: string; mode: CiBuildMode; cacheHit?: boolean; digest: string; packageCacheHit?: boolean;
packageDigest?: string; run?: (task: CiBuildTask, root: string) => Promise<void> }) {
for (const identity of [digest, packageDigest]) if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(identity)) throw new TypeError(`CI build keys must be exact Git object ids (ci-cache-key.mts); got ${JSON.stringify(identity)}.`);
const plan = ciBuildPlan(root, mode), pending = new Map<string, Promise<void>>(), results: { id: string; cached: boolean; seconds: number }[] = [];
/** Invalid receipts rebuild their reverse dependency closure, including when a dependency's digest matches
* but its restored files are incomplete. Explicit filters select exactly that set; pnpm orders the builds. */
export function packageRebuildClosure(packages: readonly CompiledPackageInput[], invalid: readonly string[]): string[] {
const selected = new Set(invalid);
let changed = true;
while (changed) {
changed = false;
for (const pkg of packages) if (!selected.has(pkg.name) && pkg.dependencies.some(name => selected.has(name))) {
selected.add(pkg.name); changed = true;
}
}
return packages.filter(pkg => selected.has(pkg.name)).map(pkg => pkg.name);
}

export async function buildCi({ root = resolve(import.meta.dirname, '../../..'), mode, packages, run = execute,
}: { root?: string; mode: CiBuildMode; packages?: readonly CompiledPackageInput[];
run?: (task: CiBuildTask, root: string) => Promise<void> }) {
packages ??= (await compiledCiCacheKeys({ root })).packages;
const inputs = new Map(packages.map(pkg => [`${pkg.directory}/dist`, pkg]));
const plan = ciBuildPlan(root, mode), pending = new Map<string, Promise<void>>(), results: { id: string; cached: boolean; seconds: number; rebuilt?: string[] }[] = [];
for (const task of plan) {
const parents = task.after.map(id => {
const parent = pending.get(id);
Expand All @@ -119,39 +138,42 @@ export async function buildCi({ root = resolve(import.meta.dirname, '../../..'),
});
pending.set(task.id, Promise.all(parents).then(async () => {
const started = performance.now();
const identity = task.id === 'packages' ? packageDigest : digest;
const hit = task.id === 'packages' ? packageCacheHit : cacheHit;
const cached = Boolean(hit && task.outputs?.length && task.outputs.every(output => compiledOutputsValid(root, output, identity)));
if (cached && task.id === 'packages') {
// The package dist cache intentionally excludes generated TS sources needed by typecheck/tests.
await run({ id: 'astronomy-data', after: [], command: process.execPath, args: ['packages/astronomy/cli/body-records.mts'] }, root);
} else if (!cached) {
await run(task, root);
for (const output of task.outputs ?? []) recordOutputs(root, output, identity);
}
const result = { id: task.id, cached, seconds: Number(((performance.now() - started) / 1000).toFixed(3)) };
let rebuilt: string[] | undefined;
if (task.id === 'packages') {
const outputs = task.outputs ?? [];
const invalid = outputs.filter(output => {
const pkg = inputs.get(output.path);
if (!pkg) throw new TypeError(`Missing compiled input identity: ${output.path}`);
return !compiledOutputsValid(root, output, pkg.digest);
}).map(output => inputs.get(output.path)!.name);
rebuilt = packageRebuildClosure(packages, invalid);
if (rebuilt.length) {
await run({ ...task, args: ['-r', ...rebuilt.flatMap(name => ['--filter', name]), 'build'] }, root);
for (const output of outputs) {
const pkg = inputs.get(output.path)!;
if (rebuilt.includes(pkg.name)) recordOutputs(root, output, pkg.digest);
}
}
if (!rebuilt.includes('@cssearth/astronomy') && packages.some(pkg => pkg.name === '@cssearth/astronomy')) {
// Warm dist does not restore generated TS needed by compiler/test consumers.
await run({ id: 'astronomy-data', after: [], command: process.execPath, args: ['packages/astronomy/cli/body-records.mts'] }, root);
}
} else await run(task, root);
const cached = task.id === 'packages' && rebuilt?.length === 0;
const result = { id: task.id, cached, seconds: Number(((performance.now() - started) / 1000).toFixed(3)), ...(rebuilt ? { rebuilt } : {}) };
results.push(result);
console.log(`[build-ci] ${task.id}: ${cached ? 'verified exact cache' : 'ran'} ${result.seconds}s`);
console.log(`[build-ci] ${task.id}: ${cached ? 'verified package receipts' : 'ran'} ${result.seconds}s`);
}));
}
// Do not return with concurrent generators/compilers still writing after another task fails.
const completed = await Promise.allSettled(pending.values()), failure = completed.find(result => result.status === 'rejected');
if (failure?.status === 'rejected') throw failure.reason;
return results;
}

if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
const mode = process.argv[2];
if (process.argv.length !== 3 || (mode !== 'lint' && mode !== 'full')) throw new TypeError('Usage: node .github/scripts/ci/build-ci.mts lint|full');
const flag = (name: string, fallback = false) => {
const value = process.env[name];
if (value === undefined || value === '') return fallback;
if (value !== 'true' && value !== 'false') throw new TypeError(`${name} must be true or false.`);
return value === 'true';
};
const hit = flag('CI_BUILD_CACHE_HIT');
const digest = process.env.CI_BUILD_DIGEST || ciCacheKeys().buildDigest;
const started = performance.now(), results = await buildCi({ mode, digest, cacheHit: hit,
packageCacheHit: flag('CI_PACKAGE_CACHE_HIT', hit), packageDigest: process.env.CI_PACKAGE_DIGEST || digest });
console.log(JSON.stringify({ mode, seconds: Number(((performance.now() - started) / 1000).toFixed(3)), compiledCacheHits: results.filter(result => result.cached).length }));
if (process.argv.length !== 3 || (mode !== 'lint' && mode !== 'full' && mode !== 'packages' && mode !== 'generators'))
throw new TypeError('Usage: node .github/scripts/ci/build-ci.mts lint|full|packages|generators');
const started = performance.now(), results = await buildCi({ mode });
console.log(JSON.stringify({ mode, seconds: Number(((performance.now() - started) / 1000).toFixed(3)), compiledCacheHits: results.filter(result => result.cached).length, results }));
}
17 changes: 11 additions & 6 deletions .github/scripts/ci/check-ci.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {spawn} from 'node:child_process';
import {parse} from 'yaml';
import {requireArray, requireRecord, requireString} from '@cssearth/core';
import {affectedJobNames, ALWAYS_JOBS, classifyAffectedPaths, HEAVY_JOBS, loadCiAreasConfig, localChangedPaths, needsProductionBuild} from './ci-affected.mts';
import { universeTestMatrix } from './test-shards.mts';
import {evaluateObjectScopeGate} from './object-scope-gate.mts';

interface CiStep {name:string;run:string;env:Record<string,string>;}
Expand Down Expand Up @@ -74,18 +75,21 @@ export function readCiSteps(source:string,jobName='universe', substitutions:Reco
if(job.strategy!==undefined){
const strategy=requireRecord(job.strategy),matrix=requireRecord(strategy.matrix);
if(!supported||Object.keys(strategy).sort().join(',')!=='fail-fast,matrix'||strategy['fail-fast']!==false||
Object.keys(matrix).join(',')!=='lane'||job['continue-on-error']!==undefined||
(supported.job==='universe-checks' ? Object.keys(matrix).join(',')!=='include'||matrix.include!=='${{ fromJSON(needs.changes.outputs.test_matrix).include }}' : Object.keys(matrix).join(',')!=='lane')||job['continue-on-error']!==undefined||
job.needs!=='changes'||requireString(job.if).trim().replace(/\s+/gu,' ')!=='${{ '+selectedMatrixCondition(supported)+' }}'||
requireRecord(job.env)[supported.laneEnv]!==MATRIX_LANE)
throw new Error('Local CI supports only the explicit maintained lane matrices with fail-fast disabled.');
const lanes=requireArray(matrix.lane).map(value=>requireString(value));
const rows=supported.job==='universe-checks'
? universeTestMatrix(resolve(import.meta.dirname,'../../..'),{packages:substitutions['${{ needs.changes.outputs.test_packages }}']??'all',files:substitutions['${{ needs.changes.outputs.test_files }}']??''})
: requireArray(matrix.lane).map(value=>({lane:requireString(value),shard:0,total:0}));
const lanes=rows.map(row=>row.shard?`${row.lane}-${row.shard}`:row.lane);
if(!lanes.length||new Set(lanes).size!==lanes.length||lanes.some(lane=>!/^[a-z][a-z0-9-]*$/u.test(lane)))
throw new Error('Local CI needs unique, nonempty literal matrix lanes.');
return lanes.flatMap(lane=>{
const steps=readJobSteps(workflow,job,{...substitutions,[MATRIX_LANE]:lane});
return rows.flatMap(({lane,shard,total})=>{
const steps=readJobSteps(workflow,job,{...substitutions,[MATRIX_LANE]:lane,'${{ matrix.shard }}':String(shard),'${{ matrix.total }}':String(total)});
if(!steps.length||steps.some(step=>step.env[supported.laneEnv]!==lane))
throw new Error('Local CI needs executable commands for every matrix lane without lane overrides.');
return steps.map(step=>({...step,name:`[${lane}] ${step.name}`}));
return steps.map(step=>({...step,name:`[${lane}${shard?` ${shard}/${total}`:''}] ${step.name}`}));
});
}
if(supported)throw new Error('Local CI needs the maintained lane matrix, not a single replacement job.');
Expand All @@ -111,8 +115,9 @@ function readJobSteps(workflow:Record<string,unknown>,job:Record<string,unknown>
// failed prerequisite to report and no parallel job to cancel.
if(step.if!==undefined&&CI_ONLY_CONDITIONS.includes(String(step.if).trim()))return [];
const cacheGated=step.if!==undefined&&CACHE_HIT_CONDITION.test(String(step.if).trim());
const laneSelected=step.if==="env.CI_LANE_SKIP != 'true'"&&inherited.CI_LANE_SKIP==='false';
const alwaysRun=step.if!==undefined&&ALWAYS_RUN_CONDITION.test(String(step.if).trim());
if((step.if!==undefined&&!cacheGated&&!alwaysRun)||step['working-directory']!==undefined||step['continue-on-error']!==undefined||step.shell!==undefined)
if((step.if!==undefined&&!cacheGated&&!alwaysRun&&!laneSelected)||step['working-directory']!==undefined||step['continue-on-error']!==undefined||step.shell!==undefined)
throw new Error('Local CI needs explicit support for this step execution policy.');
const result={name:requireString(step.name),run:requireString(step.run),env:{...inherited,...decodeEnvironment(step.env)}};
if(JSON.stringify(result).includes('${{'))throw new Error('Local CI cannot evaluate GitHub expressions.');
Expand Down
Loading
Loading