From 96a6e69f3713bc917cc5ebd0e16cb4d52b62137f Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 24 Jul 2026 22:40:00 -0700 Subject: [PATCH] fix(software): prevent partial command package releases - Clear stale command staging when canonical outputs are unavailable. - Reject incomplete or non-executable command artifacts during build and publish. - Prepare @agentos-software/grep 0.3.5-rc.1. --- packages/agentos-toolchain/src/aospkg.ts | 138 ++++++++++++++++-- packages/agentos-toolchain/src/build.ts | 28 +++- packages/agentos-toolchain/src/index.ts | 36 +++-- packages/agentos-toolchain/src/manifest.ts | 22 +++ packages/agentos-toolchain/src/publish.ts | 28 +++- packages/agentos-toolchain/src/stage.ts | 12 +- .../agentos-toolchain/tests/aospkg.test.ts | 45 +++++- .../agentos-toolchain/tests/lifecycle.test.ts | 53 ++++++- software/grep/package.json | 2 +- 9 files changed, 310 insertions(+), 54 deletions(-) diff --git a/packages/agentos-toolchain/src/aospkg.ts b/packages/agentos-toolchain/src/aospkg.ts index ca7deaed8a..ad115dbc06 100644 --- a/packages/agentos-toolchain/src/aospkg.ts +++ b/packages/agentos-toolchain/src/aospkg.ts @@ -16,16 +16,17 @@ import { readFileSync, writeFileSync } from "node:fs"; import { + type AgentBlock, + type CommandTarget, + decodeMountIndex, decodePackageManifest, encodeMountIndex, encodePackageManifest, - TarEntryKind, - type AgentBlock, - type CommandTarget, type ManPage, type PackageManifest, type ProvidesBlock, type TarEntry, + TarEntryKind, } from "./generated-package-format.js"; const AOSPKG_MAGIC = Uint8Array.from([0x89, 0x41, 0x4f, 0x53]); // 0x89 'A' 'O' 'S' @@ -53,24 +54,119 @@ export interface AospkgSummary { export type DecodedAospkgManifest = PackageManifest; -/** Decode the chunk1 manifest from a package container. */ -export function decodeAospkgManifest(source: Uint8Array): DecodedAospkgManifest { - const bytes = Buffer.from(source.buffer, source.byteOffset, source.byteLength); - if (bytes.length < 16 || !bytes.subarray(0, 4).equals(Buffer.from(AOSPKG_MAGIC))) { +function aospkgManifestChunk(source: Uint8Array): { + bytes: Buffer; + manifestEnd: number; + manifestPayload: Buffer; +} { + const bytes = Buffer.from( + source.buffer, + source.byteOffset, + source.byteLength, + ); + if ( + bytes.length < 16 || + !bytes.subarray(0, 4).equals(Buffer.from(AOSPKG_MAGIC)) + ) { throw new Error("invalid .aospkg header"); } if (bytes.readUInt16LE(4) !== AOSPKG_FORMAT_VERSION) { - throw new Error(`unsupported .aospkg format version: ${bytes.readUInt16LE(4)}`); + throw new Error( + `unsupported .aospkg format version: ${bytes.readUInt16LE(4)}`, + ); } const manifestLength = bytes.readUInt32LE(8); - const manifestEnd = 16 + manifestLength; + const manifestStart = 16; + const manifestEnd = manifestStart + manifestLength; if (manifestLength < 2 || manifestEnd > bytes.length) { throw new Error("invalid .aospkg manifest range"); } - const version = bytes.readUInt16LE(16); - const payload = bytes.subarray(18, manifestEnd); - if (version === PACKAGE_MANIFEST_VERSION) return decodePackageManifest(payload); - throw new Error(`unsupported package manifest version: ${version}`); + if (bytes.readUInt16LE(manifestStart) !== PACKAGE_MANIFEST_VERSION) { + throw new Error( + `unsupported package manifest version: ${bytes.readUInt16LE(manifestStart)}`, + ); + } + return { + bytes, + manifestEnd, + manifestPayload: bytes.subarray(manifestStart + 2, manifestEnd), + }; +} + +function aospkgChunks(source: Uint8Array): { + manifestPayload: Buffer; + indexPayload: Buffer; +} { + const { bytes, manifestEnd, manifestPayload } = aospkgManifestChunk(source); + const indexLength = bytes.readUInt32LE(12); + const indexEnd = manifestEnd + indexLength; + if (indexLength < 2 || indexEnd > bytes.length) { + throw new Error("invalid .aospkg mount index range"); + } + if (bytes.readUInt16LE(manifestEnd) !== PACKAGE_MANIFEST_VERSION) { + throw new Error( + `unsupported mount index version: ${bytes.readUInt16LE(manifestEnd)}`, + ); + } + return { + manifestPayload, + indexPayload: bytes.subarray(manifestEnd + 2, indexEnd), + }; +} + +/** Decode the chunk1 manifest from a package container. */ +export function decodeAospkgManifest( + source: Uint8Array, +): DecodedAospkgManifest { + return decodePackageManifest(aospkgManifestChunk(source).manifestPayload); +} + +/** + * Assert that a packed command package contains its complete declared command + * surface and that every projected target carries an executable mode bit. + */ +export function verifyAospkgCommands( + source: Uint8Array, + expectedCommands: readonly string[], +): DecodedAospkgManifest { + const chunks = aospkgChunks(source); + const manifest = decodePackageManifest(chunks.manifestPayload); + const expected = [...expectedCommands].sort(byteCompare); + const actual = manifest.commands + .map((target) => target.command) + .sort(byteCompare); + if ( + expected.length !== actual.length || + expected.some((command, index) => command !== actual[index]) + ) { + throw new Error( + `packed command set does not match declarations: expected=${expected.join(",")} ` + + `actual=${actual.join(",")}`, + ); + } + + const entries = new Map( + decodeMountIndex(chunks.indexPayload).tarEntries.map((entry) => [ + entry.path, + entry, + ]), + ); + for (const target of manifest.commands) { + const path = `/${target.entry.replace(/^\/+/, "")}`; + const entry = entries.get(path); + if (entry === undefined) { + throw new Error( + `packed command ${target.command} targets missing entry ${path}`, + ); + } + if (entry.kind === TarEntryKind.Directory || (entry.mode & 0o111) === 0) { + throw new Error( + `packed command ${target.command} targets non-executable entry ${path} ` + + `(mode ${(entry.mode & 0o7777).toString(8)})`, + ); + } + } + return manifest; } interface SourceManifestJson { @@ -106,7 +202,10 @@ interface RawTarMember { /** Pack `sourceTar` into a `.aospkg` at `dest`. The source * `agentos-package.json` must carry `name` and `version`. */ -export function packAospkgFromTar(sourceTar: string, dest: string): AospkgSummary { +export function packAospkgFromTar( + sourceTar: string, + dest: string, +): AospkgSummary { const source = readFileSync(sourceTar); const { bytes, summary } = packAospkgFromTarBytes(source); writeFileSync(dest, bytes); @@ -288,7 +387,11 @@ function indexEntry(member: RawTarMember): TarEntry | undefined { linkTarget: member.linkTarget, }; } - if (member.typeflag === "0" || member.typeflag === "\0" || member.typeflag === "7") { + if ( + member.typeflag === "0" || + member.typeflag === "\0" || + member.typeflag === "7" + ) { return { ...base, kind: TarEntryKind.File, @@ -387,7 +490,10 @@ function manPagesFromIndex(sortedPaths: string[]): ManPage[] { if (parts.length !== 2) return []; return [{ section: parts[0], page: parts[1] }]; }) - .sort((a, b) => byteCompare(a.section, b.section) || byteCompare(a.page, b.page)); + .sort( + (a, b) => + byteCompare(a.section, b.section) || byteCompare(a.page, b.page), + ); } function isProjectableCommandName(name: string): boolean { diff --git a/packages/agentos-toolchain/src/build.ts b/packages/agentos-toolchain/src/build.ts index 5727353a2f..78518347f6 100644 --- a/packages/agentos-toolchain/src/build.ts +++ b/packages/agentos-toolchain/src/build.ts @@ -1,20 +1,22 @@ -import { - execFileSync, -} from "node:child_process"; +import { execFileSync } from "node:child_process"; import { chmodSync, cpSync, existsSync, mkdirSync, - readFileSync, readdirSync, + readFileSync, rmSync, statSync, writeFileSync, } from "node:fs"; import { join, resolve } from "node:path"; -import { packAospkgFromTar } from "./aospkg.js"; -import { readManifest, unscopedName } from "./manifest.js"; +import { packAospkgFromTar, verifyAospkgCommands } from "./aospkg.js"; +import { + declaredCommandNames, + readManifest, + unscopedName, +} from "./manifest.js"; export interface BuildResult { name: string; @@ -106,6 +108,17 @@ export function build(packageDirInput?: string): BuildResult { .map((entry) => entry.name) .sort() : []; + const declaredCommands = declaredCommandNames(srcManifest); + if ( + commands.length > 0 && + (commands.length !== declaredCommands.length || + commands.some((command, index) => command !== declaredCommands[index])) + ) { + throw new Error( + `refusing partial command package ${name}: declared=${declaredCommands.join(",")} ` + + `staged=${commands.join(",")}`, + ); + } const outDir = join(packageDir, "dist", "package"); rmSync(outDir, { recursive: true, force: true }); @@ -149,6 +162,9 @@ export function build(packageDirInput?: string): BuildResult { const outAospkg = join(packageDir, "dist", "package.aospkg"); rmSync(outAospkg, { force: true }); packAospkgFromTar(outTar, outAospkg); + if (commands.length > 0) { + verifyAospkgCommands(readFileSync(outAospkg), declaredCommands); + } process.stdout.write( commands.length > 0 diff --git a/packages/agentos-toolchain/src/index.ts b/packages/agentos-toolchain/src/index.ts index b229a94750..be4436d6f3 100644 --- a/packages/agentos-toolchain/src/index.ts +++ b/packages/agentos-toolchain/src/index.ts @@ -1,21 +1,33 @@ -export { pack, verifyPackageDir, type PackOptions, type PackResult } from "./pack.js"; +export { + type AospkgSummary, + type DecodedAospkgManifest, + decodeAospkgManifest, + packAospkgFromTar, + packAospkgFromTarBytes, + verifyAospkgCommands, +} from "./aospkg.js"; +export { type BuildResult, build } from "./build.js"; export { detectExecutableKind, + type ExecutableKind, isNativeKind, parseShebangInterpreter, - type ExecutableKind, } from "./header.js"; -export { stage, type StageOptions, type StageResult } from "./stage.js"; -export { build, type BuildResult } from "./build.js"; export { - publish, - resolveTag, + type AgentosPackageManifest, + declaredCommandNames, + readManifest, +} from "./manifest.js"; +export { + type PackOptions, + type PackResult, + pack, + verifyPackageDir, +} from "./pack.js"; +export { type PublishOptions, type PublishResult, + publish, + resolveTag, } from "./publish.js"; -export { readManifest, type AgentosPackageManifest } from "./manifest.js"; -export { - packAospkgFromTar, - packAospkgFromTarBytes, - type AospkgSummary, -} from "./aospkg.js"; +export { type StageOptions, type StageResult, stage } from "./stage.js"; diff --git a/packages/agentos-toolchain/src/manifest.ts b/packages/agentos-toolchain/src/manifest.ts index a8b417c274..794158e914 100644 --- a/packages/agentos-toolchain/src/manifest.ts +++ b/packages/agentos-toolchain/src/manifest.ts @@ -21,6 +21,28 @@ export interface AgentosPackageManifest { stubs?: string[]; } +/** Flat command names a built package must project, in deterministic order. */ +export function declaredCommandNames( + manifest: AgentosPackageManifest | undefined, +): string[] { + const names = [ + ...(manifest?.commands ?? []), + ...Object.keys(manifest?.aliases ?? {}), + ...(manifest?.stubs ?? []), + ]; + const seen = new Set(); + for (const name of names) { + if (typeof name !== "string" || name.length === 0 || name.includes("/")) { + throw new Error(`invalid declared command name: ${JSON.stringify(name)}`); + } + if (seen.has(name)) { + throw new Error(`command is declared more than once: ${name}`); + } + seen.add(name); + } + return [...seen].sort(); +} + export function readManifest( packageDir: string, ): AgentosPackageManifest | undefined { diff --git a/packages/agentos-toolchain/src/publish.ts b/packages/agentos-toolchain/src/publish.ts index 7537dc760b..d397625917 100644 --- a/packages/agentos-toolchain/src/publish.ts +++ b/packages/agentos-toolchain/src/publish.ts @@ -1,6 +1,8 @@ import { spawnSync } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; +import { verifyAospkgCommands } from "./aospkg.js"; +import { declaredCommandNames, readManifest } from "./manifest.js"; export interface PublishOptions { packageDir: string; @@ -24,7 +26,9 @@ export interface PublishResult { * tag must keep resolving a deliberate release, never whatever was published * last from a dev machine or CI branch. */ -export function resolveTag(options: Pick): string { +export function resolveTag( + options: Pick, +): string { if (options.latest) { if (options.tag !== undefined && options.tag !== "latest") { throw new Error( @@ -35,7 +39,7 @@ export function resolveTag(options: Pick): str } if (options.tag === "latest") { throw new Error( - 'refusing implicit `--tag latest` — pass --latest to move the latest pointer', + "refusing implicit `--tag latest` — pass --latest to move the latest pointer", ); } return options.tag ?? "dev"; @@ -92,6 +96,22 @@ export function publish(options: PublishOptions): PublishResult { `${pkg.name} is not built (no dist/index.js in ${packageDir}) — build it first`, ); } + const declaredCommands = declaredCommandNames(readManifest(packageDir)); + if (declaredCommands.length > 0) { + const packagePath = join(packageDir, "dist", "package.aospkg"); + if (!existsSync(packagePath)) { + throw new Error( + `${pkg.name} declares commands but has no built dist/package.aospkg`, + ); + } + try { + verifyAospkgCommands(readFileSync(packagePath), declaredCommands); + } catch (error) { + throw new Error( + `refusing to publish invalid command artifact for ${pkg.name}: ${String(error)}`, + ); + } + } const inPnpmWorkspace = findUp(packageDir, "pnpm-workspace.yaml") !== undefined; @@ -106,7 +126,9 @@ export function publish(options: PublishOptions): PublishResult { const result = spawnSync(pm, args, { cwd: packageDir, stdio: "inherit" }); if (result.error) throw result.error; if (result.status !== 0) { - throw new Error(`${pm} publish failed for ${pkg.name} (exit ${result.status})`); + throw new Error( + `${pm} publish failed for ${pkg.name} (exit ${result.status})`, + ); } return { name: pkg.name, version: pkg.version, tag }; } diff --git a/packages/agentos-toolchain/src/stage.ts b/packages/agentos-toolchain/src/stage.ts index 92725f1874..c9a578e449 100644 --- a/packages/agentos-toolchain/src/stage.ts +++ b/packages/agentos-toolchain/src/stage.ts @@ -45,6 +45,7 @@ export function stage(options: StageOptions): StageResult { const commands = manifest?.commands ?? []; const aliases = manifest?.aliases ?? {}; const stubs = manifest?.stubs ?? []; + const binDir = join(packageDir, "bin"); if ( commands.length === 0 && stubs.length === 0 && @@ -58,15 +59,18 @@ export function stage(options: StageOptions): StageResult { if (!existsSync(commandsDir)) { if (ifMissing === "skip") { + rmSync(binDir, { recursive: true, force: true }); process.stdout.write( `stage: commands dir not found (${commandsDir}) — leaving bin/ unstaged (placeholder package)\n`, ); - return { staged: [], missing: [...commands, ...stubs, ...Object.keys(aliases)] }; + return { + staged: [], + missing: [...commands, ...stubs, ...Object.keys(aliases)], + }; } throw new Error(`stage: commands dir not found: ${commandsDir}`); } - const binDir = join(packageDir, "bin"); rmSync(binDir, { recursive: true, force: true }); mkdirSync(binDir, { recursive: true }); @@ -116,8 +120,6 @@ export function stage(options: StageOptions): StageResult { } process.stdout.write(`stage: WARN ${detail}\n`); } - process.stdout.write( - `staged ${staged.length} command(s) into ${binDir}\n`, - ); + process.stdout.write(`staged ${staged.length} command(s) into ${binDir}\n`); return { staged, missing }; } diff --git a/packages/agentos-toolchain/tests/aospkg.test.ts b/packages/agentos-toolchain/tests/aospkg.test.ts index 5bfd4d4242..46bc30d6cc 100644 --- a/packages/agentos-toolchain/tests/aospkg.test.ts +++ b/packages/agentos-toolchain/tests/aospkg.test.ts @@ -1,22 +1,34 @@ import { execFileSync } from "node:child_process"; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; -import { decodeAospkgManifest, packAospkgFromTarBytes } from "../src/aospkg.js"; +import { + decodeAospkgManifest, + packAospkgFromTarBytes, + verifyAospkgCommands, +} from "../src/aospkg.js"; const dirs: string[] = []; afterEach(() => { - for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); }); -function sourceTar(): Buffer { +function sourceTar(mode = 0o755): Buffer { const dir = mkdtempSync(join(tmpdir(), "agentos-aospkg-runtime-")); dirs.push(dir); mkdirSync(join(dir, "bin")); writeFileSync(join(dir, "bin", "agent"), "#!/usr/bin/env node\n"); - chmodSync(join(dir, "bin", "agent"), 0o755); + chmodSync(join(dir, "bin", "agent"), mode); writeFileSync( join(dir, "agentos-package.json"), JSON.stringify({ @@ -39,4 +51,27 @@ describe("package manifest", () => { expect(bytes.readUInt16LE(16)).toBe(1); expect(manifest.agent?.acpEntrypoint).toBe("agent"); }); + + test("decodes the manifest without requiring later container chunks", () => { + const { bytes } = packAospkgFromTarBytes(sourceTar()); + const manifestEnd = 16 + bytes.readUInt32LE(8); + expect( + decodeAospkgManifest(bytes.subarray(0, manifestEnd)).agent?.acpEntrypoint, + ).toBe("agent"); + }); + + test("verifies the complete executable command contract", () => { + const { bytes } = packAospkgFromTarBytes(sourceTar()); + expect(verifyAospkgCommands(bytes, ["agent"]).commands).toHaveLength(1); + expect(() => verifyAospkgCommands(bytes, ["agent", "missing"])).toThrow( + /expected=agent,missing actual=agent/, + ); + }); + + test("rejects packed command targets without an executable mode bit", () => { + const { bytes } = packAospkgFromTarBytes(sourceTar(0o644)); + expect(() => verifyAospkgCommands(bytes, ["agent"])).toThrow( + /non-executable entry \/bin\/agent/, + ); + }); }); diff --git a/packages/agentos-toolchain/tests/lifecycle.test.ts b/packages/agentos-toolchain/tests/lifecycle.test.ts index 2f96e1496e..6f1a7e6a1c 100644 --- a/packages/agentos-toolchain/tests/lifecycle.test.ts +++ b/packages/agentos-toolchain/tests/lifecycle.test.ts @@ -14,7 +14,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; import { build } from "../src/build.js"; -import { resolveTag } from "../src/publish.js"; +import { publish, resolveTag } from "../src/publish.js"; import { stage } from "../src/stage.js"; const dirs: string[] = []; @@ -61,8 +61,12 @@ describe("stage", () => { ["bash", "cat", "df", "id", "linked-sh", "more", "sh"].sort(), ); // Symlink sources are dereferenced into real files. - expect(lstatSync(join(pkg, "bin", "linked-sh")).isSymbolicLink()).toBe(false); - expect(readFileSync(join(pkg, "bin", "linked-sh"), "utf8")).toBe("\0asm-sh"); + expect(lstatSync(join(pkg, "bin", "linked-sh")).isSymbolicLink()).toBe( + false, + ); + expect(readFileSync(join(pkg, "bin", "linked-sh"), "utf8")).toBe( + "\0asm-sh", + ); expect(readFileSync(join(pkg, "bin", "bash"), "utf8")).toBe("\0asm-sh"); expect(readFileSync(join(pkg, "bin", "id"), "utf8")).toBe("\0asm-stubs"); for (const command of result.staged) { @@ -97,6 +101,8 @@ describe("stage", () => { test("missing commands dir with if-missing=skip leaves a placeholder", () => { const pkg = makePackageDir({ commands: ["sh"] }); + mkdirSync(join(pkg, "bin")); + writeFileSync(join(pkg, "bin", "stale"), "old"); const result = stage({ packageDir: pkg, commandsDir: join(pkg, "does-not-exist"), @@ -145,9 +151,9 @@ describe("build", () => { ); // Staging fields are build-time only — they must not ship at runtime. expect(runtimeManifest).toEqual({ name: "fake", version: "1.2.3" }); - expect(readFileSync(join(pkg, "dist", "package", "bin", "bash"), "utf8")).toBe( - "\0asm-sh", - ); + expect( + readFileSync(join(pkg, "dist", "package", "bin", "bash"), "utf8"), + ).toBe("\0asm-sh"); for (const command of result.commands) { expect( statSync(join(pkg, "dist", "package", "bin", command)).mode & 0o777, @@ -162,6 +168,15 @@ describe("build", () => { expect(existsSync(result.outTar)).toBe(true); expect(existsSync(join(pkg, "dist", "package", "bin"))).toBe(false); }); + + test("refuses to assemble a partially staged command package", () => { + const commandsDir = makeCommandsDir(); + const pkg = makePackageDir({ commands: ["sh", "missing"] }); + stage({ packageDir: pkg, commandsDir, ifMissing: "skip" }); + expect(() => build(pkg)).toThrow( + /refusing partial command package.*declared=missing,sh staged=sh/, + ); + }); }); describe("resolveTag", () => { @@ -175,3 +190,29 @@ describe("resolveTag", () => { expect(() => resolveTag({ latest: true, tag: "dev" })).toThrow(/conflicts/); }); }); + +describe("publish", () => { + test("refuses a command package without a packed runtime artifact", () => { + const pkg = makePackageDir({ commands: ["sh"] }); + mkdirSync(join(pkg, "dist")); + writeFileSync(join(pkg, "dist", "index.js"), "export default {};\n"); + expect(() => publish({ packageDir: pkg, dryRun: true })).toThrow( + /declares commands but has no built dist\/package\.aospkg/, + ); + }); + + test("refuses a packed artifact missing a newly declared command", () => { + const commandsDir = makeCommandsDir(); + const pkg = makePackageDir({ commands: ["sh"] }); + stage({ packageDir: pkg, commandsDir }); + build(pkg); + writeFileSync(join(pkg, "dist", "index.js"), "export default {};\n"); + writeFileSync( + join(pkg, "agentos-package.json"), + JSON.stringify({ commands: ["sh", "cat"] }), + ); + expect(() => publish({ packageDir: pkg, dryRun: true })).toThrow( + /expected=cat,sh actual=sh/, + ); + }); +}); diff --git a/software/grep/package.json b/software/grep/package.json index a8add5e5d4..a47a76ec0a 100644 --- a/software/grep/package.json +++ b/software/grep/package.json @@ -1,6 +1,6 @@ { "name": "@agentos-software/grep", - "version": "0.3.4", + "version": "0.3.5-rc.1", "type": "module", "license": "Apache-2.0", "description": "GNU grep pattern matching for agentos VMs (grep, egrep, fgrep)",