diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index cc5835e081..8c2d45d605 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -150,12 +150,20 @@ jobs: restore-keys: | ${{ runner.os }}-turbo-${{ hashFiles('**/pnpm-lock.yaml') }}- ${{ runner.os }}-turbo- - - name: Run non-core coverage - run: pnpm turbo run test:coverage --filter="!@roo-code/core" --log-order grouped --output-logs new-only + - name: Run non-extension package coverage + run: pnpm turbo run test:coverage --filter="!@roo-code/core" --filter="!zoo-code" --log-order grouped --output-logs new-only + - name: Run extension unit coverage + run: pnpm turbo run test:coverage:unit --filter="zoo-code" --log-order grouped --output-logs new-only + - name: Verify extension coverage contract + run: pnpm --dir src run verify:coverage-contract + - name: Run extension dist smoke test + run: pnpm turbo run test:dist --filter="zoo-code" --log-order grouped --output-logs new-only - name: Run core unit coverage run: pnpm turbo run test:coverage:unit --filter="@roo-code/core" --log-order grouped --output-logs new-only - name: Run core integration coverage run: pnpm turbo run test:coverage:integration --filter="@roo-code/core" --log-order grouped --output-logs new-only + - name: Verify extension unit coverage report + run: node src/scripts/verify-lcov.mjs src/coverage/unit/lcov.info - name: Save Turbo cache if: steps.turbo-cache.outputs.cache-hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -177,7 +185,7 @@ jobs: uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: files: >- - src/coverage/lcov.info, + src/coverage/unit/lcov.info, packages/cloud/coverage/lcov.info, packages/telemetry/coverage/lcov.info, apps/cli/coverage/lcov.info @@ -214,7 +222,7 @@ jobs: with: name: coverage-reports-${{ matrix.name }} path: | - src/coverage/lcov.info + src/coverage/unit/lcov.info webview-ui/coverage/lcov.info packages/cloud/coverage/lcov.info packages/telemetry/coverage/lcov.info diff --git a/src/package.json b/src/package.json index 7467b50b7b..3e873ebbf7 100644 --- a/src/package.json +++ b/src/package.json @@ -442,6 +442,7 @@ "lint": "eslint . --ext=ts --max-warnings=0", "check-types": "tsc --noEmit", "test": "vitest run", + "verify:coverage-contract": "node scripts/verify-coverage-contract.mjs", "test:unit": "vitest run --config vitest.unit.config.ts", "test:dist": "vitest run --config vitest.dist.config.ts", "test:coverage": "vitest run --coverage", diff --git a/src/scripts/verify-coverage-contract.mjs b/src/scripts/verify-coverage-contract.mjs new file mode 100644 index 0000000000..1f6003dff1 --- /dev/null +++ b/src/scripts/verify-coverage-contract.mjs @@ -0,0 +1,29 @@ +import { spawnSync } from "node:child_process" +import process from "node:process" + +const pnpm = process.platform === "win32" ? process.env.npm_execpath : "pnpm" +if (!pnpm) throw new Error("pnpm executable path is unavailable") +const command = process.platform === "win32" ? process.execPath : pnpm +const args = process.platform === "win32" ? [pnpm] : [] +const result = spawnSync( + command, + [...args, "turbo", "run", "test:coverage:unit", "test:dist", "--filter=zoo-code", "--dry=json"], + { encoding: "utf8" }, +) +if (result.status !== 0) { + const details = [result.error?.message, result.signal, result.stderr, result.stdout].filter(Boolean).join("\n") + throw new Error(details || `pnpm exited with status ${result.status ?? "unknown"}`) +} + +const graph = JSON.parse(result.stdout) +const coverageTask = graph.tasks.find(({ taskId }) => taskId === "zoo-code#test:coverage:unit") +const distTask = graph.tasks.find(({ taskId }) => taskId === "zoo-code#test:dist") +if (!coverageTask) throw new Error("Unit coverage task missing") +if (graph.tasks.some(({ taskId }) => taskId === "zoo-code#prepare:tree-sitter-wasms")) + throw new Error("Removed WASM preparation task remains in the graph") +if (coverageTask.dependencies.includes("zoo-code#bundle")) throw new Error("Unit coverage must not depend on bundle") +if (!coverageTask.dependencies.includes("@roo-code/types#build")) + throw new Error("Unit coverage must depend on the types build") +if (!Object.hasOwn(coverageTask.inputs, "package.json")) throw new Error("Unit coverage must hash package.json") +if (!coverageTask.hashOfExternalDependencies) throw new Error("Unit coverage must hash external dependencies") +if (!distTask?.dependencies.includes("zoo-code#bundle")) throw new Error("Dist smoke test must depend on bundle") diff --git a/src/scripts/verify-lcov.mjs b/src/scripts/verify-lcov.mjs new file mode 100644 index 0000000000..c4220dbf00 --- /dev/null +++ b/src/scripts/verify-lcov.mjs @@ -0,0 +1,46 @@ +import fs from "node:fs" +import process from "node:process" +import { fileURLToPath } from "node:url" + +export function verifyLcov(content) { + let inRecord = false + let anyCovered = false + let linesFound + let linesHit + + for (const line of content.split(/\r?\n/)) { + if (line.startsWith("SF:")) { + if (inRecord) throw new Error("LCOV source record is not terminated") + if (!line.slice(3)) throw new Error("LCOV source path is empty") + inRecord = true + linesFound = undefined + linesHit = undefined + } else if (line.startsWith("LF:")) { + if (!inRecord) throw new Error("LCOV line count is outside a source record") + if (linesFound !== undefined) throw new Error("LCOV source record has duplicate line counts") + const found = line.slice(3) + if (!/^\d+$/.test(found)) throw new Error("LCOV line count is not a decimal integer") + linesFound = BigInt(found) + } else if (line.startsWith("LH:")) { + if (!inRecord) throw new Error("LCOV hit count is outside a source record") + if (linesHit !== undefined) throw new Error("LCOV source record has duplicate hit counts") + const hits = line.slice(3) + if (!/^\d+$/.test(hits)) throw new Error("LCOV hit count is not a decimal integer") + linesHit = BigInt(hits) + } else if (line === "end_of_record") { + if (!inRecord) throw new Error("LCOV terminator is outside a source record") + if (linesFound === undefined || linesHit === undefined) + throw new Error("LCOV source record has incomplete line summaries") + if (linesHit > linesFound) throw new Error("LCOV hit count exceeds lines found") + if (linesHit > 0n) anyCovered = true + inRecord = false + } + } + + if (inRecord) throw new Error("LCOV source record is not terminated") + if (!anyCovered) throw new Error("LCOV report has no covered lines") +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + verifyLcov(fs.readFileSync(process.argv[2], "utf8")) +} diff --git a/src/scripts/verify-lcov.spec.mjs b/src/scripts/verify-lcov.spec.mjs new file mode 100644 index 0000000000..849c722979 --- /dev/null +++ b/src/scripts/verify-lcov.spec.mjs @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest" + +import { verifyLcov } from "./verify-lcov.mjs" + +describe("verifyLcov", () => { + it("accepts complete records with covered lines", () => { + expect(() => verifyLcov("SF:file.ts\nLF:1\nLH:1\nend_of_record\n")).not.toThrow() + }) + + it.each([ + ["an empty source path", "SF:\nLF:1\nLH:1\nend_of_record\n"], + ["an unterminated record", "SF:file.ts\nLH:1\n"], + ["a zero-hit report", "SF:file.ts\nLF:1\nLH:0\nend_of_record\n"], + ["a line count outside a record", "LF:1\n"], + ["a hit count outside a record", "LH:1\n"], + ["a terminator outside a record", "end_of_record\n"], + ["consecutive source records", "SF:first.ts\nSF:second.ts\nLF:1\nLH:1\nend_of_record\n"], + ["a record without lines found", "SF:file.ts\nLH:1\nend_of_record\n"], + ["an infinite line count", "SF:file.ts\nLF:Infinity\nLH:1\nend_of_record\n"], + ["a fractional line count", "SF:file.ts\nLF:1.5\nLH:1\nend_of_record\n"], + ["an exponential line count", "SF:file.ts\nLF:1e3\nLH:1\nend_of_record\n"], + ["an infinite hit count", "SF:file.ts\nLH:Infinity\nend_of_record\n"], + ["a fractional hit count", "SF:file.ts\nLH:1.5\nend_of_record\n"], + ["an exponential hit count", "SF:file.ts\nLH:1e3\nend_of_record\n"], + ["duplicate line counts", "SF:file.ts\nLF:1\nLF:0\nLH:0\nend_of_record\n"], + ["duplicate hit counts", "SF:file.ts\nLF:1\nLH:1\nLH:0\nend_of_record\n"], + ["more hit lines than found lines", "SF:file.ts\nLF:0\nLH:1\nend_of_record\n"], + ])("rejects %s", (_, content) => { + expect(() => verifyLcov(content)).toThrow() + }) +}) diff --git a/src/services/tree-sitter/__tests__/helpers.ts b/src/services/tree-sitter/__tests__/helpers.ts index 3f9f4c247c..904137aaf7 100644 --- a/src/services/tree-sitter/__tests__/helpers.ts +++ b/src/services/tree-sitter/__tests__/helpers.ts @@ -4,6 +4,8 @@ import * as path from "path" import tsxQuery from "../queries/tsx" import { Parser, Language } from "web-tree-sitter" +import { loadTestGrammar } from "./wasm" + vi.mock("fs/promises") export const mockedFs = vi.mocked(fs) @@ -34,16 +36,6 @@ export async function initializeTreeSitter() { // Initialize directly using the default export or the module itself await Parser.init() - // Override the Parser.Language.load to use dist directory - const originalLoad = Language.load - - Language.load = async (wasmPath: string) => { - const filename = path.basename(wasmPath) - const correctPath = path.join(process.cwd(), "dist", filename) - // console.log(`Redirecting WASM load from ${wasmPath} to ${correctPath}`) - return originalLoad(correctPath) - } - initializedTreeSitter = { Parser, Language } } @@ -84,8 +76,7 @@ export async function testParseSourceCodeDefinitions( const parser = new Parser() // Load language and configure parser - const wasmPath = path.join(process.cwd(), `dist/${wasmFile}`) - const lang = await Language.load(wasmPath) + const lang = await loadTestGrammar(path.basename(wasmFile)) parser.setLanguage(lang) // Create a real query @@ -113,8 +104,7 @@ export async function testParseSourceCodeDefinitions( export async function inspectTreeStructure(content: string, language: string = "typescript"): Promise { const { Parser, Language } = await initializeTreeSitter() const parser = new Parser() - const wasmPath = path.join(process.cwd(), `dist/tree-sitter-${language}.wasm`) - const lang = await Language.load(wasmPath) + const lang = await loadTestGrammar(`tree-sitter-${language}.wasm`) parser.setLanguage(lang) // Parse the content diff --git a/src/services/tree-sitter/__tests__/wasm.spec.ts b/src/services/tree-sitter/__tests__/wasm.spec.ts new file mode 100644 index 0000000000..c3114ff164 --- /dev/null +++ b/src/services/tree-sitter/__tests__/wasm.spec.ts @@ -0,0 +1,87 @@ +import fs from "fs" +import os from "os" +import path from "path" +import { afterEach, beforeAll, describe, expect, it } from "vitest" +import { Parser } from "web-tree-sitter" + +import { loadTestGrammar } from "./wasm" + +const requiredGrammars = [ + "c", + "cpp", + "c_sharp", + "css", + "dart", + "elisp", + "elixir", + "embedded_template", + "go", + "html", + "java", + "javascript", + "json", + "kotlin", + "lua", + "ocaml", + "php", + "python", + "ruby", + "rust", + "scala", + "solidity", + "swift", + "systemrdl", + "tlaplus", + "toml", + "tsx", + "typescript", + "vue", + "zig", +] + +async function captureLoadFailure(filename: string, directory: string) { + let error: unknown + try { + await loadTestGrammar(filename, directory) + } catch (caught) { + error = caught + } + if (!(error instanceof Error) || !(error.cause instanceof Error)) { + throw new Error("Expected a contextual grammar load error with an Error cause") + } + expect(error.message).toContain(error.cause.message) + return error +} + +describe("dependency-owned Tree-sitter grammars", () => { + const temporaryDirectories: string[] = [] + + beforeAll(() => Parser.init()) + afterEach(() => temporaryDirectories.splice(0).forEach((directory) => fs.rmSync(directory, { recursive: true }))) + + it.each(requiredGrammars)("loads tree-sitter-%s.wasm", async (grammar) => { + const language = await loadTestGrammar(`tree-sitter-${grammar}.wasm`) + expect(() => new Parser().setLanguage(language)).not.toThrow() + }) + + it("reports a missing dependency artifact with its filename and resolved path", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tree-sitter-wasm-missing-")) + temporaryDirectories.push(directory) + + const error = await captureLoadFailure("tree-sitter-missing.wasm", directory) + expect(error.message).toContain( + `Failed to load Tree-sitter grammar tree-sitter-missing.wasm from ${path.join(directory, "tree-sitter-missing.wasm")}`, + ) + }) + + it("reports a malformed dependency artifact with its filename and resolved path", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tree-sitter-wasm-malformed-")) + temporaryDirectories.push(directory) + fs.writeFileSync(path.join(directory, "tree-sitter-malformed.wasm"), "not wasm") + + const error = await captureLoadFailure("tree-sitter-malformed.wasm", directory) + expect(error.message).toContain( + `Failed to load Tree-sitter grammar tree-sitter-malformed.wasm from ${path.join(directory, "tree-sitter-malformed.wasm")}`, + ) + }) +}) diff --git a/src/services/tree-sitter/__tests__/wasm.ts b/src/services/tree-sitter/__tests__/wasm.ts new file mode 100644 index 0000000000..c9dc4d80fc --- /dev/null +++ b/src/services/tree-sitter/__tests__/wasm.ts @@ -0,0 +1,14 @@ +import path from "path" +import { Language } from "web-tree-sitter" + +export const TEST_WASM_DIR = path.join(__dirname, "../../../node_modules/tree-sitter-wasms/out") + +export async function loadTestGrammar(filename: string, directory = TEST_WASM_DIR) { + const wasmPath = path.join(directory, filename) + try { + return await Language.load(wasmPath) + } catch (error) { + const detail = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to load Tree-sitter grammar ${filename} from ${wasmPath}: ${detail}`, { cause: error }) + } +}