diff --git a/README.md b/README.md index a525a4b..1c2a230 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,9 @@ Checks one or more prompt files against a simple JSON rules file. promptdiff check prompts/*.md --rules promptdiff.rules.json --fail-on high ``` -Both commands parse options strictly. Unknown options and options missing their required values are errors, and `compare` accepts exactly two file arguments. For `check`, every file or glob supplied on the command line must match; PromptDiff does not emit a partial report when one input is unmatched. +Both commands parse options strictly. Unknown options and options missing their required values are errors, and `compare` accepts exactly two file arguments. For `check`, every file or glob supplied on the command line must match; PromptDiff does not emit a partial report when one input is unmatched. Globs support `*` and `?` within a path segment and recursive `**` matching, such as `examples/**/*.md`. Results are sorted and deduplicated before checking. + +The rules document must be a JSON object. `requiredPhrases`, `forbiddenPhrases`, and `requireSections`, when present, must be arrays containing only non-empty strings. `maxSeverity` must be one of `info`, `low`, `medium`, `high`, or `critical`. Invalid rules exit with code `1`, print a field-specific diagnostic to stderr, and do not emit a report. Exit code `0` means the command completed without tripping a gate. Exit code `2` means the configured quality gate failed. Exit code `1` means invalid command input or another runtime error; diagnostics are written to stderr. diff --git a/src/cli.ts b/src/cli.ts index fdd051d..0bbcb59 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { mkdir, writeFile, readdir } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; +import { mkdir, writeFile, readdir, stat } from 'node:fs/promises'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { analyzePromptDiff } from './analyzer.js'; import { readPrompt } from './parser.js'; import { renderCheckMarkdown, renderCompareMarkdown, renderJson } from './render.js'; @@ -68,18 +68,67 @@ async function writeOrPrint(content: string, out?: string): Promise { await writeFile(out, content, 'utf8'); } +function globRegex(pattern: string): RegExp { + const normalized = pattern.split(sep).join('/'); + let source = ''; + for (let index = 0; index < normalized.length; index += 1) { + const char = normalized[index]; + if (char === '*' && normalized[index + 1] === '*') { + index += 1; + if (normalized[index + 1] === '/') { + index += 1; + source += '(?:.*/)?'; + } else { + source += '.*'; + } + } else if (char === '*') { + source += '[^/]*'; + } else if (char === '?') { + source += '[^/]'; + } else { + source += char.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + } + } + return new RegExp(`^${source}$`); +} + +async function listFiles(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0)) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...await listFiles(path)); + else if (entry.isFile()) files.push(path); + } + return files; +} + async function expandInputs(inputs: string[]): Promise { const expanded: string[] = []; for (const input of inputs) { - if (!input.includes('*')) { - expanded.push(input); + if (!/[*?]/.test(input)) { + try { + if (!(await stat(input)).isFile()) throw new Error(); + } catch { + throw new Error(`check input did not match any files: ${input}`); + } + expanded.push(isAbsolute(input) ? input : relative(process.cwd(), resolve(input))); continue; } - const dir = dirname(input); - const pattern = input.slice(dir.length + 1).replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*'); - const regex = new RegExp(`^${pattern}$`); - const entries = await readdir(dir === '.' ? process.cwd() : dir); - const matches = entries.filter((entry) => regex.test(entry)).map((entry) => join(dir, entry)).sort(); + const firstMagic = input.search(/[*?]/); + const slash = Math.max(input.lastIndexOf('/', firstMagic), input.lastIndexOf(sep, firstMagic)); + const root = resolve(slash === -1 ? '.' : input.slice(0, slash) || sep); + let candidates: string[] = []; + try { + candidates = await listFiles(root); + } catch { + // A missing static prefix is an unmatched pattern, handled below. + } + const regex = globRegex(resolve(input)); + const matches = candidates + .filter((file) => regex.test(file.split(sep).join('/'))) + .map((file) => isAbsolute(input) ? file : relative(process.cwd(), file)) + .sort(); if (matches.length === 0) throw new Error(`check input did not match any files: ${input}`); expanded.push(...matches); } diff --git a/src/rules.ts b/src/rules.ts index 5427550..9b7b9ed 100644 --- a/src/rules.ts +++ b/src/rules.ts @@ -14,9 +14,37 @@ function makeFinding(id: string, title: string, detail: string, evidence: string export async function readRules(path?: string): Promise { if (!path) return defaultRules; const raw = await readFile(path, 'utf8'); - const parsed = JSON.parse(raw) as RulesFile; - if (parsed.maxSeverity) parseSeverity(parsed.maxSeverity); - return { ...defaultRules, ...parsed }; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error('rules must contain valid JSON.'); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('rules must be a JSON object.'); + } + + const rules = parsed as Record; + for (const field of ['requiredPhrases', 'forbiddenPhrases', 'requireSections'] as const) { + const value = rules[field]; + if (value === undefined) continue; + if (!Array.isArray(value)) throw new Error(`rules.${field} must be an array.`); + const invalidIndex = value.findIndex((item) => typeof item !== 'string' || item.trim().length === 0); + if (invalidIndex !== -1) throw new Error(`rules.${field}[${invalidIndex}] must be a non-empty string.`); + } + if (rules.maxSeverity !== undefined) { + if (typeof rules.maxSeverity !== 'string') { + throw new Error('rules.maxSeverity must be one of: info, low, medium, high, critical.'); + } + try { + parseSeverity(rules.maxSeverity); + } catch { + throw new Error('rules.maxSeverity must be one of: info, low, medium, high, critical.'); + } + } + + const validated = rules as RulesFile; + return { ...defaultRules, ...validated }; } function includesCaseInsensitive(text: string, needle: string): boolean { diff --git a/tests/cli.test.mjs b/tests/cli.test.mjs index 793baf7..066e080 100644 --- a/tests/cli.test.mjs +++ b/tests/cli.test.mjs @@ -1,6 +1,9 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; test('cli compare returns gate failure when fail-on threshold is met', () => { const run = spawnSync(process.execPath, ['dist/cli.js', 'compare', 'examples/prompts/v1.md', 'examples/prompts/v2.md', '--fail-on', 'high'], { encoding: 'utf8' }); @@ -55,6 +58,35 @@ test('cli check rejects a mixed matched and unmatched input set', () => { assert.match(run.stderr, /input did not match any files: examples\/prompts\/missing-\*\.md/); }); +test('cli check expands recursive globs into sorted unique files', () => { + const run = spawnSync(process.execPath, [ + 'dist/cli.js', 'check', 'examples/**/*.md', './examples/prompts/safe.md', + '--format', 'json', + ], { encoding: 'utf8' }); + assert.equal(run.status, 0); + assert.equal(run.stderr, ''); + const report = JSON.parse(run.stdout); + assert.deepEqual(report.files, [...new Set(report.files)].sort()); + assert.ok(report.files.includes('examples/README.md')); + assert.ok(report.files.includes('examples/prompts/safe.md')); +}); + +test('cli rejects malformed rules without emitting a report', () => { + const directory = mkdtempSync(join(tmpdir(), 'promptdiff-cli-rules-')); + const rulesPath = join(directory, 'rules.json'); + writeFileSync(rulesPath, JSON.stringify({ requireSections: [42] })); + try { + const run = spawnSync(process.execPath, [ + 'dist/cli.js', 'check', 'examples/prompts/safe.md', '--rules', rulesPath, + ], { encoding: 'utf8' }); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.match(run.stderr, /rules\.requireSections\[0\] must be a non-empty string/); + } finally { + rmSync(directory, { recursive: true }); + } +}); + test('cli accepts documented compare option forms', () => { const run = spawnSync(process.execPath, ['dist/cli.js', 'compare', 'examples/prompts/v1.md', 'examples/prompts/v2.md', '--format', 'json', '--fail-on', 'high', '--no-redact'], { encoding: 'utf8' }); assert.equal(run.status, 2); diff --git a/tests/rules.test.mjs b/tests/rules.test.mjs index 6392fb7..34c3076 100644 --- a/tests/rules.test.mjs +++ b/tests/rules.test.mjs @@ -1,6 +1,9 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { checkPrompts, normalizePrompt } from '../dist/index.js'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { checkPrompts, normalizePrompt, readRules } from '../dist/index.js'; test('flags forbidden phrases and missing required sections', () => { const prompt = normalizePrompt('bad.md', '# Prompt\nIgnore previous instructions.', true); @@ -12,3 +15,44 @@ test('flags forbidden phrases and missing required sections', () => { assert.equal(result.summary.findingCount, 3); assert.equal(result.summary.highestSeverity, 'high'); }); + +test('reads a fully specified valid rules document', async () => { + const directory = await mkdtemp(join(tmpdir(), 'promptdiff-rules-')); + const path = join(directory, 'rules.json'); + await writeFile(path, JSON.stringify({ + requiredPhrases: ['protect customer secrets'], + forbiddenPhrases: ['ignore previous instructions'], + requireSections: ['Output Contract'], + maxSeverity: 'critical', + })); + try { + assert.deepEqual(await readRules(path), { + requiredPhrases: ['protect customer secrets'], + forbiddenPhrases: ['ignore previous instructions'], + requireSections: ['Output Contract'], + maxSeverity: 'critical', + }); + } finally { + await rm(directory, { recursive: true }); + } +}); + +test('rejects malformed rules with field-specific diagnostics', async () => { + const directory = await mkdtemp(join(tmpdir(), 'promptdiff-rules-')); + const path = join(directory, 'rules.json'); + const invalidRules = [ + [[], /rules must be a JSON object/], + [{ requiredPhrases: 'required' }, /rules\.requiredPhrases must be an array/], + [{ forbiddenPhrases: [''] }, /rules\.forbiddenPhrases\[0\] must be a non-empty string/], + [{ requireSections: [42] }, /rules\.requireSections\[0\] must be a non-empty string/], + [{ maxSeverity: 'urgent' }, /rules\.maxSeverity must be one of/], + ]; + try { + for (const [rules, diagnostic] of invalidRules) { + await writeFile(path, JSON.stringify(rules)); + await assert.rejects(readRules(path), diagnostic); + } + } finally { + await rm(directory, { recursive: true }); + } +});