From 3b37823a17e9db67ea5fba9ff4485f02f86313d0 Mon Sep 17 00:00:00 2001 From: Alexander Moskvin Date: Wed, 7 Oct 2026 10:25:53 +1100 Subject: [PATCH 1/2] feat(data): verification methods and links data model (refs #191) --- CHANGELOG.md | 11 + data/README.md | 4 + data/verification-links-schema.json | 45 ++++ data/verification-links/README.md | 9 + data/verification-methods-schema.json | 96 ++++++++ data/verification-methods.json | 5 + docs/VERIFICATION_METHODS.md | 120 ++++++++++ scripts/validate.js | 17 ++ scripts/verification.js | 323 ++++++++++++++++++++++++++ scripts/verification.test.mjs | 271 +++++++++++++++++++++ 10 files changed, 901 insertions(+) create mode 100644 data/verification-links-schema.json create mode 100644 data/verification-links/README.md create mode 100644 data/verification-methods-schema.json create mode 100644 data/verification-methods.json create mode 100644 docs/VERIFICATION_METHODS.md create mode 100644 scripts/verification.js create mode 100644 scripts/verification.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d28e67..b960fea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,17 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). scope changes (LLM01 cross-modal 2.2.3/2.2.4, LLM04 artifact provenance 3.1.1/3.1.3, LLM05 fine-tuning subversion 6.1.2/3.5.1, LLM08 hidden context 10.2.4, LLM10 generated code 9.3.7). AISVS has no requirement for scanning generated code itself; the LLM10 section says so. +- **Verification methods data model** (#191, follow-up to #101 / #186): how to check that a control is + implemented, held as data. `data/verification-methods.json` holds each method once (`VM-NNNN` ids, + `examine` / `interview` / `test`, procedure, expected result, provenance, review state). + `data/verification-links/.json` links methods to controls, at control level or on one risk–control + row, each link with its own review. Both have schemas (`verification-methods-schema.json`, + `verification-links-schema.json`), and `scripts/validate.js` enforces them plus the cross-file rules through + `scripts/verification.js` (tested in `scripts/verification.test.mjs`). A source's original wording is optional and + included only where its licence allows; otherwise the method cites the source by id and url. Licence + compatibility is judged in review, not by the validator. v1 is data, schemas and validator only: no generator, + export or webapp changes. Documented in `docs/VERIFICATION_METHODS.md`. No methods yet; the pilot follows in a + separate PR. ### Changed diff --git a/data/README.md b/data/README.md index 2ff8634..05a21b8 100644 --- a/data/README.md +++ b/data/README.md @@ -14,6 +14,10 @@ dashboard integration, and downstream tooling without scraping Markdown. | `incidents-schema.json` | JSON Schema for incident entries | | `incidents.json` | 50 real-world + research AI security incidents with MAESTRO layer attribution | | `tools-supplement.json` | Supplemental tool entries merged into entries at generation time | +| `verification-methods.json` | Verification methods: how to check a control is implemented (see `docs/VERIFICATION_METHODS.md`) | +| `verification-methods-schema.json` | JSON Schema for verification methods | +| `verification-links/` | Links from verification methods to controls, one file per framework registry id | +| `verification-links-schema.json` | JSON Schema for a verification links file | | `entries/` | 41 machine-readable JSON files — one per OWASP entry (LLM01–LLM10, ASI01–ASI10, DSGAI01–DSGAI21) | --- diff --git a/data/verification-links-schema.json b/data/verification-links-schema.json new file mode 100644 index 0000000..e50e182 --- /dev/null +++ b/data/verification-links-schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/verification-links-schema.json", + "title": "GenAI Security Crosswalk — Verification Links Schema", + "description": "Links between verification methods and controls, one file per framework: data/verification-links/.json. See docs/VERIFICATION_METHODS.md and issue #191.", + "type": "object", + "required": ["version", "framework", "links"], + "additionalProperties": false, + "properties": { + "version": { "type": "string", "minLength": 1 }, + "framework": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$", + "description": "Registry id (data/frameworks/.json). Must equal the file name." + }, + "links": { + "type": "array", + "description": "Sorted by control_id, then entry_id (control-level links first), then method_id.", + "items": { "$ref": "#/definitions/Link" } + } + }, + "definitions": { + "Link": { + "type": "object", + "required": ["method_id", "control_id", "entry_id", "reviewed_by"], + "additionalProperties": false, + "properties": { + "method_id": { "type": "string", "pattern": "^VM-\\d{4}$" }, + "control_id": { "type": "string", "minLength": 1, "description": "A control_id in the framework's registry." }, + "entry_id": { + "description": "null: the link applies to the control wherever it is mapped. An entry id: only to that risk-control row.", + "oneOf": [ + { "type": "null" }, + { "type": "string", "pattern": "^(LLM\\d{2}|ASI\\d{2}|DSGAI\\d{2}|AST\\d{2})$" } + ] + }, + "reviewed_by": { + "type": "array", + "items": { "type": "string", "pattern": "\\S" }, + "description": "Named humans who confirmed the method verifies this control or row. Empty means draft." + } + } + } + } +} \ No newline at end of file diff --git a/data/verification-links/README.md b/data/verification-links/README.md new file mode 100644 index 0000000..17b709c --- /dev/null +++ b/data/verification-links/README.md @@ -0,0 +1,9 @@ +# Verification links + +One file per framework, named by its registry id: `.json`, matching +`data/frameworks/.json`. Each file links verification methods from +`data/verification-methods.json` to that framework's controls. + +The format is defined in `data/verification-links-schema.json` and explained in +[docs/VERIFICATION_METHODS.md](../../docs/VERIFICATION_METHODS.md). +`scripts/validate.js` checks every file here. diff --git a/data/verification-methods-schema.json b/data/verification-methods-schema.json new file mode 100644 index 0000000..bae95b0 --- /dev/null +++ b/data/verification-methods-schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/verification-methods-schema.json", + "title": "GenAI Security Crosswalk — Verification Methods Schema", + "description": "Verification methods: how to check that a control is implemented. Each method is held once and linked to controls through data/verification-links/. See docs/VERIFICATION_METHODS.md and issue #191.", + "type": "object", + "required": ["version", "methods"], + "additionalProperties": false, + "properties": { + "version": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "methods": { + "type": "array", + "items": { "$ref": "#/definitions/Method" } + } + }, + "definitions": { + "Method": { + "type": "object", + "required": ["id", "name", "method_type", "procedure", "expected_result", "source", "status", "reviewed_by"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^VM-\\d{4}$", + "description": "Project ID. Sequential, opaque, never reused or renumbered." + }, + "name": { "type": "string", "minLength": 1, "maxLength": 80 }, + "method_type": { + "enum": ["examine", "interview", "test"], + "description": "Assessment method (NIST SP 800-53A)." + }, + "procedure": { "type": "string", "minLength": 1, "description": "What the assessor does." }, + "expected_result": { "type": "string", "minLength": 1, "description": "What a pass looks like. Must be observable." }, + "source": { + "description": "Source the method is derived from; null for a method authored in the project.", + "oneOf": [{ "type": "null" }, { "$ref": "#/definitions/Source" }] + }, + "status": { "enum": ["draft", "reviewed", "deprecated"] }, + "reviewed_by": { + "type": "array", + "items": { "type": "string", "pattern": "\\S" }, + "description": "Named humans who reviewed the method, as for schema v2 mapping rows. Empty while draft." + }, + "frequency": { "$ref": "#/definitions/Frequency" }, + "evidence": { "type": "string", "minLength": 1, "description": "Artefact(s) the check produces or consumes." }, + "review_date": { "type": "string", "format": "date", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "status_note": { + "type": "string", + "minLength": 1, + "description": "Status context. Required when deprecated: the reason, and the replacement ID if there is one." + } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "reviewed" } }, "required": ["status"] }, + "then": { "properties": { "reviewed_by": { "minItems": 1 } } } + }, + { + "if": { "properties": { "status": { "const": "deprecated" } }, "required": ["status"] }, + "then": { "required": ["status_note"] } + } + ] + }, + "Source": { + "type": "object", + "required": ["name", "version", "id", "url", "license"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "id": { "type": "string", "minLength": 1, "description": "Identifier within the source, e.g. C02." }, + "url": { "type": "string", "pattern": "^https?://" }, + "license": { + "type": "string", + "minLength": 1, + "description": "SPDX identifier where one exists (e.g. CC-BY-SA-4.0); otherwise a short name such as Proprietary." + }, + "text": { + "type": "string", + "minLength": 1, + "description": "The source's original wording, unchanged. Include only if the licence allows reproduction in this CC BY-SA 4.0 repository." + } + } + }, + "Frequency": { + "type": "object", + "required": ["mode"], + "additionalProperties": false, + "description": "Recommended minimum frequency, not a requirement. An object so that interval or triggers can be added later.", + "properties": { + "mode": { "enum": ["continuous", "periodic", "event_driven"] } + } + } + } +} \ No newline at end of file diff --git a/data/verification-methods.json b/data/verification-methods.json new file mode 100644 index 0000000..9618d92 --- /dev/null +++ b/data/verification-methods.json @@ -0,0 +1,5 @@ +{ + "version": "1.0", + "description": "Verification methods: how to check a control is implemented. See docs/VERIFICATION_METHODS.md.", + "methods": [] +} \ No newline at end of file diff --git a/docs/VERIFICATION_METHODS.md b/docs/VERIFICATION_METHODS.md new file mode 100644 index 0000000..8e4b47a --- /dev/null +++ b/docs/VERIFICATION_METHODS.md @@ -0,0 +1,120 @@ +# Verification methods + +A mapping row says that a control addresses a risk. A verification method says +how to check that the control is actually in place. Methods and their links to +controls are held as data, agreed in issue #191 after the pilot in #186. + +| File | Holds | Schema | +|---|---|---| +| `data/verification-methods.json` | Each method, once | `data/verification-methods-schema.json` | +| `data/verification-links/.json` | Links methods to framework's controls | `data/verification-links-schema.json` | + +Both are hand-edited source files. `scripts/validate.js` checks them on every +full run, through `scripts/verification.js`. Version 1 is data, schemas and +validation only: the generator, the exports and the webapp do not read these +files yet. + +--- + +## Methods + +| Field | Required | Content | +|---|---|---| +| `id` | yes | `VM-` and four digits. Sequential, never reused or renumbered. | +| `name` | yes | Short title, at most 80 characters | +| `method_type` | yes | One of: `examine`, `interview`, `test` (below) | +| `procedure` | yes | What the assessor does | +| `expected_result` | yes | What a pass looks like. Must be observable. | +| `source` | yes | The source the method is derived from, or `null` if it was written here | +| `status` | yes | One of: `draft`, `reviewed`, `deprecated` | +| `reviewed_by` | yes | Named humans who reviewed the method. Empty while `draft`. | +| `frequency` | no | Recommended minimum: `{ "mode": "continuous" }`, `periodic` or `event_driven` | +| `evidence` | no | Artefacts the check produces or consumes | +| `review_date` | no | `YYYY-MM-DD` of the latest review | +| `status_note` | no | Status context. Required when `deprecated`: the reason and any replacement id. | + +`method_type` follows the assessment methods of NIST SP 800-53A. A method that +combines types records its main one. + +| Value | Meaning | +|---|---| +| `examine` | Review, inspect or analyse artefacts: documents, configurations, logs, records | +| `interview` | Discuss with the people responsible for the control | +| `test` | Exercise the control under defined conditions and compare actual with expected behaviour | + +### Sources and licences + +`source` holds `name`, `version`, `id`, `url`, `license` and, optionally, `text`. +Write `license` as an SPDX identifier where one exists (`CC-BY-SA-4.0`). + +- **Include `text`** (the source's original wording, unchanged) only if the + source's licence allows reproducing it in this CC BY-SA 4.0 repository. The + method's own fields are then adapted from it. +- **Otherwise cite by `id` and `url` only**, for example for proprietary + standards. + +The validator does not judge licences; contributors and reviewers do, in the PR. + +### Status and changes + +- `reviewed` needs at least one name in `reviewed_by`. An agent never adds itself. +- An editorial change (clarity, typos) keeps the id. A change to what is tested + or to the pass criteria is a new method. The old one becomes `deprecated`, with + `status_note` naming the replacement, and its links are re-pointed in the same PR. +- Deprecated methods stay in the file, so an id never comes to mean something else. + +--- + +## Links + +A links file is named after a framework's registry id and declares it: + +```json +{ + "version": "1.0", + "framework": "cosai", + "links": [ + { "method_id": "VM-0001", "control_id": "WS4-AGT-2.2", "entry_id": null, "reviewed_by": [] }, + { "method_id": "VM-0002", "control_id": "WS4-AGT-2.2", "entry_id": "ASI03", "reviewed_by": [] } + ] +} +``` + +| Field | Content | +|---|---| +| `method_id` | A method in `data/verification-methods.json` | +| `control_id` | A control in `data/frameworks/.json` | +| `entry_id` | `null` for a control-level link; an entry id (`LLM01`, `ASI04`, `DSGAI07`, `AST01`) for one risk–control row | +| `reviewed_by` | Named humans who confirmed the method verifies this control or row. Empty means draft. | + +**Control level or row level?** Ask whether the method verifies the control the +same way regardless of the risk. + +- **Yes:** `entry_id: null`. One link covers every row of the control. +- **No:** set `entry_id`. A control stating an outcome ("limit agent actions") + is usually checked differently for each risk it is mapped to. + +Links are sorted by `control_id`, then `entry_id` with control-level links first, +then `method_id`. Sorting spreads parallel additions across the file, so they +rarely conflict. + +--- + +## Validation + +Errors fail the build: + +- a field missing, undeclared, or outside its allowed values or pattern +- a duplicate method id or a duplicate link +- `reviewed` without a reviewer, or `deprecated` without a `status_note` +- a links file whose name differs from its `framework`, or with no registry +- a link to a method, control or (for row-level links) mapping row that does not + exist. Row-level links are matched through the registry's display name, since + mapping rows are keyed by name and the registry id is the stable key. +- links out of order + +Warnings flag review states that would overstate each other: + +- a reviewed link on a method that is still `draft` +- a reviewed row-level link on a mapping row with no reviewer +- a link to a `deprecated` method diff --git a/scripts/validate.js b/scripts/validate.js index 93ddaed..32cd03b 100644 --- a/scripts/validate.js +++ b/scripts/validate.js @@ -1112,6 +1112,22 @@ function checkEvidence() { return bad === 0; } +/** + * Verification methods and links (issue #191). + * + * data/verification-methods.json and data/verification-links/.json + * are hand-edited source files. The rules live in scripts/verification.js so the + * tests can drive them with fixtures; this only reports what it finds. + */ +function checkVerificationData() { + const { checkVerification } = require('./verification'); + const r = checkVerification(ROOT); + for (const e of r.errors) fail(e.file, e.msg); + for (const w of r.warnings) warn(w.file, w.msg); + if (!r.errors.length) pass('Verification', r.summary); + return r.errors.length === 0; +} + function run() { const args = process.argv.slice(2); const quickMode = args.includes('--quick'); @@ -1165,6 +1181,7 @@ function run() { checkControlIdShapes(); checkIncidentIds(); checkEvidence(); + checkVerificationData(); } // Encoding guard — mapping files plus the shared/root markdown they link to diff --git a/scripts/verification.js b/scripts/verification.js new file mode 100644 index 0000000..ac2642b --- /dev/null +++ b/scripts/verification.js @@ -0,0 +1,323 @@ +'use strict'; +/** + * Verification methods and their links to controls (issue #191). + * + * data/verification-methods.json each method held once + * data/verification-links/.json method ↔ control links, per framework + * + * A method says how to check that a control is implemented. A link says that a + * method verifies a control: everywhere the control is mapped (`entry_id: null`) + * or on one risk–control row (`entry_id` set). The two carry separate reviews, + * because "is this method well written?" and "does it verify this control?" are + * different judgments. + * + * v1 is data, schemas and this validator only. Nothing in generate.js, the + * exports or the webapp reads these files yet. + * + * The allowed values (enums, id patterns, length limits) are read from the two + * schemas, so the schemas stay the single source for them. The schemas cannot + * express the cross-file rules — a link's method, control and mapping row must + * exist — so those live here. + * + * checkVerification() returns findings; scripts/validate.js reports them. + * Errors fail the build. Warnings flag review states that would overstate each + * other: a reviewed link on a draft method, or on a mapping row nobody reviewed. + */ + +const fs = require('fs'); +const path = require('path'); + +const METHODS_FILE = path.join('data', 'verification-methods.json'); +const LINKS_DIR = path.join('data', 'verification-links'); +const METHODS_SCHEMA = path.join('data', 'verification-methods-schema.json'); +const LINKS_SCHEMA = path.join('data', 'verification-links-schema.json'); + +const rel = (root, p) => path.relative(root, p).split(path.sep).join('/'); +const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); +const isNonEmptyString = (v) => typeof v === 'string' && v.trim().length > 0; +const isNamedList = (v) => Array.isArray(v) && v.every(isNonEmptyString); + +function readJson(file) { + try { + return { doc: JSON.parse(fs.readFileSync(file, 'utf8')) }; + } catch (e) { + return { error: e.message }; + } +} + +/** The vocabulary both schemas define, extracted once. */ +function loadRules(root) { + const ms = JSON.parse(fs.readFileSync(path.join(root, METHODS_SCHEMA), 'utf8')); + const ls = JSON.parse(fs.readFileSync(path.join(root, LINKS_SCHEMA), 'utf8')); + const method = ms.definitions.Method; + const source = ms.definitions.Source; + const link = ls.definitions.Link; + return { + methodsTop: { required: ms.required, keys: Object.keys(ms.properties) }, + method: { required: method.required, keys: Object.keys(method.properties) }, + source: { required: source.required, keys: Object.keys(source.properties) }, + frequency: { required: ms.definitions.Frequency.required, keys: Object.keys(ms.definitions.Frequency.properties) }, + linksTop: { required: ls.required, keys: Object.keys(ls.properties) }, + link: { required: link.required, keys: Object.keys(link.properties) }, + methodId: new RegExp(method.properties.id.pattern), + nameMax: method.properties.name.maxLength, + methodTypes: method.properties.method_type.enum, + statuses: method.properties.status.enum, + frequencyModes: ms.definitions.Frequency.properties.mode.enum, + url: new RegExp(source.properties.url.pattern), + date: new RegExp(method.properties.review_date.pattern), + frameworkId: new RegExp(ls.properties.framework.pattern), + entryId: new RegExp(link.properties.entry_id.oneOf[1].pattern), + }; +} + +/** Required keys present, no undeclared keys. Returns messages. */ +function shape(obj, { required, keys }, label) { + const out = []; + for (const k of required) if (!(k in obj)) out.push(`${label}: missing required field "${k}"`); + for (const k of Object.keys(obj)) if (!keys.includes(k)) out.push(`${label}: unknown field "${k}"`); + return out; +} + +// ─── Methods ────────────────────────────────────────────────────────────────── + +function checkMethod(m, i, R) { + const label = `methods[${i}]${isNonEmptyString(m && m.id) ? ` ${m.id}` : ''}`; + if (!isObject(m)) return [`${label}: not an object`]; + const out = shape(m, R.method, label); + + if ('id' in m && !(typeof m.id === 'string' && R.methodId.test(m.id))) { + out.push(`${label}: id "${m.id}" does not match ${R.methodId.source}`); + } + if ('name' in m) { + if (!isNonEmptyString(m.name)) out.push(`${label}: name must be a non-empty string`); + else if (m.name.length > R.nameMax) out.push(`${label}: name is ${m.name.length} characters (max ${R.nameMax})`); + } + for (const k of ['procedure', 'expected_result', 'evidence', 'status_note']) { + if (k in m && !isNonEmptyString(m[k])) out.push(`${label}: ${k} must be a non-empty string`); + } + if ('method_type' in m && !R.methodTypes.includes(m.method_type)) { + out.push(`${label}: method_type "${m.method_type}" is not ${R.methodTypes.join('|')}`); + } + if ('status' in m && !R.statuses.includes(m.status)) { + out.push(`${label}: status "${m.status}" is not ${R.statuses.join('|')}`); + } + if ('reviewed_by' in m && !isNamedList(m.reviewed_by)) { + out.push(`${label}: reviewed_by must be a list of names`); + } + if (m.status === 'reviewed' && !(Array.isArray(m.reviewed_by) && m.reviewed_by.length)) { + out.push(`${label}: status "reviewed" but reviewed_by is empty; only a named reviewer may mark a method reviewed`); + } + if (m.status === 'deprecated' && !('status_note' in m)) { + out.push(`${label}: status "deprecated" requires status_note (the reason, and the replacement id if any)`); + } + if ('review_date' in m && !(typeof m.review_date === 'string' && R.date.test(m.review_date))) { + out.push(`${label}: review_date "${m.review_date}" is not YYYY-MM-DD`); + } + if ('frequency' in m) { + if (!isObject(m.frequency)) out.push(`${label}: frequency must be an object`); + else { + out.push(...shape(m.frequency, R.frequency, `${label} frequency`)); + if ('mode' in m.frequency && !R.frequencyModes.includes(m.frequency.mode)) { + out.push(`${label}: frequency.mode "${m.frequency.mode}" is not ${R.frequencyModes.join('|')}`); + } + } + } + if ('source' in m && m.source !== null) { + const s = m.source; + if (!isObject(s)) out.push(`${label}: source must be an object or null`); + else { + out.push(...shape(s, R.source, `${label} source`)); + for (const k of R.source.keys) { + if (k in s && !isNonEmptyString(s[k])) out.push(`${label}: source.${k} must be a non-empty string`); + } + if (isNonEmptyString(s.url) && !R.url.test(s.url)) out.push(`${label}: source.url must start with http:// or https://`); + } + } + return out; +} + +function checkMethods(root, R, findings) { + const file = path.join(root, METHODS_FILE); + const methods = new Map(); + if (!fs.existsSync(file)) return methods; + const where = rel(root, file); + + const { doc, error } = readJson(file); + if (error) { findings.errors.push({ file: where, msg: `not valid JSON: ${error}` }); return methods; } + if (!isObject(doc)) { findings.errors.push({ file: where, msg: 'top level must be an object' }); return methods; } + for (const msg of shape(doc, R.methodsTop, 'file')) findings.errors.push({ file: where, msg }); + if (!Array.isArray(doc.methods)) { + if ('methods' in doc) findings.errors.push({ file: where, msg: 'methods must be an array' }); + return methods; + } + + doc.methods.forEach((m, i) => { + for (const msg of checkMethod(m, i, R)) findings.errors.push({ file: where, msg }); + if (isObject(m) && typeof m.id === 'string') { + if (methods.has(m.id)) { + findings.errors.push({ file: where, msg: `methods[${i}]: id ${m.id} is used more than once` }); + } else { + methods.set(m.id, m); + } + } + }); + return methods; +} + +// ─── Links ──────────────────────────────────────────────────────────────────── + +/** Registry id → { name, controls }, and mapping rows keyed entry|name|control. */ +function loadContext(root) { + const registries = new Map(); + const fwDir = path.join(root, 'data', 'frameworks'); + if (fs.existsSync(fwDir)) { + for (const f of fs.readdirSync(fwDir).filter((n) => n.endsWith('.json'))) { + const { doc } = readJson(path.join(fwDir, f)); + if (!isObject(doc) || !doc.id) continue; + registries.set(doc.id, { + name: doc.name, + controls: new Set((doc.controls || []).map((c) => c.control_id)), + }); + } + } + const rows = new Map(); + const entDir = path.join(root, 'data', 'entries'); + if (fs.existsSync(entDir)) { + for (const f of fs.readdirSync(entDir).filter((n) => n.endsWith('.json'))) { + const { doc } = readJson(path.join(entDir, f)); + if (!isObject(doc)) continue; + for (const m of doc.mappings || []) rows.set(`${doc.id}|${m.framework}|${m.control_id}`, m); + } + } + return { registries, rows }; +} + +const sortKey = (l) => [String(l.control_id), l.entry_id === null ? '' : String(l.entry_id), String(l.method_id)]; +function compareLinks(a, b) { + const ka = sortKey(a), kb = sortKey(b); + for (let i = 0; i < ka.length; i++) { + if (ka[i] < kb[i]) return -1; + if (ka[i] > kb[i]) return 1; + } + return 0; +} + +function checkLinksFile(root, file, R, methods, ctx, findings) { + const where = rel(root, file); + const err = (msg) => findings.errors.push({ file: where, msg }); + const warn = (msg) => findings.warnings.push({ file: where, msg }); + const stem = path.basename(file, '.json'); + + const { doc, error } = readJson(file); + if (error) return err(`not valid JSON: ${error}`); + if (!isObject(doc)) return err('top level must be an object'); + for (const msg of shape(doc, R.linksTop, 'file')) err(msg); + + if (doc.framework !== stem) err(`framework "${doc.framework}" does not match the file name "${stem}"`); + else if (!R.frameworkId.test(stem)) err(`framework id "${stem}" is not a registry id`); + const registry = ctx.registries.get(doc.framework); + if (doc.framework === stem && !registry) err(`no registry data/frameworks/${stem}.json`); + + if (!Array.isArray(doc.links)) { + if ('links' in doc) err('links must be an array'); + return 0; + } + + const seen = new Set(); + let reviewed = 0; + doc.links.forEach((l, i) => { + const label = `links[${i}]`; + if (!isObject(l)) return err(`${label}: not an object`); + const shapeErrors = shape(l, R.link, label); + shapeErrors.forEach(err); + if (shapeErrors.length) return; + + const tag = `${label} ${l.method_id} → ${l.control_id}${l.entry_id ? ` @ ${l.entry_id}` : ''}`; + if (!(typeof l.method_id === 'string' && R.methodId.test(l.method_id))) err(`${tag}: method_id does not match ${R.methodId.source}`); + if (!isNonEmptyString(l.control_id)) err(`${tag}: control_id must be a non-empty string`); + if (!(l.entry_id === null || (typeof l.entry_id === 'string' && R.entryId.test(l.entry_id)))) { + err(`${tag}: entry_id must be null or an entry id such as ASI04`); + } + if (!isNamedList(l.reviewed_by)) err(`${tag}: reviewed_by must be a list of names`); + + const key = `${l.method_id}|${l.control_id}|${l.entry_id}`; + if (seen.has(key)) err(`${tag}: duplicate link`); + seen.add(key); + + const method = methods.get(l.method_id); + if (!method) err(`${tag}: method ${l.method_id} is not in ${METHODS_FILE.split(path.sep).join('/')}`); + else if (method.status === 'deprecated') warn(`${tag}: method ${l.method_id} is deprecated; re-point or delete the link`); + + if (registry && !registry.controls.has(l.control_id)) { + err(`${tag}: control ${l.control_id} is not in data/frameworks/${doc.framework}.json`); + } + + // The registry id is the key; mapping rows carry the registry's display name. + let row; + if (registry && typeof l.entry_id === 'string') { + row = ctx.rows.get(`${l.entry_id}|${registry.name}|${l.control_id}`); + if (!row) err(`${tag}: no mapping row for ${l.entry_id} × ${registry.name} ${l.control_id}`); + } + + const linkReviewed = Array.isArray(l.reviewed_by) && l.reviewed_by.length > 0; + if (linkReviewed) { + reviewed++; + if (method && method.status === 'draft') { + warn(`${tag}: link is reviewed but method ${l.method_id} is still draft`); + } + if (row && !(Array.isArray(row.reviewed_by) && row.reviewed_by.length)) { + warn(`${tag}: link is reviewed but the mapping row ${l.entry_id} × ${l.control_id} is unreviewed`); + } + } + }); + + // A sorted file spreads parallel additions across it instead of all at its end. + for (let i = 1; i < doc.links.length; i++) { + const a = doc.links[i - 1], b = doc.links[i]; + if (isObject(a) && isObject(b) && compareLinks(a, b) > 0) { + err(`links are not sorted at links[${i}] (expected order: control_id, entry_id with control-level first, method_id)`); + break; + } + } + findings.counts.links += doc.links.length; + findings.counts.reviewedLinks += reviewed; + return doc.links.length; +} + +// ─── Entry point ────────────────────────────────────────────────────────────── + +/** + * @param {string} root repository root + * @returns {{ errors: {file:string,msg:string}[], warnings: {file:string,msg:string}[], + * counts: {methods:number, reviewedMethods:number, linkFiles:number, links:number, reviewedLinks:number}, + * summary: string }} + */ +function checkVerification(root) { + const findings = { + errors: [], + warnings: [], + counts: { methods: 0, reviewedMethods: 0, linkFiles: 0, links: 0, reviewedLinks: 0 }, + }; + const R = loadRules(root); + const methods = checkMethods(root, R, findings); + findings.counts.methods = methods.size; + findings.counts.reviewedMethods = [...methods.values()].filter((m) => m.status === 'reviewed').length; + + const dir = path.join(root, LINKS_DIR); + if (fs.existsSync(dir)) { + const files = fs.readdirSync(dir).filter((n) => n.endsWith('.json')).sort(); + if (files.length) { + const ctx = loadContext(root); + for (const f of files) checkLinksFile(root, path.join(dir, f), R, methods, ctx, findings); + } + findings.counts.linkFiles = files.length; + } + + const c = findings.counts; + findings.summary = `${c.methods} method(s), ${c.reviewedMethods} reviewed; ` + + `${c.links} link(s) in ${c.linkFiles} file(s), ${c.reviewedLinks} reviewed`; + return findings; +} + +module.exports = { checkVerification, loadRules, compareLinks }; \ No newline at end of file diff --git a/scripts/verification.test.mjs b/scripts/verification.test.mjs new file mode 100644 index 0000000..a1508d8 --- /dev/null +++ b/scripts/verification.test.mjs @@ -0,0 +1,271 @@ +/** + * verification.test.mjs — the rules behind data/verification-methods.json and + * data/verification-links/ (issue #191). + * + * Each rule gets a fixture that breaks it. The schemas are also compiled with + * Ajv and run against the same fixtures, so the JSON Schema and the hand-written + * checks in scripts/verification.js cannot drift apart on the rules both express. + */ + +import test, { after } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { checkVerification } = require(path.join(ROOT, 'scripts', 'verification.js')); + +const readJson = (p) => JSON.parse(fs.readFileSync(path.join(ROOT, p), 'utf8')); +const METHODS_SCHEMA = readJson('data/verification-methods-schema.json'); +const LINKS_SCHEMA = readJson('data/verification-links-schema.json'); + +const ajv = new Ajv({ allErrors: true, strict: false }); +addFormats(ajv); +const validMethodsDoc = ajv.compile(METHODS_SCHEMA); +const validLinksDoc = ajv.compile(LINKS_SCHEMA); + +// ─── Fixtures ───────────────────────────────────────────────────────────────── + +const method = (over = {}) => ({ + id: 'VM-0001', + name: 'Injection test across ingestion paths', + method_type: 'test', + procedure: 'Submit a maintained injection payload set through each ingestion path.', + expected_result: 'No payload alters the agent goal; all attempts are logged.', + source: null, + status: 'draft', + reviewed_by: [], + ...over, +}); + +const source = (over = {}) => ({ + name: 'Example Catalogue', + version: '1.0', + id: 'C02', + url: 'https://example.org/catalogue', + license: 'CC-BY-SA-4.0', + text: 'Injection test executed through each ingestion path.', + ...over, +}); + +const link = (over = {}) => ({ + method_id: 'VM-0001', + control_id: 'C1', + entry_id: null, + reviewed_by: [], + ...over, +}); + +// Every throwaway repository is deleted when the tests finish. +const tempRoots = []; +after(() => { for (const r of tempRoots) fs.rmSync(r, { recursive: true, force: true }); }); + +/** + * A throwaway repository: the real schemas, one framework registry (`fw`, name + * "FW") with controls C1 and C2, and one entry ASI01 mapping FW:C1. The ASI01 row + * is reviewed unless `rowReviewed` is false. `methods: null` writes no methods file. + */ +function repo({ methods = [method()], links, linkFiles, rowReviewed = true } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'verif-')); + tempRoots.push(root); + const w = (p, obj) => { + fs.mkdirSync(path.dirname(path.join(root, p)), { recursive: true }); + fs.writeFileSync(path.join(root, p), typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2)); + }; + w('data/verification-methods-schema.json', METHODS_SCHEMA); + w('data/verification-links-schema.json', LINKS_SCHEMA); + w('data/frameworks/fw.json', { + id: 'fw', name: 'FW', controls: [{ control_id: 'C1' }, { control_id: 'C2' }], + }); + w('data/entries/ASI01.json', { + id: 'ASI01', + mappings: [{ framework: 'FW', control_id: 'C1', reviewed_by: rowReviewed ? ['A. Reviewer'] : [] }], + }); + if (methods !== null) w('data/verification-methods.json', { version: '1.0', methods }); + if (links !== undefined) w('data/verification-links/fw.json', { version: '1.0', framework: 'fw', links }); + for (const [name, doc] of Object.entries(linkFiles || {})) w(`data/verification-links/${name}`, doc); + return root; +} + +const run = (opts) => checkVerification(repo(opts)); +const errorsMatch = (r, re) => r.errors.some((e) => re.test(e.msg)); +const warningsMatch = (r, re) => r.warnings.some((e) => re.test(e.msg)); + +// ─── The repository's own data ──────────────────────────────────────────────── + +test('the repository verification data is valid', () => { + const r = checkVerification(ROOT); + assert.deepEqual(r.errors, []); + assert.ok(validMethodsDoc(readJson('data/verification-methods.json')), JSON.stringify(validMethodsDoc.errors)); + const dir = path.join(ROOT, 'data', 'verification-links'); + for (const f of fs.existsSync(dir) ? fs.readdirSync(dir).filter((n) => n.endsWith('.json')) : []) { + assert.ok(validLinksDoc(readJson(`data/verification-links/${f}`)), `${f}: ${JSON.stringify(validLinksDoc.errors)}`); + } +}); + +test('the links schema entry_id pattern matches the entry id pattern in data/schema.json', () => { + const entryPattern = readJson('data/schema.json').properties.id.pattern; + assert.equal(LINKS_SCHEMA.definitions.Link.properties.entry_id.oneOf[1].pattern, entryPattern); +}); + +test('no verification files at all is not an error', () => { + const r = run({ methods: null }); + assert.deepEqual(r.errors, []); + assert.equal(r.counts.methods, 0); +}); + +// ─── Methods: structure ─────────────────────────────────────────────────────── + +test('a complete method passes, with and without optional fields', () => { + const full = method({ + source: source(), + frequency: { mode: 'event_driven' }, + evidence: 'Test report with per-path results.', + review_date: '2026-10-06', + }); + const r = run({ methods: [full, method({ id: 'VM-0002' })] }); + assert.deepEqual(r.errors, []); + assert.equal(r.counts.methods, 2); + assert.ok(validMethodsDoc({ version: '1.0', methods: [full] }), JSON.stringify(validMethodsDoc.errors)); +}); + +const methodCases = [ + ['a missing required field', { ...method(), procedure: undefined }, /missing required field "procedure"/], + ['an undeclared field', method({ severity: 'high' }), /unknown field "severity"/], + ['an id outside VM-NNNN', method({ id: 'VM-12' }), /does not match/], + ['a name over 80 characters', method({ name: 'x'.repeat(81) }), /max 80/], + ['an unknown method_type', method({ method_type: 'observe' }), /method_type "observe"/], + ['an unknown status', method({ status: 'approved' }), /status "approved"/], + ['reviewed with no reviewer', method({ status: 'reviewed', reviewed_by: [] }), /reviewed_by is empty/], + ['a blank reviewer name', method({ reviewed_by: [' '] }), /list of names/], + ['deprecated with no note', method({ status: 'deprecated' }), /requires status_note/], + ['a malformed review_date', method({ review_date: '06/10/2026' }), /YYYY-MM-DD/], + ['an unknown frequency mode', method({ frequency: { mode: 'weekly' } }), /frequency.mode "weekly"/], + ['an undeclared frequency field', method({ frequency: { mode: 'periodic', interval: 'P3M' } }), /unknown field "interval"/], + ['a source missing its url', method({ source: { ...source(), url: undefined } }), /missing required field "url"/], + ['a source url that is not http(s)', method({ source: source({ url: 'ftp://x' }) }), /http/], +]; + +for (const [what, m, re] of methodCases) { + test(`a method with ${what} fails — validator and schema agree`, () => { + const clean = JSON.parse(JSON.stringify(m)); // drops undefined keys + const r = run({ methods: [clean] }); + assert.ok(errorsMatch(r, re), `expected ${re}, got ${JSON.stringify(r.errors)}`); + assert.equal(validMethodsDoc({ version: '1.0', methods: [clean] }), false, 'schema should reject it too'); + }); +} + +test('method ids are unique', () => { + const r = run({ methods: [method(), method({ name: 'Another' })] }); + assert.ok(errorsMatch(r, /VM-0001 is used more than once/)); +}); + +// ─── Methods: source ────────────────────────────────────────────────────────── + +test('source text is optional and not tied to a licence list', () => { + // Whether a licence allows reproducing text is checked in review, not here. + for (const m of [ + method({ source: source({ license: 'CC-BY-SA-4.0' }) }), + method({ source: source({ license: 'Proprietary', text: undefined }) }), + method({ source: source({ license: 'Some-Other-Licence-2.0' }) }), + ]) { + const clean = JSON.parse(JSON.stringify(m)); + assert.deepEqual(run({ methods: [clean] }).errors, [], clean.source.license); + assert.ok(validMethodsDoc({ version: '1.0', methods: [clean] }), JSON.stringify(validMethodsDoc.errors)); + } +}); + +// ─── Links ──────────────────────────────────────────────────────────────────── + +test('a control-level and a row-level link pass', () => { + const r = run({ links: [link(), link({ entry_id: 'ASI01' })] }); + assert.deepEqual(r.errors, []); + assert.equal(r.counts.links, 2); + assert.ok(validLinksDoc({ version: '1.0', framework: 'fw', links: [link(), link({ entry_id: 'ASI01' })] })); +}); + +test('the file name must be the registry id it declares', () => { + const r = run({ linkFiles: { 'other.json': { version: '1.0', framework: 'fw', links: [] } } }); + assert.ok(errorsMatch(r, /does not match the file name "other"/)); +}); + +test('the framework must have a registry', () => { + const r = run({ linkFiles: { 'nope.json': { version: '1.0', framework: 'nope', links: [] } } }); + assert.ok(errorsMatch(r, /no registry data\/frameworks\/nope.json/)); +}); + +test('a link must name an existing method', () => { + const r = run({ links: [link({ method_id: 'VM-0099' })] }); + assert.ok(errorsMatch(r, /method VM-0099 is not in/)); +}); + +test('a link must name a control in the registry', () => { + const r = run({ links: [link({ control_id: 'C9' })] }); + assert.ok(errorsMatch(r, /control C9 is not in data\/frameworks\/fw.json/)); +}); + +test('a row-level link needs a mapping row, matched through the registry name', () => { + assert.ok(errorsMatch(run({ links: [link({ control_id: 'C2', entry_id: 'ASI01' })] }), /no mapping row for ASI01 × FW C2/)); + assert.ok(errorsMatch(run({ links: [link({ entry_id: 'ASI02' })] }), /no mapping row for ASI02/)); +}); + +test('entry_id is null or an entry id', () => { + const r = run({ links: [link({ entry_id: 'ASI1' })] }); + assert.ok(errorsMatch(r, /entry_id must be null or an entry id/)); + assert.equal(validLinksDoc({ version: '1.0', framework: 'fw', links: [link({ entry_id: 'ASI1' })] }), false); +}); + +test('a link has exactly the declared fields', () => { + const r = run({ links: [{ ...link(), framework: 'fw' }] }); + assert.ok(errorsMatch(r, /unknown field "framework"/)); + assert.equal(validLinksDoc({ version: '1.0', framework: 'fw', links: [{ ...link(), framework: 'fw' }] }), false); +}); + +test('duplicate links are rejected', () => { + const r = run({ links: [link(), link()] }); + assert.ok(errorsMatch(r, /duplicate link/)); +}); + +test('links must be sorted: control_id, then control-level before row-level, then method_id', () => { + const methods = [method(), method({ id: 'VM-0002' })]; + const sorted = [link(), link({ method_id: 'VM-0002' }), link({ entry_id: 'ASI01' }), link({ control_id: 'C2' })]; + assert.deepEqual(run({ methods, links: sorted }).errors, []); + assert.ok(errorsMatch(run({ methods, links: [link({ control_id: 'C2' }), link()] }), /not sorted at links\[1\]/)); + assert.ok(errorsMatch(run({ methods, links: [link({ entry_id: 'ASI01' }), link()] }), /not sorted/)); + assert.ok(errorsMatch(run({ methods, links: [link({ method_id: 'VM-0002' }), link()] }), /not sorted/)); +}); + +// ─── Review states must not overstate each other ────────────────────────────── + +test('a reviewed link on a draft method is a warning, not an error', () => { + const r = run({ links: [link({ reviewed_by: ['A. Reviewer'] })] }); + assert.deepEqual(r.errors, []); + assert.ok(warningsMatch(r, /link is reviewed but method VM-0001 is still draft/)); +}); + +test('a reviewed row-level link on an unreviewed mapping row is a warning', () => { + const methods = [method({ status: 'reviewed', reviewed_by: ['A. Reviewer'] })]; + const reviewedLink = link({ entry_id: 'ASI01', reviewed_by: ['B. Reviewer'] }); + const r = run({ methods, links: [reviewedLink], rowReviewed: false }); + assert.deepEqual(r.errors, []); + assert.ok(warningsMatch(r, /mapping row ASI01 × C1 is unreviewed/)); + assert.deepEqual(run({ methods, links: [reviewedLink], rowReviewed: true }).warnings, []); +}); + +test('draft links raise no review warnings', () => { + const r = run({ links: [link({ entry_id: 'ASI01' })], rowReviewed: false }); + assert.deepEqual(r.warnings, []); +}); + +test('a link to a deprecated method is a warning, not an error', () => { + const methods = [method({ status: 'deprecated', status_note: 'Replaced by VM-0002' })]; + const r = run({ methods, links: [link()] }); + assert.deepEqual(r.errors, []); + assert.ok(warningsMatch(r, /VM-0001 is deprecated/)); +}); From 100a15adea581d80cd79598e89a3afb7f98ffcee Mon Sep 17 00:00:00 2001 From: emmanuelgjr Date: Fri, 9 Oct 2026 11:10:54 -0400 Subject: [PATCH 2/2] fix(verification): CI path filter, top-level types, proprietary source text Review follow-ups on #215: - validate.yml: run on changes to scripts/verification.js (both filters), so an edit to the rules alone no longer skips validation and tests. - verification.js: type-check the top-level version/description of both files and require review_date to be a real calendar day. The schema already rejected version: 1, version: "" and 2026-13-45; the validator that CI runs did not. - Schema + validator: source.text is an error when source.license names itself proprietary or "all rights reserved" (case-insensitive). The pattern lives in the schema and the validator reads it from there. Every other licence is still judged in review. 8 new fixtures, each checked against both the validator and Ajv; all 8 fail on the previous validator. validate 0 errors / 92 warnings, test:scripts 137/137, stats:check current, no generate drift. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/validate.yml | 2 ++ CHANGELOG.md | 3 ++- data/verification-methods-schema.json | 13 +++++++++++ docs/VERIFICATION_METHODS.md | 4 +++- scripts/verification.js | 28 ++++++++++++++++++++---- scripts/verification.test.mjs | 31 ++++++++++++++++++++++++++- 6 files changed, 74 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index a0010fa..6da826e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -14,6 +14,7 @@ on: - 'CROSSREF.md' - 'README.md' - 'scripts/validate.js' + - 'scripts/verification.js' - 'scripts/generate.js' - 'scripts/*.test.mjs' - 'package.json' @@ -31,6 +32,7 @@ on: - 'CROSSREF.md' - 'README.md' - 'scripts/validate.js' + - 'scripts/verification.js' - 'scripts/generate.js' - 'scripts/*.test.mjs' - 'package.json' diff --git a/CHANGELOG.md b/CHANGELOG.md index b960fea..efccccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,8 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). `verification-links-schema.json`), and `scripts/validate.js` enforces them plus the cross-file rules through `scripts/verification.js` (tested in `scripts/verification.test.mjs`). A source's original wording is optional and included only where its licence allows; otherwise the method cites the source by id and url. Licence - compatibility is judged in review, not by the validator. v1 is data, schemas and validator only: no generator, + compatibility is judged in review; the validator rejects only `text` under a licence naming itself proprietary + or "all rights reserved". v1 is data, schemas and validator only: no generator, export or webapp changes. Documented in `docs/VERIFICATION_METHODS.md`. No methods yet; the pilot follows in a separate PR. diff --git a/data/verification-methods-schema.json b/data/verification-methods-schema.json index bae95b0..db6084f 100644 --- a/data/verification-methods-schema.json +++ b/data/verification-methods-schema.json @@ -81,6 +81,19 @@ "minLength": 1, "description": "The source's original wording, unchanged. Include only if the licence allows reproduction in this CC BY-SA 4.0 repository." } + }, + "if": { + "required": ["license"], + "properties": { + "license": { + "pattern": "[Pp][Rr][Oo][Pp][Rr][Ii][Ee][Tt][Aa][Rr][Yy]|[Aa][Ll][Ll] [Rr][Ii][Gg][Hh][Tt][Ss] [Rr][Ee][Ss][Ee][Rr][Vv][Ee][Dd]", + "description": "A licence that plainly forbids reproduction (case-insensitive 'proprietary' or 'all rights reserved')." + } + } + }, + "then": { + "not": { "required": ["text"] }, + "description": "A proprietary source is cited by id and url only; its wording is not copied into the repository." } }, "Frequency": { diff --git a/docs/VERIFICATION_METHODS.md b/docs/VERIFICATION_METHODS.md index 8e4b47a..0228ed8 100644 --- a/docs/VERIFICATION_METHODS.md +++ b/docs/VERIFICATION_METHODS.md @@ -53,7 +53,9 @@ Write `license` as an SPDX identifier where one exists (`CC-BY-SA-4.0`). - **Otherwise cite by `id` and `url` only**, for example for proprietary standards. -The validator does not judge licences; contributors and reviewers do, in the PR. +Contributors and reviewers judge licence compatibility in the PR. The validator +catches only the plain case: a `license` containing "proprietary" or "all rights +reserved" (any case) together with `text` is an error. ### Status and changes diff --git a/scripts/verification.js b/scripts/verification.js index ac2642b..3048a46 100644 --- a/scripts/verification.js +++ b/scripts/verification.js @@ -65,6 +65,7 @@ function loadRules(root) { statuses: method.properties.status.enum, frequencyModes: ms.definitions.Frequency.properties.mode.enum, url: new RegExp(source.properties.url.pattern), + noReproduce: new RegExp(source.if.properties.license.pattern), date: new RegExp(method.properties.review_date.pattern), frameworkId: new RegExp(ls.properties.framework.pattern), entryId: new RegExp(link.properties.entry_id.oneOf[1].pattern), @@ -79,6 +80,22 @@ function shape(obj, { required, keys }, label) { return out; } +/** The top-level scalars both files share: version required, description optional. */ +function topLevel(doc) { + const out = []; + if ('version' in doc && !isNonEmptyString(doc.version)) out.push('file: version must be a non-empty string'); + if ('description' in doc && typeof doc.description !== 'string') out.push('file: description must be a string'); + return out; +} + +/** YYYY-MM-DD that names a real calendar day, so 2026-13-45 fails. */ +function isCalendarDate(s, re) { + if (!(typeof s === 'string' && re.test(s))) return false; + const [y, m, d] = s.split('-').map(Number); + const t = new Date(Date.UTC(y, m - 1, d)); + return t.getUTCFullYear() === y && t.getUTCMonth() === m - 1 && t.getUTCDate() === d; +} + // ─── Methods ────────────────────────────────────────────────────────────────── function checkMethod(m, i, R) { @@ -111,8 +128,8 @@ function checkMethod(m, i, R) { if (m.status === 'deprecated' && !('status_note' in m)) { out.push(`${label}: status "deprecated" requires status_note (the reason, and the replacement id if any)`); } - if ('review_date' in m && !(typeof m.review_date === 'string' && R.date.test(m.review_date))) { - out.push(`${label}: review_date "${m.review_date}" is not YYYY-MM-DD`); + if ('review_date' in m && !isCalendarDate(m.review_date, R.date)) { + out.push(`${label}: review_date "${m.review_date}" is not a YYYY-MM-DD calendar date`); } if ('frequency' in m) { if (!isObject(m.frequency)) out.push(`${label}: frequency must be an object`); @@ -132,6 +149,9 @@ function checkMethod(m, i, R) { if (k in s && !isNonEmptyString(s[k])) out.push(`${label}: source.${k} must be a non-empty string`); } if (isNonEmptyString(s.url) && !R.url.test(s.url)) out.push(`${label}: source.url must start with http:// or https://`); + if (typeof s.license === 'string' && R.noReproduce.test(s.license) && 'text' in s) { + out.push(`${label}: source.license "${s.license}" does not allow reproduction; cite by id and url and drop source.text`); + } } } return out; @@ -146,7 +166,7 @@ function checkMethods(root, R, findings) { const { doc, error } = readJson(file); if (error) { findings.errors.push({ file: where, msg: `not valid JSON: ${error}` }); return methods; } if (!isObject(doc)) { findings.errors.push({ file: where, msg: 'top level must be an object' }); return methods; } - for (const msg of shape(doc, R.methodsTop, 'file')) findings.errors.push({ file: where, msg }); + for (const msg of [...shape(doc, R.methodsTop, 'file'), ...topLevel(doc)]) findings.errors.push({ file: where, msg }); if (!Array.isArray(doc.methods)) { if ('methods' in doc) findings.errors.push({ file: where, msg: 'methods must be an array' }); return methods; @@ -212,7 +232,7 @@ function checkLinksFile(root, file, R, methods, ctx, findings) { const { doc, error } = readJson(file); if (error) return err(`not valid JSON: ${error}`); if (!isObject(doc)) return err('top level must be an object'); - for (const msg of shape(doc, R.linksTop, 'file')) err(msg); + for (const msg of [...shape(doc, R.linksTop, 'file'), ...topLevel(doc)]) err(msg); if (doc.framework !== stem) err(`framework "${doc.framework}" does not match the file name "${stem}"`); else if (!R.frameworkId.test(stem)) err(`framework id "${stem}" is not a registry id`); diff --git a/scripts/verification.test.mjs b/scripts/verification.test.mjs index a1508d8..c9c71c6 100644 --- a/scripts/verification.test.mjs +++ b/scripts/verification.test.mjs @@ -146,6 +146,9 @@ const methodCases = [ ['a blank reviewer name', method({ reviewed_by: [' '] }), /list of names/], ['deprecated with no note', method({ status: 'deprecated' }), /requires status_note/], ['a malformed review_date', method({ review_date: '06/10/2026' }), /YYYY-MM-DD/], + ['a review_date that is not a calendar day', method({ review_date: '2026-13-45' }), /calendar date/], + ['proprietary source text', method({ source: source({ license: 'Proprietary (AIUC)' }) }), /does not allow reproduction/], + ['all-rights-reserved source text', method({ source: source({ license: 'All rights reserved' }) }), /does not allow reproduction/], ['an unknown frequency mode', method({ frequency: { mode: 'weekly' } }), /frequency.mode "weekly"/], ['an undeclared frequency field', method({ frequency: { mode: 'periodic', interval: 'P3M' } }), /unknown field "interval"/], ['a source missing its url', method({ source: { ...source(), url: undefined } }), /missing required field "url"/], @@ -161,6 +164,31 @@ for (const [what, m, re] of methodCases) { }); } +const topLevelCases = [ + ['methods', 'a numeric version', { version: 1, methods: [] }, /version must be a non-empty string/], + ['methods', 'an empty version', { version: '', methods: [] }, /version must be a non-empty string/], + ['methods', 'a non-string description', { version: '1.0', description: 5, methods: [] }, /description must be a string/], + ['links', 'a numeric version', { version: 1, framework: 'fw', links: [] }, /version must be a non-empty string/], + ['links', 'an empty version', { version: '', framework: 'fw', links: [] }, /version must be a non-empty string/], +]; + +for (const [kind, what, doc, re] of topLevelCases) { + test(`a ${kind} file with ${what} fails — validator and schema agree`, () => { + const r = kind === 'methods' + ? checkVerification(withMethodsDoc(doc)) + : run({ linkFiles: { 'fw.json': doc } }); + assert.ok(errorsMatch(r, re), `expected ${re}, got ${JSON.stringify(r.errors)}`); + assert.equal((kind === 'methods' ? validMethodsDoc : validLinksDoc)(doc), false, 'schema should reject it too'); + }); +} + +/** A throwaway repository whose methods file is `doc` verbatim. */ +function withMethodsDoc(doc) { + const root = repo(); + fs.writeFileSync(path.join(root, 'data/verification-methods.json'), JSON.stringify(doc)); + return root; +} + test('method ids are unique', () => { const r = run({ methods: [method(), method({ name: 'Another' })] }); assert.ok(errorsMatch(r, /VM-0001 is used more than once/)); @@ -169,7 +197,8 @@ test('method ids are unique', () => { // ─── Methods: source ────────────────────────────────────────────────────────── test('source text is optional and not tied to a licence list', () => { - // Whether a licence allows reproducing text is checked in review, not here. + // Whether a licence allows reproducing text is checked in review, not here — + // except a licence that plainly forbids it (see the proprietary cases above). for (const m of [ method({ source: source({ license: 'CC-BY-SA-4.0' }) }), method({ source: source({ license: 'Proprietary', text: undefined }) }),