Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,17 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
scope changes (LLM01 cross-modal 2.2.3/2.2.4, LLM04 artifact provenance 3.1.1/3.1.3, LLM05 fine-tuning subversion
6.1.2/3.5.1, LLM08 hidden context 10.2.4, LLM10 generated code 9.3.7). AISVS has no requirement for scanning
generated code itself; the LLM10 section says so.
- **Verification methods data model** (#191, follow-up to #101 / #186): how to check that a control is
implemented, held as data. `data/verification-methods.json` holds each method once (`VM-NNNN` ids,
`examine` / `interview` / `test`, procedure, expected result, provenance, review state).
`data/verification-links/<framework-id>.json` links methods to controls, at control level or on one risk–control
row, each link with its own review. Both have schemas (`verification-methods-schema.json`,
`verification-links-schema.json`), and `scripts/validate.js` enforces them plus the cross-file rules through
`scripts/verification.js` (tested in `scripts/verification.test.mjs`). A source's original wording is optional and
included only where its licence allows; otherwise the method cites the source by id and url. Licence
compatibility is judged in review, not by the validator. v1 is data, schemas and validator only: no generator,
export or webapp changes. Documented in `docs/VERIFICATION_METHODS.md`. No methods yet; the pilot follows in a
separate PR.

### Changed

Expand Down
4 changes: 4 additions & 0 deletions data/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ dashboard integration, and downstream tooling without scraping Markdown.
| `incidents-schema.json` | JSON Schema for incident entries |
| `incidents.json` | 50 real-world + research AI security incidents with MAESTRO layer attribution |
| `tools-supplement.json` | Supplemental tool entries merged into entries at generation time |
| `verification-methods.json` | Verification methods: how to check a control is implemented (see `docs/VERIFICATION_METHODS.md`) |
| `verification-methods-schema.json` | JSON Schema for verification methods |
| `verification-links/` | Links from verification methods to controls, one file per framework registry id |
| `verification-links-schema.json` | JSON Schema for a verification links file |
| `entries/` | 41 machine-readable JSON files — one per OWASP entry (LLM01–LLM10, ASI01–ASI10, DSGAI01–DSGAI21) |

---
Expand Down
45 changes: 45 additions & 0 deletions data/verification-links-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/verification-links-schema.json",
"title": "GenAI Security Crosswalk — Verification Links Schema",
"description": "Links between verification methods and controls, one file per framework: data/verification-links/<framework-id>.json. See docs/VERIFICATION_METHODS.md and issue #191.",
"type": "object",
"required": ["version", "framework", "links"],
"additionalProperties": false,
"properties": {
"version": { "type": "string", "minLength": 1 },
"framework": {
"type": "string",
"pattern": "^[a-z0-9]+(-[a-z0-9]+)*$",
"description": "Registry id (data/frameworks/<id>.json). Must equal the file name."
},
"links": {
"type": "array",
"description": "Sorted by control_id, then entry_id (control-level links first), then method_id.",
"items": { "$ref": "#/definitions/Link" }
}
},
"definitions": {
"Link": {
"type": "object",
"required": ["method_id", "control_id", "entry_id", "reviewed_by"],
"additionalProperties": false,
"properties": {
"method_id": { "type": "string", "pattern": "^VM-\\d{4}$" },
"control_id": { "type": "string", "minLength": 1, "description": "A control_id in the framework's registry." },
"entry_id": {
"description": "null: the link applies to the control wherever it is mapped. An entry id: only to that risk-control row.",
"oneOf": [
{ "type": "null" },
{ "type": "string", "pattern": "^(LLM\\d{2}|ASI\\d{2}|DSGAI\\d{2}|AST\\d{2})$" }
]
},
"reviewed_by": {
"type": "array",
"items": { "type": "string", "pattern": "\\S" },
"description": "Named humans who confirmed the method verifies this control or row. Empty means draft."
}
}
}
}
}
9 changes: 9 additions & 0 deletions data/verification-links/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Verification links

One file per framework, named by its registry id: `<framework-id>.json`, matching
`data/frameworks/<framework-id>.json`. Each file links verification methods from
`data/verification-methods.json` to that framework's controls.

The format is defined in `data/verification-links-schema.json` and explained in
[docs/VERIFICATION_METHODS.md](../../docs/VERIFICATION_METHODS.md).
`scripts/validate.js` checks every file here.
96 changes: 96 additions & 0 deletions data/verification-methods-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/verification-methods-schema.json",
"title": "GenAI Security Crosswalk — Verification Methods Schema",
"description": "Verification methods: how to check that a control is implemented. Each method is held once and linked to controls through data/verification-links/. See docs/VERIFICATION_METHODS.md and issue #191.",
"type": "object",
"required": ["version", "methods"],
"additionalProperties": false,
"properties": {
"version": { "type": "string", "minLength": 1 },
"description": { "type": "string" },
"methods": {
"type": "array",
"items": { "$ref": "#/definitions/Method" }
}
},
"definitions": {
"Method": {
"type": "object",
"required": ["id", "name", "method_type", "procedure", "expected_result", "source", "status", "reviewed_by"],
"additionalProperties": false,
"properties": {
"id": {
"type": "string",
"pattern": "^VM-\\d{4}$",
"description": "Project ID. Sequential, opaque, never reused or renumbered."
},
"name": { "type": "string", "minLength": 1, "maxLength": 80 },
"method_type": {
"enum": ["examine", "interview", "test"],
"description": "Assessment method (NIST SP 800-53A)."
},
"procedure": { "type": "string", "minLength": 1, "description": "What the assessor does." },
"expected_result": { "type": "string", "minLength": 1, "description": "What a pass looks like. Must be observable." },
"source": {
"description": "Source the method is derived from; null for a method authored in the project.",
"oneOf": [{ "type": "null" }, { "$ref": "#/definitions/Source" }]
},
"status": { "enum": ["draft", "reviewed", "deprecated"] },
"reviewed_by": {
"type": "array",
"items": { "type": "string", "pattern": "\\S" },
"description": "Named humans who reviewed the method, as for schema v2 mapping rows. Empty while draft."
},
"frequency": { "$ref": "#/definitions/Frequency" },
"evidence": { "type": "string", "minLength": 1, "description": "Artefact(s) the check produces or consumes." },
"review_date": { "type": "string", "format": "date", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" },
"status_note": {
"type": "string",
"minLength": 1,
"description": "Status context. Required when deprecated: the reason, and the replacement ID if there is one."
}
},
"allOf": [
{
"if": { "properties": { "status": { "const": "reviewed" } }, "required": ["status"] },
"then": { "properties": { "reviewed_by": { "minItems": 1 } } }
},
{
"if": { "properties": { "status": { "const": "deprecated" } }, "required": ["status"] },
"then": { "required": ["status_note"] }
}
]
},
"Source": {
"type": "object",
"required": ["name", "version", "id", "url", "license"],
"additionalProperties": false,
"properties": {
"name": { "type": "string", "minLength": 1 },
"version": { "type": "string", "minLength": 1 },
"id": { "type": "string", "minLength": 1, "description": "Identifier within the source, e.g. C02." },
"url": { "type": "string", "pattern": "^https?://" },
"license": {
"type": "string",
"minLength": 1,
"description": "SPDX identifier where one exists (e.g. CC-BY-SA-4.0); otherwise a short name such as Proprietary."
},
"text": {
"type": "string",
"minLength": 1,
"description": "The source's original wording, unchanged. Include only if the licence allows reproduction in this CC BY-SA 4.0 repository."
}
}
},
"Frequency": {
"type": "object",
"required": ["mode"],
"additionalProperties": false,
"description": "Recommended minimum frequency, not a requirement. An object so that interval or triggers can be added later.",
"properties": {
"mode": { "enum": ["continuous", "periodic", "event_driven"] }
}
}
}
}
5 changes: 5 additions & 0 deletions data/verification-methods.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"version": "1.0",
"description": "Verification methods: how to check a control is implemented. See docs/VERIFICATION_METHODS.md.",
"methods": []
}
120 changes: 120 additions & 0 deletions docs/VERIFICATION_METHODS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# Verification methods

A mapping row says that a control addresses a risk. A verification method says
how to check that the control is actually in place. Methods and their links to
controls are held as data, agreed in issue #191 after the pilot in #186.

| File | Holds | Schema |
|---|---|---|
| `data/verification-methods.json` | Each method, once | `data/verification-methods-schema.json` |
| `data/verification-links/<framework-id>.json` | Links methods to framework's controls | `data/verification-links-schema.json` |

Both are hand-edited source files. `scripts/validate.js` checks them on every
full run, through `scripts/verification.js`. Version 1 is data, schemas and
validation only: the generator, the exports and the webapp do not read these
files yet.

---

## Methods

| Field | Required | Content |
|---|---|---|
| `id` | yes | `VM-` and four digits. Sequential, never reused or renumbered. |
| `name` | yes | Short title, at most 80 characters |
| `method_type` | yes | One of: `examine`, `interview`, `test` (below) |
| `procedure` | yes | What the assessor does |
| `expected_result` | yes | What a pass looks like. Must be observable. |
| `source` | yes | The source the method is derived from, or `null` if it was written here |
| `status` | yes | One of: `draft`, `reviewed`, `deprecated` |
| `reviewed_by` | yes | Named humans who reviewed the method. Empty while `draft`. |
| `frequency` | no | Recommended minimum: `{ "mode": "continuous" }`, `periodic` or `event_driven` |
| `evidence` | no | Artefacts the check produces or consumes |
| `review_date` | no | `YYYY-MM-DD` of the latest review |
| `status_note` | no | Status context. Required when `deprecated`: the reason and any replacement id. |

`method_type` follows the assessment methods of NIST SP 800-53A. A method that
combines types records its main one.

| Value | Meaning |
|---|---|
| `examine` | Review, inspect or analyse artefacts: documents, configurations, logs, records |
| `interview` | Discuss with the people responsible for the control |
| `test` | Exercise the control under defined conditions and compare actual with expected behaviour |

### Sources and licences

`source` holds `name`, `version`, `id`, `url`, `license` and, optionally, `text`.
Write `license` as an SPDX identifier where one exists (`CC-BY-SA-4.0`).

- **Include `text`** (the source's original wording, unchanged) only if the
source's licence allows reproducing it in this CC BY-SA 4.0 repository. The
method's own fields are then adapted from it.
- **Otherwise cite by `id` and `url` only**, for example for proprietary
standards.

The validator does not judge licences; contributors and reviewers do, in the PR.

### Status and changes

- `reviewed` needs at least one name in `reviewed_by`. An agent never adds itself.
- An editorial change (clarity, typos) keeps the id. A change to what is tested
or to the pass criteria is a new method. The old one becomes `deprecated`, with
`status_note` naming the replacement, and its links are re-pointed in the same PR.
- Deprecated methods stay in the file, so an id never comes to mean something else.

---

## Links

A links file is named after a framework's registry id and declares it:

```json
{
"version": "1.0",
"framework": "cosai",
"links": [
{ "method_id": "VM-0001", "control_id": "WS4-AGT-2.2", "entry_id": null, "reviewed_by": [] },
{ "method_id": "VM-0002", "control_id": "WS4-AGT-2.2", "entry_id": "ASI03", "reviewed_by": [] }
]
}
```

| Field | Content |
|---|---|
| `method_id` | A method in `data/verification-methods.json` |
| `control_id` | A control in `data/frameworks/<framework-id>.json` |
| `entry_id` | `null` for a control-level link; an entry id (`LLM01`, `ASI04`, `DSGAI07`, `AST01`) for one risk–control row |
| `reviewed_by` | Named humans who confirmed the method verifies this control or row. Empty means draft. |

**Control level or row level?** Ask whether the method verifies the control the
same way regardless of the risk.

- **Yes:** `entry_id: null`. One link covers every row of the control.
- **No:** set `entry_id`. A control stating an outcome ("limit agent actions")
is usually checked differently for each risk it is mapped to.

Links are sorted by `control_id`, then `entry_id` with control-level links first,
then `method_id`. Sorting spreads parallel additions across the file, so they
rarely conflict.

---

## Validation

Errors fail the build:

- a field missing, undeclared, or outside its allowed values or pattern
- a duplicate method id or a duplicate link
- `reviewed` without a reviewer, or `deprecated` without a `status_note`
- a links file whose name differs from its `framework`, or with no registry
- a link to a method, control or (for row-level links) mapping row that does not
exist. Row-level links are matched through the registry's display name, since
mapping rows are keyed by name and the registry id is the stable key.
- links out of order

Warnings flag review states that would overstate each other:

- a reviewed link on a method that is still `draft`
- a reviewed row-level link on a mapping row with no reviewer
- a link to a `deprecated` method
17 changes: 17 additions & 0 deletions scripts/validate.js
Original file line number Diff line number Diff line change
Expand Up @@ -1112,6 +1112,22 @@ function checkEvidence() {
return bad === 0;
}

/**
* Verification methods and links (issue #191).
*
* data/verification-methods.json and data/verification-links/<framework-id>.json
* are hand-edited source files. The rules live in scripts/verification.js so the
* tests can drive them with fixtures; this only reports what it finds.
*/
function checkVerificationData() {
const { checkVerification } = require('./verification');
const r = checkVerification(ROOT);
for (const e of r.errors) fail(e.file, e.msg);
for (const w of r.warnings) warn(w.file, w.msg);
if (!r.errors.length) pass('Verification', r.summary);
return r.errors.length === 0;
}

function run() {
const args = process.argv.slice(2);
const quickMode = args.includes('--quick');
Expand Down Expand Up @@ -1165,6 +1181,7 @@ function run() {
checkControlIdShapes();
checkIncidentIds();
checkEvidence();
checkVerificationData();
}

// Encoding guard — mapping files plus the shared/root markdown they link to
Expand Down
Loading