diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..58b3645 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +# The syntax fixture corpus is bytes, not text. +# +# `packages/core/tests/fixtures/syntax/` holds three files that differ from one another only in +# their line endings, and the CI matrix runs the suite on windows-latest, where git's default +# `core.autocrlf=true` rewrites every LF on checkout. Under that rewrite `line-endings-lf.idf` and +# `line-endings-mixed.idf` both arrive as CRLF, the three fixtures become one fixture, and the tests +# written for the distinction keep passing while measuring nothing. +# +# `-text` disables all end-of-line conversion for these paths in both directions. +packages/core/tests/fixtures/syntax/*.idf -text diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30a3c96..35bdedc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -229,6 +229,50 @@ jobs: - run: npm test + budget: + name: The performance budget + runs-on: ubuntu-latest + + # ITS OWN JOB, and deliberately not a step inside `test`. The language service exists so that + # an answer about the cursor costs the statement rather than the file, and a requirement with + # no measurement behind it is a hope: bench/budget.mjs is the measurement, and it fails. + # + # Why it is not in the test suite. A timing threshold wired into the same run as the + # correctness tests means a noisy neighbour on the runner blocks a documentation typo, and a + # gate that fails work it has nothing to do with is a gate somebody switches off within the + # month. Here a breach reports against itself and names the ratio that moved. + # + # Why it can fail on a shared runner at all. Every enforced number is a RATIO between two + # figures measured in the same process on the same machine in the same run: a cursor answer + # against parseIdf over the same text, scanIdf against parseIdf, parseIdf against lex, the + # same cursor answer on the reference model against a file one hundredth its size, and the + # same answer at the two ends of one comment-free file. A slower runner moves both halves + # together and the ratio does not move. Absolute milliseconds are printed for a human to read + # and nothing is held against them. + # + # The last two are the ones that pin the design, and they pin different axes. A ratio against + # parseIdf could be met by a merely fast reparse; independence from file SIZE could not, and a + # reparse shows a hundredfold difference on any machine. But size independence compares two + # files and cannot see a cost that grows with the OFFSET, because both of its readings sit at + # the same place in their own file. That is a real defect rather than a hypothetical one: a + # backward search for an exclamation mark reads the whole prefix in a file that holds none, + # and it measured 925x here while every other gate on this list stayed green. + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + # After the build: the benchmark reads dist, which is what an npm consumer receives, rather + # than the sources a bundler would transform. + - run: npx tsc --build + + - run: npm run check:bench + docs: name: Docs runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ca88c7e..622b98c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -133,6 +133,28 @@ jobs: npm pkg set "peerDependencies.@idfkit/core=>=$VERSION" -w "$pkg" done + # @idfkit/language peer-depends on core and carries the "0.0.0" placeholder in the + # repository for the same reason the type packages do. Published unchanged it would ask + # the registry for @idfkit/core@0.0.0, a version that has never existed, and every + # install of the language service would fail to resolve its peer. + # + # EXACT, not a caret, and contracts/language-service.md is explicit about why: the + # service reads the syntax layer core builds, so a pair that disagrees about the layer's + # shape puts findings on the wrong characters SILENTLY. Nothing fails; the underline is + # simply in the wrong place. A caret would permit exactly that pairing across a core + # patch release, and the whole argument for splitting the two packages rests on there + # being no supported install in which they differ. + # + # This sits beside, and does not contradict, the facade's caret on core a few lines + # below, which contracts/distribution.md requires so that a patch of core reaches a + # consumer without a facade release. npm resolves peers as part of building the tree, + # so `idfkit`'s "^X.Y.Z" and this "X.Y.Z" are satisfied together by X.Y.Z, and the + # narrow case where they cannot be is one where some other dependency has already + # forced core above the version this service was built against. Refusing to resolve is + # the right answer there: it is the pairing the exact range exists to reject, and a + # failed install is better than an underline in the wrong place. + npm pkg set "peerDependencies.@idfkit/core=$VERSION" -w @idfkit/language + # The facade's own dependency specs, which `npm version` does NOT touch: it rewrites # the "version" field of each workspace and nothing else. In the repository they read # "0.0.0", which is what makes the workspace link resolve. Published unchanged, they @@ -145,6 +167,7 @@ jobs: npm pkg set "dependencies.@idfkit/core=^$VERSION" -w idfkit npm pkg set "dependencies.@idfkit/schemas=^$VERSION" -w idfkit npm pkg set "peerDependencies.@idfkit/weather=^$VERSION" -w idfkit + npm pkg set "peerDependencies.@idfkit/language=^$VERSION" -w idfkit - name: Publish if: env.PUBLISH == 'true' @@ -157,7 +180,8 @@ jobs: # already-published package, so a release that failed partway can # never be completed by re-running it. # schemas first: core depends on it by exact version. weather has no - # workspace dependencies, so its position does not matter. The type packages go last: + # workspace dependencies, so its position does not matter. language peer-depends on + # core and so follows it. The type packages go last: # their peer range resolves against a core that is on the registry by then, so a # consumer installing one never sees an unsatisfiable peer. # npm applies the `latest` dist-tag on every publish unless told otherwise, and it @@ -172,7 +196,7 @@ jobs: esac echo "Publishing $VERSION under the '$NPM_TAG' dist-tag." - for pkg in @idfkit/schemas @idfkit/core @idfkit/weather \ + for pkg in @idfkit/schemas @idfkit/core @idfkit/weather @idfkit/language \ @idfkit/types-v26-1 @idfkit/types-v9-4; do if npm view "$pkg@$VERSION" version >/dev/null 2>&1; then echo "$pkg@$VERSION is already published; skipping." @@ -241,6 +265,7 @@ jobs: npm pkg set "dependencies.@idfkit/core=^$VERSION" -w idfkit npm pkg set "dependencies.@idfkit/schemas=^$VERSION" -w idfkit npm pkg set "peerDependencies.@idfkit/weather=^$VERSION" -w idfkit + npm pkg set "peerDependencies.@idfkit/language=^$VERSION" -w idfkit # npm applies the `latest` dist-tag on every publish unless told otherwise, and it # does so for a semver prerelease exactly as for a stable version: nothing about the # `-rc.1` in a version reaches the registry's idea of what `npm install ` should diff --git a/CHANGELOG.md b/CHANGELOG.md index 579ddfe..95449e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,113 @@ The packages in this repository, `@idfkit/core`, `@idfkit/schemas`, and ## [Unreleased] +This release stays on `conformance-2026.8`, the corpus level `CONFORMANCE_LEVEL` +reports, and adds no cases to it. A capability that exists in one language +asserts no cross-language agreement, so there is nothing for the corpus to +compare; the parity ledger carries the absence instead. The governance level +moves to `governance-2026.11`, which is where the names below are registered and +where that ledger entry lives. + +### Added + +- A syntax layer in `@idfkit/core`. `scanIdf(text)` returns a `SyntaxLayer`: + every statement, with the region of its type name and of each field it was + written with, plus every meaningful token in source order, packed. It takes + text and nothing else, because the layer records what the text says and never + what it means, and it never throws for any input. Text that breaks the grammar + is represented rather than stopped at: an unterminated final statement runs to + the end of the input and says so, a statement written with no type name still + gets a region, and empty text produces an empty layer. + + Nothing builds it implicitly. `parseIdf` and `lex` read the same characters + through the same scan and construct none of it, so **a caller who never names + `scanIdf` pays neither its time nor its memory**, and reading a file costs what + it did before. + + `classify(layer)` is that layer read for drawing: the stored tokens with the + gaps between them filled as `trivia`, so the sequence tiles the whole text with + no hole and no overlap, and no token crosses a line boundary, because no token + encoding in use can express one that does. It is a generator, so a consumer + colouring a viewport stops where it stops rather than materialising every token + in the file. `lineColumnAt` and `offsetAt` convert between an offset and a + 1-based line and column. + +- `@idfkit/language`, the opt-in language service for IDF text, reachable under + the shared name at `idfkit/language`. It answers a cursor and positions + findings, and does nothing else: + + - `contextAt(text, offset, schema?)` reports what the cursor is on: the + statement it is in, whether the offset falls on the type name, in a field, + inside a comment, or between statements, and which field it is. + - `completionsAt`, `explainAt`, and `declarationAt` answer what may be written + here, what this means in the schema's own words, and where the name under the + cursor is declared. Each returns a discriminated union rather than a list + that is sometimes empty, because "the schema permits anything here" and "I + could not consult a schema" are different states, and an editor that rendered + them alike would look broken in the first case and be silently wrong in the + second. + - `findingsIn(text, schema)` reads, validates, and gives every finding a region + plus the precision of that region, `field` or `statement`. + `position(findings, layer, schema)` does the same for findings a caller + already holds, so a consumer with its own parse pays for one scan rather than + a second read. No validator changed and no finding is filtered or reworded: + correlation is a separate step over the layer, and a caller that never asks + for a position receives exactly what it received before. + + Everything here is synchronous, free of input and output, and holds no state. + There is no service object to construct, because a service object is where + state would accumulate, and every answer takes the text itself rather than a + path, since an editor's buffer differs from the file on disk whenever there are + unsaved changes. The same code therefore runs unchanged in Node, in a browser, + in a browser worker, and behind an editor server. Nothing here imports or names + a type from any editor protocol, and nothing here ever will; a consumer + translates. + + An answer costs the statement rather than the file. The statement containing an + offset is found by scanning backwards to the nearest semicolon that is not + inside a comment, so there is no reparse, no incremental parser, and no cache. + A committed measurement under `bench/` holds that to ratios rather than to + milliseconds, which is the form that survives being run on someone else's + machine: a cursor answer at most 2 percent of `parseIdf` over the same text, + and `scanIdf` at most 1.25 times it. + + **It is not installed by default.** `npm install idfkit` places zero bytes of + the service on disk, exactly as it places no weather code. Add + `@idfkit/language` by name to get it, and importing `idfkit/language` without + it names the package to install rather than failing with + `ERR_MODULE_NOT_FOUND`. + + There is no Python counterpart, and there is not going to be one. The answers + are computed from byte offsets into the source text, and a second + implementation of that arithmetic is the drift the corpus is least able to + police, since it compares findings on `(code, line, typeName)` and never on a + column. The decision is on the parity ledger as `idf-language-service`, at + `never`, which is terminal: adding a counterpart takes a constitutional + amendment rather than an edit. What it costs a reader is stated rather than + implied. These answers need a JavaScript runtime, and `pip install idfkit` + alone does not provide them. + +### Changed + +- `ParseDiagnostic.column` is filled. It was declared in 0.2.0-rc.2 and left + undefined because the lexer counted lines and not columns; the shared scan + counts both, and a reading finding now reports the column its statement begins + at, taken at the first non-blank character and counted from 1. The field is + still optional, so nothing that treated it as absent breaks. + + A column is the one location the two libraries measure in different units, and + that is registered rather than left to be found: Python counts code points and + JavaScript counts UTF-16 code units, so the two agree everywhere except in text + containing an astral character, which in practice means an emoji in a comment. + Each unit is the one its own ecosystem's editors want, so neither is converted. + Nothing compares columns across the two libraries today; the corpus matches + findings on `(code, line, typeName)`. + +- `lex` and `parseIdf` read through the same scan the syntax layer is built from. + Both keep their surface and their behaviour; there is now one scanner rather + than two, which is what keeps a position the layer reports and a position a + finding reports from drifting apart. + ## [0.2.0-rc.2] - 2026-09-04 ### Added @@ -51,11 +158,14 @@ The packages in this repository, `@idfkit/core`, `@idfkit/schemas`, and on `message`: the corpus compares findings on `(code, line, typeName)` and never on wording. - `column` and `filepath` are declared but not yet filled: the lexer counts - lines and not columns, and `parseIdf` takes text rather than a path, so - neither value exists at the point a finding is built. They are optional, so a - reader must treat them as absent until the lexer tracks a column and the - file-reading edge attaches the path it read from. + Both are optional, and each is filled at the one place that knows the value. + `parseIdf` takes text and cannot know where the text came from, so `filepath` + is stamped by the Node file-reading edge: `loadIdf` and + `loadIdfWithDiagnostics` attach the path they read to every finding, on the + result and on the error alike. A caller parsing a string still gets none, + because a string names no file. `column` is filled by the reader itself, from + the statement's first non-blank character, once the shared scan arrived to + count it (see Unreleased). ### Fixed diff --git a/bench/budget.mjs b/bench/budget.mjs new file mode 100644 index 0000000..a69875c --- /dev/null +++ b/bench/budget.mjs @@ -0,0 +1,862 @@ +#!/usr/bin/env node +/** + * The committed performance budget (tasks T056 to T060). + * + * WHAT THIS IS FOR + * + * `contracts/performance-budget.md` opens with the problem it exists to solve: + * "Performance is a requirement here rather than a hope, so it needs a + * measurement that can fail. There is none today: the figures this feature was + * designed against were taken by hand and are recorded nowhere in the + * repository." This is that measurement. It reads the model `corpus.mjs` + * generates, times every path the contract names, in one process on one machine + * in one run, and exits non-zero when a ratio moves. + * + * WHY EVERY GATE IS A RATIO + * + * Absolute milliseconds cannot fail reliably. A continuous-integration runner + * varies by more than the margin being defended, so a wall-clock threshold is + * either loose enough to miss a real regression or tight enough to fail on a + * noisy neighbour, and a gate that fails randomly is a gate somebody disables. + * Every enforced number below is therefore a ratio between two figures measured + * in this same run, which cancels the machine out: if a cursor answer costs less + * than a stated fraction of a full read of the same text here, it costs less + * than that fraction everywhere. + * + * Milliseconds are still printed, because a human reading a run wants to know + * whether the machine is fast, and because SC-001's "under 10 ms at the 95th + * percentile" is stated in milliseconds. They are reported and never enforced, + * and the output says so. + * + * THE FOUR GATES, AND WHICH HAZARD EACH ONE GUARDS + * + * a cursor answer / parseIdf FR-033, SC-001. The answer must cost a small + * fraction of a full read of the same text. + * scanIdf / parseIdf SC-002. The syntax layer must not cost more + * than a read plus a quarter. + * parseIdf / lex FR-005, SC-002. `parseIdf` must not start + * building a layer for callers who never asked + * for one. See the note on this gate below. + * big file / small file FR-033, and the one that pins the design. A + * ratio against `parseIdf` alone could be met + * by a merely fast reparse. Independence from + * file size could not. + * + * WHY THE THIRD GATE IS DIVIDED BY `lex` AND NOT BY ITS RECORDED MILLISECONDS + * + * The contract records `parseIdf` at 39 to 50 ms, hand-measured. Held against + * that figure directly the gate would be wall-clock again, with all the failure + * modes described above: the regression it exists to catch is a parse that + * quietly acquires the layer's work, which on this machine would move 40 ms to + * about 60 ms, and no absolute threshold both catches a 1.5x move and survives a + * runner that is 1.5x slower. + * + * `lex` is the yardstick because it is the same read over the same bytes without + * any of the layer: same scanner, same text, same allocation profile. A machine + * that is slow at one is slow at the other, so the ratio holds across machines + * while a parse that started building a layer moves it by half again. + * + * The recorded milliseconds are kept in `RECORDED_MS` and printed beside the run + * because the contract records them, but they do not agree with this machine and + * were never going to: `lex` measures 13 to 14 ms here against a recorded 35 ms. + * So `CALIBRATION` records the ratio measured in the same sitting as the budget + * it defends, which is the only honest basis for a threshold. + * + * WHAT IT DOES NOT CATCH + * + * A regression in the scanner that both `lex` and `parseIdf` share moves both + * halves of the third gate together and passes. That is a real limit rather than + * an oversight, and the absolute milliseconds printed above the gates are what a + * human reads to see it. No within-run ratio can do better, and the alternative + * fails on Tuesdays. + * + * USAGE + * + * npm run bench or node bench/budget.mjs + * npm run check:bench the same thing, under the name CI calls + * + * It reads `dist`, which is what an npm consumer receives, so run `npx tsc + * --build` first. Exit 0 passed, 1 a gate broke, 2 it could not run. + */ + +import { existsSync } from 'node:fs'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { commentFreeModel, referenceModel, smallModel } from './corpus.mjs'; + +/* -------------------------------------------------------------------------- */ +/* The budgets, as data (FR-035) */ +/* -------------------------------------------------------------------------- */ + +/** + * @typedef {object} Budgets + * The enforced thresholds. Held here as an exported object rather than written + * into a sentence or hidden in a comparison, because FR-035 asks for budgets + * "next to the measurement as data, not in prose", so that changing one is a + * visible diff somebody has to defend in review. + * @property {number} cursorShareOfParse Largest share of `parseIdf` one cursor answer may cost. + * @property {number} scanOverParse Largest multiple of `parseIdf` that `scanIdf` may cost. + * @property {number} parseOverLex Largest multiple of `lex` that `parseIdf` may cost. + * @property {number} fileSizeIndependence Largest factor between the same answer on the two models. + * @property {number} offsetIndependence Largest factor between the two ends of one comment-free file. + * @property {number} cursorP95Ms SC-001's absolute figure. Reported, never enforced. + */ + +/** @type {Budgets} */ +export const BUDGETS = { + // 2% of a 40 ms read is under 1 ms, comfortably inside SC-001's 10 ms and + // comfortably inside a 60 Hz frame. Measured today at about 0.03%, so this is + // not a threshold the implementation is pressed against; it is the line below + // which the answer is bounded by the statement rather than by the file. + cursorShareOfParse: 0.02, + + // SC-002, verbatim. Measured today at about 0.50. + scanOverParse: 1.25, + + // Measured at 2.64 to 3.00 across five runs (see CALIBRATION). The gate sits + // 20% above the worst of those, which leaves room for a machine that balances + // the two differently, and well below the 4.4 a parse that built a layer inside + // itself would land at on the figures this run prints. + parseOverLex: 3.6, + + // Deliberate, and the reasoning matters more than the number. The same answer + // at the same probe offsets measures 1.65 to 1.68 (contextAt) and 1.39 to 1.49 + // (completionsAt) between the two models, and neither is a size effect: the probe + // statement is byte-identical in both files and sits behind a different + // preceding statement in each, so the backward scan starts a different distance + // out. Anything whose cost is linear in the file lands near 93, which is the + // ratio of the two models' sizes. The factor's whole job is to sit between + // those two populations, and 3 is an order of magnitude below the linear one + // while leaving 1.8x above the worst honest reading. + fileSizeIndependence: 3, + + // The same reasoning as above, turned on the other axis. `fileSizeIndependence` + // compares two files and so cannot see a cost that grows with the OFFSET rather + // than with the size: both of its readings sit at the same place in their own + // file. This one compares the first statement and the last of one file, and it + // uses a comment-free file because that is the shape that exposes the defect. + // + // Measured today at about 1. The defect this gate exists to catch measured + // 8,385 before it was fixed: `insideComment` searched backwards for an + // exclamation mark, which reads the whole prefix when the file holds none, so + // every answer cost grew with how far into the file the cursor sat. Nothing + // else in this script could see it, because the reference model puts a comment + // on nearly every line and the backward search always stopped within one. + offsetIndependence: 3, + + // SC-001. Reported, not enforced: an absolute millisecond figure is a statement + // about a reference machine, and this script does not know which machine it is + // on. Printed so a human can see the answer is three orders of magnitude inside + // it rather than one. + cursorP95Ms: 10, +}; + +/** + * @typedef {object} RecordedBaseline + * A hand-measured figure from `contracts/performance-budget.md`, in milliseconds. + * `null` where the contract records the path as new. + * @property {number | null} low + * @property {number | null} high + */ + +/** + * The contract's hand-measured baselines. Reported for humans, never enforced. + * + * They were taken on another machine in another sitting and they do not describe + * this one: `lex` is recorded at 35 ms and measures 13 to 14 ms here. They are + * printed anyway, unadjusted, because a baseline quietly rewritten to match the + * machine that failed against it is not a baseline. + * + * @type {Record} + */ +export const RECORDED_MS = { + lex: { low: 35, high: 35 }, + parseIdf: { low: 39, high: 50 }, + validateDocument: { low: 39, high: 39 }, + writeIdf: { low: 47, high: 47 }, + scanIdf: { low: null, high: null }, + classify: { low: null, high: null }, +}; + +/** + * @typedef {object} Observed + * The span a ratio was seen to move over when a budget above it was set. + * @property {number} runs How many runs it was watched across. + * @property {number} low + * @property {number} high + */ + +/** + * What the enforced ratios actually measured when the budgets above were set. + * + * A threshold is only defensible if somebody can see what was measured to reach + * it, and a threshold set against a figure nobody wrote down is a threshold that + * gets loosened the first time it fails. Medians, after discarded warm-up, on the + * machine and date below. + * + * @type {{ measured: string } & Record} + */ +export const CALIBRATION = { + measured: '2026-09-04, Apple Silicon laptop, Node 22.12, darwin arm64', + parseOverLex: { runs: 5, low: 2.64, high: 3.0 }, + scanOverParse: { runs: 5, low: 0.48, high: 0.5 }, + contextAtAcrossModels: { runs: 3, low: 1.65, high: 1.68 }, + completionsAtAcrossModels: { runs: 3, low: 1.39, high: 1.49 }, +}; + +/* -------------------------------------------------------------------------- */ +/* How much is measured */ +/* -------------------------------------------------------------------------- */ + +/** + * Warm-up and timed iterations for the whole-file paths. + * + * Warm-up is discarded rather than averaged in. The first call into a cold + * function measures the optimising compiler, not the code: `lex` costs 43 ms on + * its first run against 13 ms once it is warm, and a benchmark that folded that + * in would report a third of its number as start-up and call it a regression the + * day somebody reordered the file. + * + * 21 iterations, so the 95th percentile is the second largest rather than the + * largest, and one scheduler blip does not become the reported figure. + */ +const WHOLE_FILE = { warmup: 3, iterations: 21 }; + +/** + * The same, for a cursor answer, which costs microseconds rather than + * milliseconds and therefore needs far more samples to say anything about a + * percentile. + * + * Time-boxed rather than fixed, and the reason is the failure this gate exists + * to catch. An answer that costs 4 microseconds affords 3,000 samples in twelve + * milliseconds. An answer that regressed into reparsing the file costs 20 + * milliseconds, and the same 3,000 samples at each of ten offsets in two models + * would take three quarters of an hour: the gate would report the regression as + * a build timeout rather than as a broken ratio, and a timeout says nothing about + * which ratio moved. So each offset gets a wall-clock allowance, and the number + * of samples falls out of what one call costs. 20 is the floor because a + * percentile over fewer than that is not a percentile. + */ +const ANSWER = { warmupMs: 60, budgetMs: 60, minIterations: 20, maxIterations: 3000 }; + +/* -------------------------------------------------------------------------- */ +/* Loading what the build produced */ +/* -------------------------------------------------------------------------- */ + +/** + * Modules this reads, and the one instruction that fixes all of them. + * + * `dist`, not `src`, on the same grounds the example-file sweep gives: it is + * what an npm consumer receives. The language package is reached by module + * rather than through its index, which is a liberty a benchmark may take and a + * consumer may not: the index assembles the published surface, and this measures + * the implementation behind it. + */ +const BUILT = { + core: '../packages/core/dist/index.js', + coreNode: '../packages/core/dist/node.js', + cursor: '../packages/language/dist/cursor.js', + complete: '../packages/language/dist/complete.js', +}; + +/** + * @typedef {object} Built + * Everything the run calls, loaded from `dist`. + * @property {Function} lex + * @property {Function} parseIdf + * @property {Function} validateDocument + * @property {Function} writeIdf + * @property {Function} scanIdf + * @property {Function} classify + * @property {Function} getIdfVersion + * @property {Function} schemaFor + * @property {Function} contextAt + * @property {Function} completionsAt + */ + +/** + * Load them, or say the one thing that fixes an unbuilt checkout. + * + * Missing output is not a failed budget and must not be reported as one: a gate + * that says a ratio broke when what really happened is that nobody ran the build + * teaches a reader to distrust it. Hence the separate exit code. + * + * @returns {Promise} + */ +async function loadBuilt() { + for (const [name, specifier] of Object.entries(BUILT)) { + if (existsSync(fileURLToPath(new URL(specifier, import.meta.url)))) continue; + console.error(`could not run: ${name} is not built (${specifier})`); + console.error( + 'Run `npx tsc --build` first. This measures dist, which is what a consumer gets.' + ); + process.exit(2); + } + const core = await import(BUILT.core); + const coreNode = await import(BUILT.coreNode); + const cursor = await import(BUILT.cursor); + const complete = await import(BUILT.complete); + return { + lex: core.lex, + parseIdf: core.parseIdf, + validateDocument: core.validateDocument, + writeIdf: core.writeIdf, + scanIdf: core.scanIdf, + classify: core.classify, + getIdfVersion: core.getIdfVersion, + schemaFor: coreNode.schemaFor, + contextAt: cursor.contextAt, + completionsAt: complete.completionsAt, + }; +} + +/* -------------------------------------------------------------------------- */ +/* The harness */ +/* -------------------------------------------------------------------------- */ + +/** + * @typedef {object} Timing + * What one measured path cost, in milliseconds. + * @property {number} median + * @property {number} p95 + * @property {number} min + * @property {number} max + * @property {number} samples + */ + +/** + * Somewhere for every measured result to go. + * + * A call whose result is dropped is a call the optimiser may delete, and a + * benchmark that deleted the thing it was timing would report a very good + * number. Touching the result once, outside the timed region, is enough to keep + * every call alive. + */ +let sink = 0; + +/** @param {unknown} produced */ +function keep(produced) { + sink += produced === undefined || produced === null ? 0 : 1; +} + +/** + * @param {Float64Array} sorted + * @param {number} q + * @returns {number} + */ +function quantile(sorted, q) { + const index = Math.ceil(q * sorted.length) - 1; + return sorted[Math.min(sorted.length - 1, Math.max(0, index))]; +} + +/** + * @param {Float64Array} durations + * @returns {Timing} + */ +function summarize(durations) { + const sorted = Float64Array.from(durations).sort(); + return { + median: quantile(sorted, 0.5), + // The 95th percentile, never the mean. A mean hides exactly the failure this + // feature exists to prevent: an answer that is fast on average and + // occasionally takes 40 ms drops a frame, and the reader experiences the + // 40 ms rather than the average. + p95: quantile(sorted, 0.95), + min: sorted[0], + max: sorted[sorted.length - 1], + samples: sorted.length, + }; +} + +/** + * Time one whole-file path. + * + * @param {() => unknown} run + * @returns {Timing} + */ +function timeWholeFile(run) { + for (let i = 0; i < WHOLE_FILE.warmup; i += 1) keep(run()); + const durations = new Float64Array(WHOLE_FILE.iterations); + for (let i = 0; i < WHOLE_FILE.iterations; i += 1) { + const started = performance.now(); + const produced = run(); + durations[i] = performance.now() - started; + keep(produced); + } + return summarize(durations); +} + +/** + * Time one cursor answer at one offset, one call at a time. + * + * One call per timed region, rather than a thousand calls divided by a thousand. + * A percentile over a batch is a percentile over batches, which is the mean again + * wearing a different name, and the number SC-001 is about is what one answer + * costs when a reader is waiting for it. + * + * @param {(offset: number) => unknown} ask + * @param {number} offset + * @returns {Float64Array} + */ +function timeAnswerAt(ask, offset) { + // Warm up on a clock rather than a count, so a cheap answer gets thousands of + // calls to compile against and an expensive one still leaves the loop. + const warmupUntil = performance.now() + ANSWER.warmupMs; + let warmed = 0; + while (warmed < ANSWER.maxIterations && performance.now() < warmupUntil) { + keep(ask(offset)); + warmed += 1; + } + + // One warm call decides how many will fit in the allowance. + const probeStarted = performance.now(); + keep(ask(offset)); + const each = Math.max(performance.now() - probeStarted, Number.EPSILON); + const iterations = Math.min( + ANSWER.maxIterations, + Math.max(ANSWER.minIterations, Math.floor(ANSWER.budgetMs / each)) + ); + + const durations = new Float64Array(iterations); + for (let i = 0; i < iterations; i += 1) { + const started = performance.now(); + const produced = ask(offset); + durations[i] = performance.now() - started; + keep(produced); + } + return durations; +} + +/** + * @param {readonly Float64Array[]} parts + * @returns {Float64Array} + */ +function pool(parts) { + const total = parts.reduce((n, part) => n + part.length, 0); + const all = new Float64Array(total); + let at = 0; + for (const part of parts) { + all.set(part, at); + at += part.length; + } + return all; +} + +/* -------------------------------------------------------------------------- */ +/* What is measured, and where */ +/* -------------------------------------------------------------------------- */ + +/** + * @typedef {object} Probe + * One offset a cursor answer is asked at. + * @property {string} label What makes this offset worth asking at. + * @property {number} offset Where it is in that model's text. + * @property {boolean} shared Whether the same offset exists in both models, which is + * what makes it usable for the file-size-independence comparison. + */ + +/** + * The offsets one model is sampled at. + * + * The five from `cursors` are the awkward ones, and they are in the text because + * `corpus.mjs` put them there: the first statement of the file behind a 444-line + * comment header, the last vertex of the longest statement in the file, inside a + * comment, a blank line between two statements, and the trailing whitespace at + * end of file. The five from `probe` sit inside one statement that is written + * character for character the same way in both models, which is what makes the + * two files comparable at all. + * + * Only the probe offsets are `shared`. The header the first statement sits behind + * is 444 lines in one model and 12 in the other, so an answer asked there is + * measuring the header rather than the file, and comparing the two would report a + * size effect that is really a fixture difference. + * + * @param {import('./corpus.mjs').GeneratedModel} model + * @returns {Probe[]} + */ +function probesFor(model) { + return [ + ...Object.entries(model.cursors).map(([label, offset]) => ({ label, offset, shared: false })), + ...Object.entries(model.probe).map(([label, offset]) => ({ + label: `probe.${label}`, + offset, + shared: true, + })), + ]; +} + +/** + * @typedef {object} Answer + * One cursor answer, named and bound to the schema it is asked against. + * @property {string} name + * @property {(text: string, offset: number) => unknown} ask + */ + +/** + * Every cursor answer the service offers today. + * + * `explainAt` and `declarationAt` join this list when phases 8 and 9 land, one + * entry each, and every gate below then covers them without further edit. + * + * `completionsAt` is asked with no document, which is the keystroke path. Handing + * it one would fold in the reference-name question, which is a whole-document + * question by nature and is answered from a document the caller already holds + * rather than from one this service parses. + * + * @param {unknown} schema + * @param {Built} built + * @returns {Answer[]} + */ +function answersAgainst(schema, built) { + return [ + { name: 'contextAt', ask: (text, offset) => built.contextAt(text, offset, schema) }, + { name: 'completionsAt', ask: (text, offset) => built.completionsAt(text, offset, schema) }, + ]; +} + +/* -------------------------------------------------------------------------- */ +/* The gates */ +/* -------------------------------------------------------------------------- */ + +/** + * @typedef {object} Gate + * One enforced ratio. + * @property {string} name What is being divided by what. + * @property {number} measured This run's figure. + * @property {number} budget The line it must stay under, from `BUDGETS`. + * @property {'percent' | 'times'} unit How to print both. + * @property {string} because What a breach means, said in the failure. + */ + +/** + * @param {number} value + * @param {Gate['unit']} unit + * @returns {string} + */ +function inUnit(value, unit) { + return unit === 'percent' ? `${(value * 100).toFixed(3)}%` : `${value.toFixed(2)}x`; +} + +/* -------------------------------------------------------------------------- */ +/* The report */ +/* -------------------------------------------------------------------------- */ + +/** @param {number} ms @returns {string} */ +function msOf(ms) { + return ms >= 1 ? `${ms.toFixed(2)} ms` : `${ms.toFixed(4)} ms`; +} + +/** @param {string} path @returns {string} */ +function recordedOf(path) { + const recorded = RECORDED_MS[path]; + if (recorded === undefined || recorded.low === null) return 'new'; + return recorded.low === recorded.high + ? `${recorded.low} ms` + : `${recorded.low} to ${recorded.high} ms`; +} + +/** @param {import('./corpus.mjs').GeneratedModel} shape @returns {string} */ +function shapeOf(shape) { + return ( + `${shape.bytes.toLocaleString().padStart(9)} bytes ` + + `${shape.statements.toLocaleString().padStart(6)} statements ` + + `${shape.lines.toLocaleString().padStart(6)} lines ` + + `${shape.meaningfulTokens.toLocaleString().padStart(7)} tokens` + ); +} + +/* -------------------------------------------------------------------------- */ +/* The run */ +/* -------------------------------------------------------------------------- */ + +/** + * Measure everything, print it, and say whether a gate broke. + * + * Behind a function and behind the guard at the foot of the file, on the same + * terms as `corpus.mjs`: the budgets above are worth importing and reading, and + * importing them should not cost twenty seconds of measurement. + * + * @returns {Promise} the process exit code + */ +async function main() { + const built = await loadBuilt(); + const model = referenceModel(); + const variant = smallModel(); + const commentFree = commentFreeModel(); + + // `schemaFor`, not `bundle.load`, because that is the path a reader takes and + // it is the one that resolves a declared version onto a bundled one: the corpus + // declares 26.1 and the bundle carries 26.1.0. + const schema = await built.schemaFor(built.getIdfVersion(model.text)); + const answers = answersAgainst(schema, built); + + // Everything the timed regions need but do not measure: a document to validate + // and to write, and a layer to classify. Built once, outside every timer, so + // that `classify` reports what classification costs rather than what a scan + // plus a classification costs. + const document = built.parseIdf(model.text, schema).document; + const layer = built.scanIdf(model.text); + + /** @type {Record} */ + const wholeFile = { + lex: timeWholeFile(() => built.lex(model.text)), + parseIdf: timeWholeFile(() => built.parseIdf(model.text, schema)), + validateDocument: timeWholeFile(() => built.validateDocument(document)), + writeIdf: timeWholeFile(() => built.writeIdf(document)), + scanIdf: timeWholeFile(() => built.scanIdf(model.text)), + classify: timeWholeFile(() => { + let counted = 0; + for (const _token of built.classify(layer)) counted += 1; + return counted; + }), + }; + + /** + * Every cursor answer, at every offset, in both models. + * + * @type {Map, variant: Map }>} + */ + const answerTimings = new Map(); + const probes = probesFor(model); + const variantProbes = probesFor(variant).filter((probe) => probe.shared); + + for (const answer of answers) { + /** @type {Map} */ + const byOffset = new Map(); + for (const probe of probes) { + byOffset.set( + probe.label, + timeAnswerAt((offset) => answer.ask(model.text, offset), probe.offset) + ); + } + /** @type {Map} */ + const inVariant = new Map(); + for (const probe of variantProbes) { + inVariant.set( + probe.label, + timeAnswerAt((offset) => answer.ask(variant.text, offset), probe.offset) + ); + } + answerTimings.set(answer.name, { byOffset, variant: inVariant }); + } + + /** Every offset in the reference model, pooled. @param {string} name @returns {Timing} */ + const pooledAnswer = (name) => + summarize(pool([...(answerTimings.get(name)?.byOffset.values() ?? [])])); + + /** The shared probe offsets only, in one model or the other. @returns {Timing} */ + const pooledProbe = (name, where) => { + const timings = answerTimings.get(name); + const parts = + where === 'variant' + ? [...(timings?.variant.values() ?? [])] + : probes + .filter((probe) => probe.shared) + .map((probe) => timings?.byOffset.get(probe.label)) + .filter((part) => part !== undefined); + return summarize(pool(parts)); + }; + + /* ------------------------------------------------------------------------ */ + /* The gates, computed */ + /* ------------------------------------------------------------------------ */ + + /** @type {Gate[]} */ + const gates = []; + + for (const answer of answers) { + gates.push({ + name: `${answer.name} p95 / parseIdf median`, + measured: pooledAnswer(answer.name).p95 / wholeFile.parseIdf.median, + budget: BUDGETS.cursorShareOfParse, + unit: 'percent', + because: + 'FR-033 and SC-001. A cursor answer must cost a small fraction of a full read of the ' + + 'same text. The p95 is the numerator because the reader experiences the slow answer, ' + + "not the average one; parseIdf's median is the denominator because a denominator taken " + + 'at its own p95 would quietly widen the allowance.', + }); + } + + gates.push({ + name: 'scanIdf median / parseIdf median', + measured: wholeFile.scanIdf.median / wholeFile.parseIdf.median, + budget: BUDGETS.scanOverParse, + unit: 'times', + because: + 'SC-002. Building the syntax layer must stay within a quarter again of reading the file ' + + 'into a document, or the layer is not something a consumer can build on a whim.', + }); + + gates.push({ + name: 'parseIdf median / lex median', + measured: wholeFile.parseIdf.median / wholeFile.lex.median, + budget: BUDGETS.parseOverLex, + unit: 'times', + because: + 'FR-005 and SC-002. lex is the same read over the same bytes with none of the layer, so ' + + 'this ratio moves when parseIdf starts doing work a caller who never named scanIdf did ' + + 'not ask for. The recorded milliseconds are printed above and are not the gate; see the ' + + 'header for why an absolute threshold cannot do this job.', + }); + + for (const answer of answers) { + const big = pooledProbe(answer.name, 'model'); + const small = pooledProbe(answer.name, 'variant'); + gates.push({ + // Medians, not percentiles, and this is the one place the choice goes the + // other way. The p95 is what a reader experiences and is what the first + // gate enforces; this one is a statement about the shape of the cost + // function, and over microsecond samples the median is the stabler + // estimator of it by a wide margin. Across three runs the median ratio + // moved between 1.65 and 1.68 while the p95 ratio moved between 1.33 and + // 2.50. + name: `${answer.name}, ${model.bytes.toLocaleString()} bytes vs ${variant.bytes.toLocaleString()} bytes`, + measured: Math.max(big.median / small.median, small.median / big.median), + budget: BUDGETS.fileSizeIndependence, + unit: 'times', + because: + 'FR-033, and the assertion that pins the design. The same question at the same offsets ' + + 'in a statement written character for character the same way in both files. A design ' + + 'that reparsed, or that indexed the lines, or that consulted a SyntaxLayer, is linear ' + + "in the file and lands near the ratio of the two files' sizes, which is about 93.", + }); + } + + // The other axis: cost that grows with the offset rather than with the file. + // Both readings are in ONE file, so a size ratio cannot explain a breach, and + // the file carries no comments because that is the shape that exposes it. + for (const answer of answers) { + const near = summarize( + timeAnswerAt((offset) => answer.ask(commentFree.text, offset), commentFree.near) + ); + const far = summarize( + timeAnswerAt((offset) => answer.ask(commentFree.text, offset), commentFree.far) + ); + gates.push({ + name: `${answer.name}, last vs first statement, no comments`, + measured: Math.max(far.median / near.median, near.median / far.median), + budget: BUDGETS.offsetIndependence, + unit: 'times', + because: + 'FR-033, on the axis the two-model gate cannot see. Both readings are in the same file, ' + + 'so its size divides out and only the distance from the start differs. Any search that ' + + 'runs backwards to a character the file does not contain reads the whole prefix, and ' + + 'this ratio then grows with the file: it measured 8,385 when insideComment searched ' + + 'backwards for an exclamation mark. The reference model cannot show it, because a ' + + 'comment on nearly every line stops that search within one.', + }); + } + + /* ------------------------------------------------------------------------ */ + /* The report, printed */ + /* ------------------------------------------------------------------------ */ + + console.log('idfkit language-service performance budget (FR-033 to FR-035, SC-001, SC-002)'); + console.log(''); + console.log(` reference model ${shapeOf(model)}`); + console.log(` variant ${shapeOf(variant)}`); + console.log(` schema EnergyPlus ${schema.version}`); + console.log(` node ${process.version} on ${process.platform} ${process.arch}`); + console.log(''); + console.log(' MILLISECONDS BELOW ARE INFORMATIONAL. They describe this machine and nothing'); + console.log(' else, and no threshold is held against them. THE RATIOS FURTHER DOWN ARE THE'); + console.log(' GATE: each is measured inside this one run, so a faster or slower machine moves'); + console.log(' both halves of it together.'); + console.log(''); + console.log( + ` ${'path'.padEnd(20)}${'median'.padStart(12)}${'p95'.padStart(12)}` + + `${'samples'.padStart(10)} recorded` + ); + for (const [path, timing] of Object.entries(wholeFile)) { + console.log( + ` ${path.padEnd(20)}${msOf(timing.median).padStart(12)}${msOf(timing.p95).padStart(12)}` + + `${String(timing.samples).padStart(10)} ${recordedOf(path)}` + ); + } + for (const answer of answers) { + const timing = pooledAnswer(answer.name); + console.log( + ` ${answer.name.padEnd(20)}${msOf(timing.median).padStart(12)}` + + `${msOf(timing.p95).padStart(12)}${String(timing.samples).padStart(10)} new` + ); + } + console.log(''); + console.log(' cursor answers, 95th percentile in ms, by offset'); + console.log(` ${'offset'.padEnd(24)}${answers.map((a) => a.name.padStart(16)).join('')}`); + for (const probe of probes) { + const cells = answers.map((answer) => { + const durations = answerTimings.get(answer.name)?.byOffset.get(probe.label); + return (durations === undefined ? '' : summarize(durations).p95.toFixed(4)).padStart(16); + }); + console.log(` ${probe.label.padEnd(24)}${cells.join('')}`); + } + console.log(''); + console.log(' GATES. These are what fails.'); + console.log(''); + console.log(` ${'gate'.padEnd(56)}${'measured'.padStart(12)}${'budget'.padStart(12)}`); + + /** @type {Gate[]} */ + const broken = []; + for (const gate of gates) { + const failed = gate.measured > gate.budget; + if (failed) broken.push(gate); + console.log( + ` ${gate.name.padEnd(56)}${inUnit(gate.measured, gate.unit).padStart(12)}` + + `${inUnit(gate.budget, gate.unit).padStart(12)} ${failed ? 'BROKE' : 'pass'}` + ); + } + console.log(''); + + const worstAnswerP95 = Math.max(...answers.map((answer) => pooledAnswer(answer.name).p95)); + console.log( + ` reported, not enforced: SC-001 asks for a cursor answer under ${BUDGETS.cursorP95Ms} ms at` + + ` the 95th\n percentile. The slowest answer here is ${msOf(worstAnswerP95)}, on this machine.` + ); + console.log(''); + + // Reading the sink keeps every measured call reachable, so that nothing timed + // above could have been optimised away for having an unused result. + if (!Number.isFinite(sink)) { + console.log('::error::the measurement lost its results, so nothing above was measured'); + return 2; + } + + if (broken.length > 0) { + for (const gate of broken) { + console.log( + `::error::${gate.name} measured ${inUnit(gate.measured, gate.unit)} against a budget of ` + + `${inUnit(gate.budget, gate.unit)}, which is ${(gate.measured / gate.budget).toFixed(2)}x ` + + 'its budget' + ); + console.log(` ${gate.because}`); + console.log(''); + } + console.log( + `FAIL: ${broken.length} of ${gates.length} gates broke. ` + + 'Milliseconds move with the machine and these do not, so a breach here is a change in ' + + 'this repository rather than a change in the runner.' + ); + return 1; + } + + // The last two batches of gates, each one per answer: file-size independence + // first, then offset independence. Reported apart because they are different + // claims and one passing says nothing about the other. + const worstOf = (batch) => Math.max(...batch.map((gate) => gate.measured)); + const bySize = worstOf(gates.slice(-2 * answers.length, -answers.length)); + const byOffset = worstOf(gates.slice(-answers.length)); + console.log( + `PASS: all ${gates.length} gates hold. The slowest cursor answer costs ` + + `${inUnit(worstAnswerP95 / wholeFile.parseIdf.median, 'percent')} of a full read of the ` + + `same text, costs within ${inUnit(bySize, 'times')} of the same in a file a hundredth ` + + `the size, and within ${inUnit(byOffset, 'times')} of the same at the far end of a ` + + 'comment-free file.' + ); + return 0; +} + +const invokedDirectly = + process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (invokedDirectly) process.exit(await main()); diff --git a/bench/corpus.mjs b/bench/corpus.mjs new file mode 100644 index 0000000..1ef1449 --- /dev/null +++ b/bench/corpus.mjs @@ -0,0 +1,891 @@ +#!/usr/bin/env node +/** + * The reference model the performance budget is measured against (task T001). + * + * WHY IT IS GENERATED AND NOT COMMITTED + * + * `contracts/performance-budget.md`: "The generator is committed. A benchmark + * against a file somebody has locally is not a benchmark." A 643 KB blob in the + * tree would work too, and would be worse in two ways: it cannot be regenerated + * at a different size without a second blob, and nobody reviewing a change to it + * can tell what moved. A generator is 20 KB of source whose diff is readable. + * + * WHY THE SEED IS FIXED AND `Math.random` IS BANNED HERE + * + * The budget is enforced as a RATIO measured within one run, and a ratio only + * cancels the machine out when both halves ran over the same bytes. Two runs + * that differ by a few hundred vertices differ by more than the margin being + * defended, so a benchmark seeded from the clock reports a different number + * every time and no regression is ever attributable. `makeRandom` below is a + * mulberry32, twelve lines, no dependency, byte-identical on every Node that + * has `Math.imul` — which is every Node this repository supports. + * + * WHAT THE SHAPE IS FOR + * + * The budget's pathological offsets (SC-001, task T059) are properties of the + * text, not of the harness, so the text has to contain them: + * + * a large comment header 444 lines of it, so the first statement is far + * from offset 0 and a cursor answer there cannot + * be cheap by accident + * many extensible repeats a `BuildingSurface:Detailed` whose vertices run + * for a hundred lines is the longest single + * statement a backward scan can land inside, and + * therefore its worst case + * trailing whitespace at EOF the `betweenStatements` state, which has no + * statement to scan back to at all + * + * `cursors` below hands those offsets to the benchmark rather than making it + * search for them, because a search that runs inside the timed region measures + * the search. + * + * THE FOUR TARGETS, AND WHICH ONES ARE EXACT + * + * The contract states four figures: 10,001 statements, 40,002 lines, roughly + * 643 KB, roughly 400,000 meaningful tokens. What this generator produces: + * + * statements 10,001 exact, asserted by `referenceModel` + * lines 40,002 exact, asserted by `referenceModel` + * size 641,576 B 0.2 percent under the stated 643 KB + * tokens 313,576 against a stated "roughly 400,000" + * + * The token figure is the one that misses, and it misses because the other three + * cannot all be met at once by text that still reads as EnergyPlus. 643 KB over + * 40,002 lines is 16 bytes a line; 400,000 tokens over the same lines is 10 + * tokens a line. An average line would have to carry ten tokens inside sixteen + * bytes, which only a dense run of single-character numeric fields does, and a + * file made of nothing else contains no name, no comment and no annotated field + * to measure the scanner against. So size is the figure held closest, because + * size is what every path being measured is linear in, and the token count is + * reported by `--stats` rather than engineered towards. + * + * USAGE + * + * node bench/corpus.mjs the reference model, to stdout + * node bench/corpus.mjs --small the one-hundredth-size variant + * node bench/corpus.mjs --stats the measured shape, as JSON + * + * and as a module: + * + * import { referenceModel, smallModel, probeOffsetsIn } from './corpus.mjs'; + */ + +import { pathToFileURL } from 'node:url'; + +/** The seed. Fixed forever; changing it invalidates every recorded baseline. */ +const SEED = 0x1dfc0de; + +/** Statements in the reference model, the Version statement included. */ +const FULL_STATEMENTS = 10001; + +/** Lines in the reference model. Every line, the comment header's included. */ +const FULL_LINES = 40002; + +/** + * Comment-header lines in the variant. + * + * Not a scaled-down 444: the preamble and its closing rule are nine lines, and + * they are the same nine lines in both files, so this is close to the smallest + * header that still says what the file is. + */ +const SMALL_HEADER_LINES = 12; + +/** + * How many statements of each shape the reference model holds. + * + * Counts rather than weights, so the composition is a reviewable number instead + * of the outcome of a thousand coin flips. The variant draws from this same list + * and stops early rather than scaling it, so the statements it holds are the same + * statements, not merely the same mix. + * + * The counts are what land the size on the contract's 643 KB. One-line statements + * dominate because their average length decides the file's size almost by itself; + * vertex runs are the only construct dense enough to keep the token count near + * the stated figure; annotated statements are the fewest and the most expensive + * per statement, and are here because they are the only shape that puts a comment + * between a value and the separator after it. + */ +const COMPOSITION = [ + { shape: 'oneLiner', count: 8719 }, + { shape: 'surface', count: 1000 }, + { shape: 'columnar', count: 200 }, + { shape: 'annotated', count: 80 }, +]; + +/** + * @typedef {object} ProbeOffsets + * Offsets into one model's text for the probe statement, which is the same text + * in the reference model and in the variant. `bench/budget.mjs` asks the same + * cursor question at each of these in both files and requires the two answers to + * cost within a small factor of each other; that is the check that a cursor + * answer's cost grows with the statement rather than with the file. + * @property {number} statementStart Offset of the probe statement's first character. + * @property {number} typeName Offset inside its type name. + * @property {number} fixedField Offset inside a fixed field's value. + * @property {number} extensibleValue Offset inside the ninth vertex, deep in the extensible run. + * @property {number} comment Offset inside one of its field comments. + */ + +/** + * @typedef {object} CursorSamples + * The offsets the percentile run samples, named for what makes each awkward. + * @property {number} firstStatement Inside the first statement, behind the comment header. + * @property {number} largeExtensible Inside the last vertex of the longest statement in the file. + * @property {number} insideComment Inside the comment header. + * @property {number} betweenStatements Inside a blank line between two statements. + * @property {number} trailingWhitespace The end of the text, past the last terminator. + */ + +/** + * @typedef {object} GeneratedModel + * @property {string} text The model itself. + * @property {number} statements Statements it holds, the Version statement included. + * @property {number} lines Lines it holds. + * @property {number} bytes Its length in UTF-8 bytes. + * @property {number} meaningfulTokens Tokens that are not whitespace, counted by `countMeaningfulTokens`. + * @property {ProbeOffsets} probe Where the probe statement sits in this text. + * @property {CursorSamples} cursors Offsets worth sampling in this text. + */ + +/** + * mulberry32. Deterministic, uniform enough for choosing between shapes, and + * small enough to read in one sitting, which a dependency would not be. + * + * @param {number} seed + * @returns {() => number} the next value in [0, 1) + */ +function makeRandom(seed) { + let state = seed >>> 0; + return function next() { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * @param {() => number} random + * @param {number} lowest inclusive + * @param {number} highest inclusive + * @returns {number} + */ +function between(random, lowest, highest) { + return lowest + Math.floor(random() * (highest - lowest + 1)); +} + +/** + * @template T + * @param {() => number} random + * @param {readonly T[]} items + * @returns {T} + */ +function oneOf(random, items) { + return items[Math.floor(random() * items.length)]; +} + +/** @param {number} n @param {number} width @returns {string} */ +function padded(n, width) { + return String(n).padStart(width, '0'); +} + +/** + * A coordinate, kept short on purpose. + * + * Vertex runs are where the size and the token count are decided: a vertex and + * its commas are six tokens, and at one to three characters a coordinate they are + * the only construct in the format that gets near the contract's ratio of tokens + * to bytes. Writing them to six decimal places, which some exporters do, would + * triple the file for the same token count. + * + * @param {() => number} random + * @returns {string} + */ +function coordinate(random) { + const roll = random(); + if (roll < 0.88) return String(between(random, 0, 9)); + if (roll < 0.98) return String(between(random, 10, 99)); + return `${between(random, 0, 9)}.${between(random, 0, 9)}`; +} + +/* -------------------------------------------------------------------------- */ +/* The statement shapes */ +/* -------------------------------------------------------------------------- */ + +/** + * @typedef {object} Statement + * @property {readonly string[]} lines Its lines, each without its newline. + * @property {number} [extensibleLine] Index into `lines` of a line deep in an extensible run. + */ + +const SHORT_ZONE = (r, i) => `Zone,Z${padded(i, 5)},0,0,0,0;`; +const SHORT_CONSTRUCTION = (r, i) => + `Construction,C${padded(i, 5)},M${padded(between(r, 1, 400), 4)};`; +const SHORT_LIMITS = (r, i) => `ScheduleTypeLimits,L${padded(i, 4)},0,1;`; + +/** + * The one-line shapes, listed with repeats rather than with weights. + * + * A repeated entry is a weight written in the only unit that matters here, + * which is bytes: the size of the reference model is decided almost entirely by + * how long the average one-line statement is, because there are eight thousand + * of them. Short objects outnumber long ones in real files too, so the list is + * not distorted to hit a number. + */ +const ONE_LINERS = [ + SHORT_ZONE, + SHORT_ZONE, + SHORT_ZONE, + SHORT_ZONE, + SHORT_ZONE, + SHORT_ZONE, + SHORT_CONSTRUCTION, + SHORT_CONSTRUCTION, + SHORT_CONSTRUCTION, + SHORT_CONSTRUCTION, + SHORT_LIMITS, + SHORT_LIMITS, + (r, i) => `Zone,Z${padded(i, 5)},0,0,0,0,1,1;`, + (r, i) => `Zone,Z${padded(i, 5)},0,0,0,0,1,1;`, + (r, i) => `Zone,Z${padded(i, 5)},0,0,0,0,1,1;`, + (r, i) => `Zone,Z${padded(i, 5)},0,0,0,0,1,1;`, + (r, i) => `Construction,C${padded(i, 5)},M${padded(between(r, 1, 400), 4)},M0001;`, + (r, i) => `Schedule:Constant,S${padded(i, 4)},L0001,${between(r, 0, 1)};`, + (r, i) => `Material:NoMass,M${padded(i, 4)},Rough,2.3,0.9,0.8;`, + (r) => `Output:Meter,${oneOf(r, METERS)},${oneOf(r, FREQUENCIES)};`, + (r) => `Output:Variable,*,${oneOf(r, OUTPUT_VARIABLES)},${oneOf(r, FREQUENCIES)};`, +]; + +const OUTPUT_VARIABLES = [ + 'Zone Air Temperature', + 'Zone Air Humidity Ratio', + 'Surface Inside Face Temperature', + 'Zone Mean Radiant Temperature', +]; + +const METERS = ['Electricity:Facility', 'NaturalGas:Facility', 'Cooling:Electricity']; + +const FREQUENCIES = ['Timestep', 'Hourly', 'Daily', 'Monthly', 'RunPeriod']; + +/** + * A dense single-line object. The commonest shape in the file and the one that + * keeps the average line short. + * + * @param {() => number} random + * @param {number} index + * @returns {Statement} + */ +function oneLiner(random, index) { + return { lines: [oneOf(random, ONE_LINERS)(random, index)] }; +} + +/** + * A `BuildingSurface:Detailed` with a vertex run, written two vertices to the + * line, which is how a geometry exporter writes them. + * + * The vertex count is drawn wide on purpose: three in five surfaces have four + * vertices, three in ten have a few dozen, and better than one in ten runs to + * between a hundred and two hundred and eighty. That last case is the one the + * budget's worst offset lands in, and it is why the tail is here at all. + * + * @param {() => number} random + * @param {number} index + * @returns {Statement} + */ +function detailedSurface(random, index) { + const roll = random(); + const vertices = + roll < 0.58 ? 4 : roll < 0.88 ? between(random, 8, 40) : between(random, 100, 280); + const lines = [ + `BuildingSurface:Detailed,S${padded(index, 5)},${oneOf(random, SURFACE_TYPES)},` + + `C${padded(between(random, 1, 400), 5)},Z${padded(between(random, 1, 900), 5)},` + + `Outdoors,,SunExposed,WindExposed,,${vertices},`, + ]; + for (let pair = 0; pair < Math.ceil(vertices / 2); pair += 1) { + const remaining = vertices - pair * 2; + const written = remaining >= 2 ? 2 : 1; + const coordinates = []; + for (let v = 0; v < written; v += 1) { + coordinates.push(coordinate(random), coordinate(random), coordinate(random)); + } + const last = pair === Math.ceil(vertices / 2) - 1; + lines.push(`${coordinates.join(',')}${last ? ';' : ','}`); + } + return { lines, extensibleLine: lines.length - 1 }; +} + +const SURFACE_TYPES = ['Wall', 'Roof', 'Floor', 'Ceiling']; + +/** + * One field to a line with no annotation, which is the other way real files are + * written and the shape that pushes the line count up without pushing the byte + * count up with it. + * + * @param {() => number} random + * @param {number} index + * @returns {Statement} + */ +function columnar(random, index) { + const shape = oneOf(random, COLUMNAR_SHAPES); + const values = shape.values(random, index); + const lines = [`${shape.typeName},`]; + values.forEach((value, position) => { + lines.push(` ${value}${position === values.length - 1 ? ';' : ','}`); + }); + return { lines }; +} + +const COLUMNAR_SHAPES = [ + { + typeName: 'Zone', + values: (r, i) => [`Z${padded(i, 5)}`, '0', '0', '0', '0', '1', '1'], + }, + { + typeName: 'Material', + values: (r, i) => [`M${padded(i, 4)}`, 'MediumRough', '0.1', '0.6', '1400', '1000'], + }, + { + typeName: 'ZoneInfiltration:DesignFlowRate', + values: (r, i) => [ + `I${padded(i, 4)}`, + `Z${padded(between(r, 1, 900), 5)}`, + `S${padded(between(r, 1, 400), 4)}`, + 'AirChanges/Hour', + '', + '', + '', + '0.5', + ], + }, + { + typeName: 'Schedule:Compact', + values: (r, i) => [ + `S${padded(i, 4)}`, + 'Any Number', + 'Through: 12/31', + 'For: AllDays', + 'Until: 24:00', + String(between(r, 0, 1)), + ], + }, +]; + +/** + * The annotated shape: one field to a line with a `!- Field Name` comment beside + * it, which is what the EnergyPlus IDF editor writes and what most real files + * look like. + * + * It is the most expensive shape per statement and the least token-dense, so its + * share is small. It is not optional, though: it is the only shape that puts a + * comment between a value and the separator that follows it, which is the case + * the scanner most easily gets wrong. + * + * @param {() => number} random + * @param {number} index + * @returns {Statement} + */ +function annotated(random, index) { + const shape = oneOf(random, ANNOTATED_SHAPES); + const values = shape.values(random, index); + const lines = [`${shape.typeName},`]; + values.forEach(([value, fieldName], position) => { + const written = ` ${value}${position === values.length - 1 ? ';' : ','}`; + lines.push(`${written.padEnd(26, ' ')}!- ${fieldName}`); + }); + return { lines }; +} + +const ANNOTATED_SHAPES = [ + { + typeName: 'Lights', + values: (r, i) => [ + [`LIGHTS_${padded(i, 4)}`, 'Name'], + [`Z${padded(between(r, 1, 900), 5)}`, 'Zone or ZoneList Name'], + [`S${padded(between(r, 1, 400), 4)}`, 'Schedule Name'], + ['Watts/Area', 'Design Level Calculation Method'], + ['', 'Lighting Level {W}'], + ['10.76', 'Watts per Zone Floor Area {W/m2}'], + ['', 'Watts per Person {W/person}'], + ['0.0', 'Return Air Fraction'], + ['0.72', 'Fraction Radiant'], + ['0.18', 'Fraction Visible'], + ], + }, + { + typeName: 'People', + values: (r, i) => [ + [`PEOPLE_${padded(i, 4)}`, 'Name'], + [`Z${padded(between(r, 1, 900), 5)}`, 'Zone or ZoneList Name'], + [`S${padded(between(r, 1, 400), 4)}`, 'Number of People Schedule Name'], + ['People/Area', 'Number of People Calculation Method'], + ['', 'Number of People'], + ['0.05', 'People per Zone Floor Area {person/m2}'], + ['', 'Zone Floor Area per Person {m2/person}'], + ['0.3', 'Fraction Radiant'], + ], + }, + { + typeName: 'WindowMaterial:Glazing', + values: (r, i) => [ + [`GLZ_${padded(i, 4)}`, 'Name'], + ['SpectralAverage', 'Optical Data Type'], + ['', 'Window Glass Spectral Data Set Name'], + ['0.003', 'Thickness {m}'], + ['0.837', 'Solar Transmittance at Normal Incidence'], + ['0.075', 'Front Side Solar Reflectance at Normal Incidence'], + ['0.075', 'Back Side Solar Reflectance at Normal Incidence'], + ['0.898', 'Visible Transmittance at Normal Incidence'], + ], + }, +]; + +/** @type {Record number, index: number) => Statement>} */ +const SHAPES = { + oneLiner, + surface: detailedSurface, + columnar, + annotated, +}; + +/* -------------------------------------------------------------------------- */ +/* The probe statement */ +/* -------------------------------------------------------------------------- */ + +/** + * The one statement that is literal rather than generated. + * + * `contracts/performance-budget.md` requires the same cursor answer measured on + * the reference model and on a file one hundredth its size, "with the cursor in a + * statement of the same shape". Same shape is not enough to compare offsets + * against, so this is the same TEXT, inserted at the same fraction of the way + * through both files. What differs between the two measurements is then the size + * of the file around it, which is exactly the variable under test. + * + * Twelve vertices: long enough that the backward scan has real work to do, short + * enough that it is an ordinary statement rather than the worst case, which + * `cursors.largeExtensible` covers separately. + */ +const PROBE_LINES = [ + 'BuildingSurface:Detailed,', + ' PROBE_SURFACE, !- Name', + ' Wall, !- Surface Type', + ' PROBE_CONSTRUCTION, !- Construction Name', + ' PROBE_ZONE, !- Zone Name', + ' Outdoors, !- Outside Boundary Condition', + ' , !- Outside Boundary Condition Object', + ' SunExposed, !- Sun Exposure', + ' WindExposed, !- Wind Exposure', + ' 0.5, !- View Factor to Ground', + ' 12, !- Number of Vertices', + ' 0.0,0.0,3.0, !- Vertex 1', + ' 4.0,0.0,3.0, !- Vertex 2', + ' 8.0,0.0,3.0, !- Vertex 3', + ' 12.0,0.0,3.0, !- Vertex 4', + ' 12.0,4.0,3.0, !- Vertex 5', + ' 12.0,8.0,3.0, !- Vertex 6', + ' 8.0,8.0,3.0, !- Vertex 7', + ' 4.0,8.0,3.0, !- Vertex 8', + ' 0.0,8.0,3.0, !- Vertex 9', + ' 0.0,4.0,3.0, !- Vertex 10', + ' 0.0,2.0,3.0, !- Vertex 11', + ' 0.0,0.0,3.0; !- Vertex 12', +]; + +const PROBE_TEXT = `${PROBE_LINES.join('\n')}\n`; + +/** The first line of the probe, which is what `probeOffsetsIn` searches for. */ +const PROBE_ANCHOR = `${PROBE_LINES[0]}\n${PROBE_LINES[1]}`; + +/** + * Offsets inside `PROBE_TEXT`, relative to its own start. Computed once, from the + * text, so that editing a line above cannot silently move a cursor onto a comma. + * + * @type {ProbeOffsets} + */ +const PROBE_RELATIVE = { + statementStart: 0, + typeName: 'Building'.length, + fixedField: PROBE_TEXT.indexOf('Wall') + 2, + extensibleValue: PROBE_TEXT.indexOf('0.0,8.0,3.0') + 4, + comment: PROBE_TEXT.indexOf('!- Surface Type') + 3, +}; + +/** + * Where the probe statement sits in a model's text. + * + * Exported so a caller that has only the text, having written it out with + * `node bench/corpus.mjs > model.idf`, can still find it without searching for a + * shape by eye. + * + * @param {string} text + * @returns {ProbeOffsets} + */ +export function probeOffsetsIn(text) { + const start = text.indexOf(PROBE_ANCHOR); + if (start === -1) { + throw new Error('this text holds no probe statement; it did not come from bench/corpus.mjs'); + } + return { + statementStart: start, + typeName: start + PROBE_RELATIVE.typeName, + fixedField: start + PROBE_RELATIVE.fixedField, + extensibleValue: start + PROBE_RELATIVE.extensibleValue, + comment: start + PROBE_RELATIVE.comment, + }; +} + +/* -------------------------------------------------------------------------- */ +/* The comment header */ +/* -------------------------------------------------------------------------- */ + +const HEADER_PREAMBLE = [ + '!-Generator idfkit bench/corpus.mjs', + '!-Option SortedOrder', + '!', + '! Synthetic reference model for the idfkit language-service budget.', + '! Generated deterministically from a fixed seed. Do not edit by hand: the', + '! benchmark compares runs against each other, so an edited copy compares a', + '! model against a different model and reports a regression that is not one.', + '!', +]; + +const HEADER_FILLER = [ + '! Ordinary EnergyPlus objects follow:', + '!', + '! zones, constructions, materials,', + '!', + '! schedules, internal gains, output', + '!', + '! requests, and detailed surfaces', + '!', + '! whose vertex runs are the longest', + '!', + '! single statements in this file.', + '!', + '! The header is long on purpose. A', + '!', + '! cursor answer asked in the first', + '!', + '! statement scans back past all of it,', + '!', + '! and a scan that is cheap only for', + '!', + '! having started near offset zero is', + '!', + '! not the scan this design claims.', + '!', +]; + +/** + * Exactly `lineCount` lines of comment, deterministically. + * + * @param {number} lineCount + * @returns {string} + */ +function commentHeader(lineCount) { + if (lineCount < HEADER_PREAMBLE.length + 1) { + throw new Error( + `the body already fills the line budget: only ${lineCount} header lines are left, ` + + `and the preamble alone needs ${HEADER_PREAMBLE.length + 1}` + ); + } + const lines = [...HEADER_PREAMBLE]; + while (lines.length < lineCount - 1) { + lines.push(HEADER_FILLER[(lines.length - HEADER_PREAMBLE.length) % HEADER_FILLER.length]); + } + lines.push('!'); + return `${lines.join('\n')}\n`; +} + +/* -------------------------------------------------------------------------- */ +/* Building a model */ +/* -------------------------------------------------------------------------- */ + +/** + * The order the shapes are emitted in: the composition above, expanded into one + * entry per statement and then shuffled with the seeded generator. + * + * Real files are grouped by class and this one is not, deliberately. Grouping + * would put every long statement in one contiguous stretch, and a percentile + * sampled at even offsets would then be sampling one shape at a time. + * + * The variant draws from this same order rather than from a scaled-down copy of + * it, and stops early. That makes the variant a genuine prefix of the reference + * model's statement stream up to the probe, which is a stronger guarantee than + * "the same mix": the statements around the probe are the same statements. + * + * @param {() => number} random + * @returns {string[]} + */ +function shapeOrder(random) { + /** @type {string[]} */ + const order = []; + for (const { shape, count } of COMPOSITION) { + for (let i = 0; i < count; i += 1) order.push(shape); + } + for (let i = order.length - 1; i > 0; i -= 1) { + const j = Math.floor(random() * (i + 1)); + [order[i], order[j]] = [order[j], order[i]]; + } + return order; +} + +/** + * @typedef {object} BuildRequest + * How far to go, and how to finish the header. Exactly one of `statements` and + * `targetBytes` is given: the reference model is specified by its statement + * count, the variant by its size, because those are the terms the contract + * states each of them in. + * @property {number} [statements] Stop once this many statements are written, the Version statement included. + * @property {number} [targetBytes] Stop once the body reaches this many bytes. + * @property {number} [targetLines] Pad the comment header until the whole file has exactly this many lines. + * @property {number} [headerLines] Use exactly this many header lines. Ignored when `targetLines` is given. + */ + +/** + * @param {BuildRequest} request + * @returns {GeneratedModel} + */ +function build(request) { + const random = makeRandom(SEED); + const order = shapeOrder(random); + + /** @type {string[]} */ + const chunks = []; + let length = 0; + let lines = 0; + let statements = 0; + + /** @param {string} chunk @param {number} newlines */ + const emit = (chunk, newlines) => { + chunks.push(chunk); + length += chunk.length; + lines += newlines; + }; + + /** @param {readonly string[]} statementLines */ + const emitStatement = (statementLines) => { + emit(`${statementLines.join('\n')}\n`, statementLines.length); + statements += 1; + }; + + emitStatement(['Version, 26.1;']); + + // How far through the file we are, in whichever unit this request was made in. + const targetStatements = request.statements ?? Infinity; + const targetBytes = request.targetBytes ?? Infinity; + const progress = () => Math.max(statements / targetStatements, length / targetBytes); + + let probeWritten = false; + let longestVertexOffset = -1; + let longestVertexLength = 0; + let firstBlankLineOffset = -1; + let shapeIndex = 0; + + for (const shape of order) { + if (!probeWritten && progress() >= 0.6) { + emitStatement(PROBE_LINES); + probeWritten = true; + } + if (progress() >= 1) break; + const statement = SHAPES[shape](random, shapeIndex + 1); + if (statement.extensibleLine !== undefined && statement.lines.length > longestVertexLength) { + longestVertexLength = statement.lines.length; + longestVertexOffset = + length + + statement.lines.slice(0, statement.extensibleLine).reduce((n, l) => n + l.length + 1, 0) + + 1; + } + emitStatement(statement.lines); + // A blank line after every statement, which is what an exporter writes and + // what gives `betweenStatements` somewhere to land, and a second one now and + // then where a real file would start a new section. + const sectionBreak = random() < 0.09; + if (firstBlankLineOffset === -1) firstBlankLineOffset = length; + emit(sectionBreak ? '\n\n' : '\n', sectionBreak ? 2 : 1); + shapeIndex += 1; + } + + const body = chunks.join(''); + const headerLineCount = + request.targetLines === undefined ? (request.headerLines ?? 8) : request.targetLines - lines; + const header = commentHeader(headerLineCount); + const text = header + body; + + const shift = header.length; + const firstStatementOffset = shift + 'Vers'.length; + + return { + text, + statements, + lines: lines + headerLineCount, + bytes: Buffer.byteLength(text, 'utf8'), + meaningfulTokens: countMeaningfulTokens(text), + probe: probeOffsetsIn(text), + cursors: { + firstStatement: firstStatementOffset, + largeExtensible: shift + longestVertexOffset, + insideComment: HEADER_PREAMBLE[0].length + 4, + betweenStatements: shift + firstBlankLineOffset, + trailingWhitespace: text.length, + }, + }; +} + +/** + * The reference model: 10,001 statements and 40,002 lines, exactly. + * + * @returns {GeneratedModel} + */ +export function referenceModel() { + const model = build({ statements: FULL_STATEMENTS, targetLines: FULL_LINES }); + assertShape(model, FULL_STATEMENTS, FULL_LINES); + return model; +} + +/** + * The one-hundredth-size variant, holding the same probe statement. + * + * Size is what is scaled, because size is the word the contract uses: "the same + * cursor answer measured on the reference model and on a file one hundredth its + * size". Its statement count and its line count fall out of that rather than + * being pinned as well, which would over-specify a file whose only job is to be + * small and to contain the probe. + * + * The divisor is applied to the reference model's measured size rather than to a + * remembered constant, so the two stay one hundredth apart when the composition + * above is edited. That costs one extra build of the reference model, once, well + * outside anything the benchmark times. + * + * @returns {GeneratedModel} + */ +export function smallModel() { + const target = Math.round(referenceModel().bytes / 100); + return build({ targetBytes: target, headerLines: SMALL_HEADER_LINES }); +} + +/** + * The reference model with every comment removed, and its two extreme offsets. + * + * WHY THIS EXISTS. The composition above puts `!- Field Name` on very nearly + * every line, which is what a human-edited file looks like and is why the model + * is written that way. It also hides one whole class of defect: any backward + * search for an exclamation mark terminates within a line here, whatever the + * implementation, so a search that is unbounded and a search that is bounded to + * the line measure the same. A machine exporter writes no comments at all, and + * against that file the unbounded form reads the entire prefix and its cost + * grows with how far into the file the cursor sits. + * + * That is not a size effect, so `fileSizeIndependence` cannot see it: both + * offsets are in the same file. It is an offset effect, and it needs its own + * model and its own gate. This one was a live defect, measured at 8,385x + * between the two ends of an 800 KB comment-free file before it was fixed. + * + * Comments are stripped from the reference model rather than generated away, so + * the statements are the same statements and nothing about the composition can + * drift between the two. + * + * Not a `GeneratedModel`: it carries no `probe`, because the probe statement is + * identified by its comment and stripping the comments is the whole point. The + * two offsets it does carry are the only ones its gate asks for. + * + * @returns {{ text: string, statements: number, lines: number, bytes: number, + * meaningfulTokens: number, near: number, far: number }} + */ +export function commentFreeModel() { + const source = referenceModel(); + // From an exclamation mark to the end of its line, the newline kept. There are + // no string literals and no escapes, so nothing can hide an exclamation mark + // from this and it needs no scanner of its own. + const text = source.text.replace(/![^\n]*/g, '').replace(/^\s*\n/, ''); + if (text.includes('!')) throw new Error('the comment-free model still carries a comment'); + + // The first statement, and the last one. Both are found in the text rather + // than computed from a length, so neither moves when the composition is edited. + const near = text.indexOf('Version') + 4; + const far = text.lastIndexOf(';'); + + return { + text, + statements: source.statements, + lines: text.split('\n').length - 1, + bytes: Buffer.byteLength(text, 'utf8'), + meaningfulTokens: countMeaningfulTokens(text), + near, + far, + }; +} + +/** + * @param {GeneratedModel} model + * @param {number} statements + * @param {number} lines + */ +function assertShape(model, statements, lines) { + if (model.statements !== statements || model.lines !== lines) { + throw new Error( + `the generator drifted: ${model.statements} statements and ${model.lines} lines, ` + + `against the ${statements} and ${lines} the contract states` + ); + } +} + +/* -------------------------------------------------------------------------- */ +/* Measuring what came out */ +/* -------------------------------------------------------------------------- */ + +/** + * Meaningful tokens: type names, written values, separators, terminators and + * comments. Whitespace is not one, and neither is an empty field between two + * commas, which has a region but no text. + * + * This is a reporting figure, not an assertion. It is counted here rather than + * by `scanIdf` on purpose: this module must run before the scanner exists, and a + * generator that cannot describe its own output until the thing it is measuring + * has been written is a generator nobody can develop against. + * + * @param {string} text + * @returns {number} + */ +export function countMeaningfulTokens(text) { + let tokens = 0; + let valueStart = -1; + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (ch === '!') { + if (valueStart !== -1 && text.slice(valueStart, i).trim() !== '') tokens += 1; + valueStart = -1; + tokens += 1; + while (i < text.length && text[i] !== '\n') i += 1; + continue; + } + if (ch === ',' || ch === ';') { + if (valueStart !== -1 && text.slice(valueStart, i).trim() !== '') tokens += 1; + valueStart = -1; + tokens += 1; + continue; + } + if (valueStart === -1 && ch.trim() !== '') valueStart = i; + } + if (valueStart !== -1 && text.slice(valueStart).trim() !== '') tokens += 1; + return tokens; +} + +/* -------------------------------------------------------------------------- */ +/* Command line */ +/* -------------------------------------------------------------------------- */ + +const invokedDirectly = + process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (invokedDirectly) { + const wantsSmall = process.argv.includes('--small'); + const model = wantsSmall ? smallModel() : referenceModel(); + if (process.argv.includes('--stats')) { + const { text: _text, ...shape } = model; + process.stdout.write(`${JSON.stringify(shape, null, 2)}\n`); + } else { + process.stdout.write(model.text); + } +} diff --git a/package-lock.json b/package-lock.json index 5b1f981..89ebfb0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -448,6 +448,10 @@ } } }, + "node_modules/@idfkit/language": { + "resolved": "packages/language", + "link": true + }, "node_modules/@idfkit/schemas": { "resolved": "packages/schemas", "link": true @@ -1818,14 +1822,29 @@ "node": ">=20" }, "peerDependencies": { + "@idfkit/language": "0.0.0", "@idfkit/weather": "0.0.0" }, "peerDependenciesMeta": { + "@idfkit/language": { + "optional": true + }, "@idfkit/weather": { "optional": true } } }, + "packages/language": { + "name": "@idfkit/language", + "version": "0.0.0", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@idfkit/core": "0.0.0" + } + }, "packages/schemas": { "name": "@idfkit/schemas", "version": "0.0.0", diff --git a/package.json b/package.json index 07fa273..a0955fe 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,11 @@ "typecheck:docs": "tsc -p tsconfig.docs.json", "test": "vitest run", "test:watch": "vitest", + "bench": "node bench/budget.mjs", "format": "prettier --write \"{packages,docs,docs-snippets}/**/*.{ts,mjs,json,md,css}\" \"*.{json,md}\"", "format:check": "prettier --check \"{packages,docs,docs-snippets}/**/*.{ts,mjs,json,md,css}\" \"*.{json,md}\"", "check:absent-component": "node scripts/check-absent-component.mjs", + "check:bench": "node bench/budget.mjs", "check:bundle-purity": "node scripts/check-bundle-purity.mjs", "check:conformance-level": "node scripts/emit-conformance.mjs --check", "check:facade": "node scripts/check-facade.mjs", diff --git a/packages/core/package.json b/packages/core/package.json index fab80fc..c6a3518 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -43,7 +43,7 @@ }, "idfkit": { "conformance": "conformance-2026.8", - "governance": "governance-2026.10" + "governance": "governance-2026.11" }, "dependencies": { "@idfkit/schemas": "0.0.0" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 415ae51..60bb660 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -43,6 +43,13 @@ export type { ParseDiagnostic, ParseOptions, ParseResult } from './parse/idf.js' export { getEpJsonVersion, parseEpJson } from './parse/epjson.js'; export type { EpJson } from './parse/epjson.js'; +export { scanIdf } from './syntax/layer.js'; +export type { Statement, SyntaxLayer } from './syntax/layer.js'; +export { classify } from './syntax/classify.js'; +export type { Token, TokenKind } from './syntax/tokens.js'; +export { lineColumnAt, offsetAt } from './syntax/region.js'; +export type { LineColumn, Region } from './syntax/region.js'; + export { writeIdf, writeObject } from './write/idf.js'; export type { ObjectWriteOptions, WriteIdfOptions } from './write/idf.js'; export { toEpJson, writeEpJson } from './write/epjson.js'; @@ -52,7 +59,7 @@ export { Severity, validateDocument, validateObject } from './validate/index.js' export type { ValidationError, ValidationResult } from './validate/index.js'; export { describeObjectType } from './introspect/describe.js'; -export type { FieldDescription, ObjectDescription } from './introspect/describe.js'; +export type { FieldDescription, ObjectDescription, ProsePool } from './introspect/describe.js'; export { docsUrlForObject, diff --git a/packages/core/src/parse/idf.ts b/packages/core/src/parse/idf.ts index a5aa62c..991e6d6 100644 --- a/packages/core/src/parse/idf.ts +++ b/packages/core/src/parse/idf.ts @@ -4,6 +4,7 @@ import { IdfDocument } from '../document.js'; import type { ExtensibleGroup, FieldValues, StoredValue } from '../object.js'; import type { AnyTypeMap, UntypedMap } from '../typemap.js'; import { lex, type LexDiagnostic, type RawObject } from './lexer.js'; +import { scan } from './scan.js'; export interface ParseDiagnostic extends LexDiagnostic { /** @@ -249,56 +250,44 @@ function stringIsLegal(definition: SlimType, field: string, value: string): bool } /** - * The 1-based line a field sits on, found by rescanning from the object's own offset. + * The 1-based line a field sits on, found by rescanning the statement from its own offset. * * The lexer records one offset per object rather than a line per field, because a finding about a - * field is rare and an array per object is not. This walks forward counting separators, stepping - * over `!` comments so a comma inside one is not mistaken for one, which is the same rule the - * lexer itself follows. + * field is rare and an array per object is not. This is the rescan that offset exists for, and it + * runs only while a finding is being built. * - * `index` counts into `RawObject.values`, which the lexer has already shifted the type name off. - * The scan starts at the type name, so it steps over one more separator than the index: the comma - * that ends `Building,` is what puts values[0] on the line after it. + * The rescan goes through the shared scanner, so "step over the comment between a separator and + * its value" is the rule the lexer itself followed rather than a second copy of it. A field's + * comma is routinely followed by `!- Field Name` on the same line, and a helper that stopped at + * the `!` would report the line the PREVIOUS value sits on, one too early. + * + * `index` counts into `RawObject.values`, which has already had the type name shifted off, so the + * field wanted is the scanner's `index + 1`: the comma that ends `Building,` is what puts + * `values[0]` on the line after it. The scan stops at that field, or at the end of the statement + * when the statement has fewer fields than the finding names, in which case the object's own line + * is the best answer available. * * @internal */ function fieldLine(text: string, object: RawObject, index: number): number { if (object.offset === undefined) return object.line; - const separators = index + 1; - let seen = 0; - let position = object.offset; - while (position < text.length && seen < separators) { - const char = text[position]; - if (char === '!') { - const newline = text.indexOf('\n', position); - if (newline < 0) break; - position = newline + 1; - continue; - } - if (char === ';') break; - if (char === ',') seen += 1; - position += 1; - } - if (seen < separators) return object.line; - - // Step over whitespace AND any comment between the separator and the value. A field's comma is - // routinely followed by `!- Field Name` on the same line, and stopping at the `!` would report - // the line the PREVIOUS value sits on, one too early. - while (position < text.length) { - const char = text[position] ?? ''; - if (char === '!') { - const newline = text.indexOf('\n', position); - if (newline < 0) break; - position = newline + 1; - continue; - } - if (!/\s/.test(char)) break; - position += 1; - } - + const wanted = index + 1; let line = object.line; - for (let i = object.offset; i < position; i += 1) if (text[i] === '\n') line += 1; + scan( + text, + { + fieldEnd(field, _start, _end, valueLine) { + if (field < wanted) return; + line = valueLine; + return false; + }, + statementEnd() { + return false; + }, + }, + { from: object.offset, line: object.line, column: object.column } + ); return line; } diff --git a/packages/core/src/parse/lexer.ts b/packages/core/src/parse/lexer.ts index cb2a79e..41392d6 100644 --- a/packages/core/src/parse/lexer.ts +++ b/packages/core/src/parse/lexer.ts @@ -1,3 +1,5 @@ +import { scan } from './scan.js'; + /** A raw object as it appears in the file, before schema interpretation. */ export interface RawObject { /** Type name exactly as written, e.g. `BuildingSurface:Detailed`. */ @@ -66,107 +68,71 @@ export interface LexOptions { /** * Split IDF text into raw objects. * - * A hand-written character scan rather than a regex. The Python library matches - * objects with a `(?:[^;!]*(?:![^\n]*\n)?)*?` inner loop; that is a nested - * quantifier, so it backtracks badly on malformed input and cannot report where - * the problem was. A scanner is about the same amount of code, is linear in the - * input, and always knows its line number. - * - * The grammar is small: - * - `!` starts a comment running to end of line - * - `,` separates fields - * - `;` terminates an object - * - everything else is field text, trimmed + * The character rules are not here: they live in `scan.ts`, which the syntax layer reads through + * as well. Two copies of "step over a comment between a separator and its value" that differ by + * one character put a finding on the wrong field, so there is one copy (research R3). * - * There are no string literals and no escape sequences, so a comma cannot occur - * inside a field value. Real files depend on that. + * This function is what remains once those rules are elsewhere: assembling values from the text + * runs the scan reports, and turning a statement into a `RawObject` or into a diagnostic. It asks + * for no comment and no region, so it pays for neither, and it builds no syntax layer. */ export function lex(text: string, options: LexOptions = {}): RawObject[] { const objects: RawObject[] = []; const report = options.onDiagnostic; - const length = text.length; - /** Field text pieces, split whenever a comment interrupts a field. */ + /** Field text pieces, one per run the scan reports, joined when the field closes. */ let chunks: string[] = []; /** Fields of the object being read; index 0 ends up being the type name. */ let values: string[] = []; - let index = 0; - let line = 1; - let fieldStart = 0; - let objectLine = 1; - let objectStarted = false; - /** Offset of the first character of the current line, for turning an offset into a column. */ - let lineStart = 0; /** - * Column the current object's type name begins at, or 0 while none has been seen. + * Where the current statement starts. * * The first NON-BLANK character, not the start of the field text: Python's regex matches the * type name itself, so an object indented three spaces reports column 4 there and has to report * column 4 here too, or the corpus compares two different notions of position. */ + let objectLine = 1; let objectColumn = 0; - /** Absolute offset of the current object's first non-blank character. */ let objectOffset = -1; - /** 1-based column of an offset on the line it falls in. Matches Python's `_line_and_column`. */ - const columnAt = (offset: number): number => offset - lineStart + 1; - - const endField = (end: number): string => { - chunks.push(text.slice(fieldStart, end)); - const value = chunks.join('').trim(); - chunks = []; - return value; - }; - - while (index < length) { - const char = text[index]; - - // The first non-blank character of an object fixes where the object starts. Recorded here - // rather than at the delimiter, because by then the leading whitespace has been consumed and - // the offset that remains points at the padding rather than at the name. - if (objectColumn === 0 && char !== undefined && !/\s/.test(char)) { + scan(text, { + statementStart(offset, line, column) { objectLine = line; - objectColumn = columnAt(index); - objectOffset = index; - } - - if (char === '!') { - // Preserve any field text seen before the comment, then resume after the - // newline. This is what lets `Zone1, !- Name` work: the comment is not - // part of the value, but the value is not finished either. - chunks.push(text.slice(fieldStart, index)); - const newline = text.indexOf('\n', index); - if (newline === -1) { - index = length; - fieldStart = length; - break; + objectColumn = column; + objectOffset = offset; + }, + + fieldText(start, end) { + chunks.push(text.slice(start, end)); + }, + + fieldEnd() { + values.push(chunks.join('').trim()); + chunks = []; + }, + + statementEnd(_end, unterminated) { + if (unterminated) { + // The scan closes the field the input ran out inside, so the trailing text arrived as the + // last value rather than as a leftover. Popping it leaves `values` holding exactly what a + // terminated statement would have held, which is what the message reads from. + const trailing = values.pop() ?? ''; + if (trailing !== '' || values.length > 0) { + report?.({ + message: `Unterminated object near "${trailing.slice(0, 40) || values[0]}" (missing ";")`, + line: objectLine, + code: 'ParseError', + column: objectColumn || undefined, + // `values` has not been shifted, because the shift happens on `;` and there was none, + // so the type name is still at the front. Reporting it is what lets the corpus compare + // this finding against Python's on `(code, line, typeName)`. + typeName: values[0], + }); + } + values = []; + return; } - index = newline + 1; - fieldStart = index; - line += 1; - lineStart = index; - if (!objectStarted && chunks.join('').trim() === '') { - chunks = []; - objectLine = line; - objectColumn = 0; - objectOffset = -1; - } - continue; - } - - if (char === ',') { - values.push(endField(index)); - objectStarted = true; - index += 1; - fieldStart = index; - continue; - } - - if (char === ';') { - values.push(endField(index)); - index += 1; - fieldStart = index; const typeName = values.shift() ?? ''; if (typeName === '') { @@ -186,46 +152,8 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { }); } values = []; - objectStarted = false; - objectLine = line; - objectColumn = 0; - objectOffset = -1; - continue; - } - - if (char === '\n') { - line += 1; - lineStart = index + 1; - if ( - !objectStarted && - chunks.join('').trim() === '' && - text.slice(fieldStart, index).trim() === '' - ) { - // Blank line before any object content: keep the start line current. - chunks = []; - fieldStart = index + 1; - objectLine = line; - objectColumn = 0; - objectOffset = -1; - } - } - - index += 1; - } - - const trailing = (chunks.join('') + text.slice(fieldStart, length)).trim(); - if (trailing !== '' || values.length > 0) { - report?.({ - message: `Unterminated object near "${trailing.slice(0, 40) || values[0]}" (missing ";")`, - line: objectLine, - code: 'ParseError', - column: objectColumn || undefined, - // `values` has not been shifted, because the shift happens on `;` and there was none, so the - // type name is still at the front. Reporting it is what lets the corpus compare this finding - // against Python's on `(code, line, typeName)`. - typeName: values[0], - }); - } + }, + }); return objects; } diff --git a/packages/core/src/parse/scan.ts b/packages/core/src/parse/scan.ts new file mode 100644 index 0000000..73c18b4 --- /dev/null +++ b/packages/core/src/parse/scan.ts @@ -0,0 +1,285 @@ +/** + * The IDF character rules, in one place. + * + * The grammar is small: + * - `!` starts a comment running to the end of its line + * - `,` separates fields + * - `;` terminates a statement + * - everything else is field text + * + * There are no string literals and no escape sequences, so a comma cannot occur inside a field + * value and an `!` always starts a comment. Real files depend on both. + * + * A hand-written character walk rather than a regex. The Python library matches objects with a + * `(?:[^;!]*(?:![^\n]*\n)?)*?` inner loop; that is a nested quantifier, so it backtracks badly on + * malformed input and cannot report where the problem was. A scan is about the same amount of + * code, is linear in the input, and always knows its line number. + * + * **Why this file exists at all.** Every position this library reports depends on the syntax layer + * and the model-building read agreeing about where a field begins and ends. Two implementations of + * "step over a comment between a separator and its value" that differ by one character put a + * finding on the wrong field, and that class of bug stays invisible until a file puts a comment + * somewhere unusual. The rules used to exist twice, once inside `lex` and once inside the + * `fieldLine` helper of `idf.ts`; both are now callers of this scan (research R3). + * + * **This module is internal and stays internal.** It is not exported from the package root. The + * syntax layer, which is public, is what a consumer outside the package reaches for; keeping the + * scan itself unexported is the reason the layer ships in core rather than beside the language + * service, because a package boundary drawn around both halves would have forced this file into a + * published export purely so the other half could reach it. + * + * @internal + */ + +const EXCLAMATION = 0x21; +const COMMA = 0x2c; +const SEMICOLON = 0x3b; +const LINE_FEED = 0x0a; + +/** + * What a caller wants told to it, and therefore what the scan pays for. + * + * The parameterisation is the handler itself: an absent member costs one `undefined` check and + * nothing else, so a caller that never asks for comments never pays to locate their ends, and a + * caller that never asks for field regions never pays to bound them. This is what keeps `lex` at + * its present cost while the syntax layer reads the same characters through the same rules. + * + * Whitespace is never reported. It is the complement of everything below and is derived by + * subtraction rather than stored, which is the same decision the layer makes about trivia + * (research R5). + * + * Every member may return `false` to stop the scan where it stands. Nothing is emitted afterwards. + * A caller that wants one field out of a statement uses this rather than reading to end of input. + * + * @internal + */ +export interface ScanHandler { + /** + * The first non-blank character of a statement, which is where the statement starts. + * + * A leading comment is not a statement start: `line` and `column` are those of the first + * character that is neither whitespace nor part of a comment. `column` is 1-based and counted + * from the start of its own line, which is what the conformance corpus compares, and it is the + * column of the type name rather than of the indentation before it. + */ + statementStart?(offset: number, line: number, column: number): boolean | void; + + /** + * One run of the current field's raw text, exactly as written and untrimmed. + * + * A field interrupted by a comment reports one run per side of it, so a caller that wants the + * value as a string joins the runs and trims the result. Only non-empty runs are reported, which + * changes no join. + */ + fieldText?(start: number, end: number): boolean | void; + + /** + * The current field is complete, closed by a separator, a terminator, or end of input. + * + * `index` is 0 for the type name and 1 for the first field after it, so a field at `index` + * corresponds to `RawObject.values[index - 1]` and to `Statement.fields[index - 1]`, both of + * which have had the type name shifted off. + * + * `start` and `end` bound the value text, trimmed, with any comment excluded. A field written + * empty between two commas still reports, with `start === end` positioned where its value would + * have begun, which is what keeps positional indexing sound through a blank extensible slot. + * + * `line` is the 1-based line `start` falls on. It is carried here because the scan already knows + * it and a caller that only wants a line should not have to index the text to recover one. + * + * A comment interrupting a field is not part of the field's region: the region covers the value + * text only, which is what makes an underline land on the value rather than on the annotation + * beside it. When text appears on both sides of an interrupting comment, which is malformed but + * representable, the region covers the first run alone rather than spanning the comment, so a + * region and a comment can never overlap and the layer's tiling invariant holds by construction. + * `lex` still joins both runs into its value, so the two disagree only on that input. + */ + fieldEnd?(index: number, start: number, end: number, line: number): boolean | void; + + /** A comma, at `offset`. Reported after the {@link ScanHandler.fieldEnd} it closes. */ + separator?(offset: number): boolean | void; + + /** A semicolon, at `offset`. Reported after the {@link ScanHandler.fieldEnd} it closes. */ + terminator?(offset: number): boolean | void; + + /** + * A comment, from its exclamation mark through the last character before the line feed. + * + * The mark is included and the line feed is not. A carriage return before that feed stays inside + * the comment, because line endings are counted as written and that is what every editor does. + */ + comment?(start: number, end: number): boolean | void; + + /** + * The statement ended. `end` is one past its last character. + * + * `unterminated` is true when the input ran out before a semicolon did, in which case the + * statement runs to end of input and its last field has just been reported. Malformed input is + * represented rather than stopped at, so the scan reports what was written and moves on. + */ + statementEnd?(end: number, unterminated: boolean): boolean | void; +} + +/** + * Where to begin, for a caller resuming inside text it has already positioned. + * + * `line` and `column` describe `from` itself, so a caller that knows a statement's line and column + * can scan that statement alone and get line numbers in the whole file's terms rather than in the + * fragment's. + * + * @internal + */ +export interface ScanOptions { + /** Offset to start at. @defaultValue 0 */ + from?: number; + /** 1-based line `from` falls on. @defaultValue 1 */ + line?: number; + /** 1-based column `from` falls on. @defaultValue 1 */ + column?: number; +} + +/** + * Walk IDF text, reporting what the handler asked for. + * + * One linear pass, no allocation of its own, and it never throws: text that violates the grammar + * is reported as what it is rather than stopped at. + * + * @internal + */ +export function scan(text: string, handler: ScanHandler, options: ScanOptions = {}): void { + const length = text.length; + + let index = options.from ?? 0; + let line = options.line ?? 1; + /** Offset of the current line's first character, which is what turns an offset into a column. */ + let lineStart = index - ((options.column ?? 1) - 1); + + /** Start of the current field's text run, moved past every comment that interrupts it. */ + let fieldStart = index; + /** 0 for the type name, 1 for the first field after it. */ + let fieldIndex = 0; + /** True once the current statement's first non-blank, non-comment character has been seen. */ + let open = false; + /** First non-blank character of the current field's value, or -1 while it has none. */ + let valueStart = -1; + /** One past the last non-blank character of the run `valueStart` falls in. */ + let valueEnd = -1; + /** Line `valueStart` falls on. */ + let valueLine = 1; + /** True once a comment has ended the run holding `valueStart`. */ + let valueClosed = false; + + /** + * Report the field the scan is inside, closed at `at`. False when a handler asked to stop. + * + * A field with no value text is positioned at `at`, which is the separator, the terminator, or + * the end of input that closed it. That is the offset a value would have begun at, because + * everything between the field's start and `at` was whitespace or comment. + */ + const closeField = (at: number): boolean => { + if (at > fieldStart && handler.fieldText?.(fieldStart, at) === false) return false; + const empty = valueStart < 0; + const stop = + handler.fieldEnd?.( + fieldIndex, + empty ? at : valueStart, + empty ? at : valueEnd, + empty ? line : valueLine + ) === false; + valueStart = -1; + valueEnd = -1; + valueClosed = false; + return !stop; + }; + + while (index < length) { + const code = text.charCodeAt(index); + + if (code === EXCLAMATION) { + // Text seen before the comment still belongs to the field, and so does text after the line + // feed: the comment interrupts the field without ending it. This is what lets + // `Zone1, !- Name` work, where the comment is no part of the value and the value is no part + // of the comment. + if (index > fieldStart && handler.fieldText?.(fieldStart, index) === false) return; + const feed = text.indexOf('\n', index); + const end = feed === -1 ? length : feed; + if (handler.comment?.(index, end) === false) return; + if (valueStart >= 0) valueClosed = true; + if (feed === -1) { + fieldStart = length; + break; + } + index = feed + 1; + fieldStart = index; + line += 1; + lineStart = index; + continue; + } + + const blank = isSpace(code); + + if (!open && !blank) { + open = true; + if (handler.statementStart?.(index, line, index - lineStart + 1) === false) return; + } + + if (code === COMMA || code === SEMICOLON) { + if (!closeField(index)) return; + if (code === COMMA) { + if (handler.separator?.(index) === false) return; + fieldIndex += 1; + } else { + if (handler.terminator?.(index) === false) return; + if (handler.statementEnd?.(index + 1, false) === false) return; + fieldIndex = 0; + open = false; + } + index += 1; + fieldStart = index; + continue; + } + + if (code === LINE_FEED) { + line += 1; + lineStart = index + 1; + } else if (!blank) { + if (valueStart < 0) { + valueStart = index; + valueEnd = index + 1; + valueLine = line; + } else if (!valueClosed) { + valueEnd = index + 1; + } + } + + index += 1; + } + + // Input ran out inside a statement. Its last field is reported like any other, so a caller reads + // what was written rather than having to reconstruct it from the leftovers. + if (!open) return; + if (!closeField(length)) return; + handler.statementEnd?.(length, true); +} + +/** + * Whether a character is whitespace, by the same definition `String.prototype.trim` uses. + * + * A field's value is trimmed, so a scan that disagreed with `trim` about one character would put + * a region beside the value rather than on it. ASCII is one comparison; the rest of the set costs + * a branch nothing outside a comment ever takes. + */ +function isSpace(code: number): boolean { + if (code < 0x80) return code === 0x20 || (code >= 0x09 && code <= 0x0d); + return ( + code === 0xa0 || + code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || + code === 0x2028 || + code === 0x2029 || + code === 0x202f || + code === 0x205f || + code === 0x3000 || + code === 0xfeff + ); +} diff --git a/packages/core/src/syntax/classify.ts b/packages/core/src/syntax/classify.ts new file mode 100644 index 0000000..50e3d32 --- /dev/null +++ b/packages/core/src/syntax/classify.ts @@ -0,0 +1,60 @@ +import type { SyntaxLayer } from './layer.js'; +import { lineStartsOf } from './region.js'; +import type { Token, TokenKind } from './tokens.js'; + +/** + * Every character of the text, as tokens, in source order. + * + * Two things happen here that the layer deliberately does not pay to store. + * + * The first is `trivia`. Whitespace is the complement of the stored tokens, so storing it would + * roughly double the token count to hold something derivable by subtraction. This function fills + * each gap as it walks, which is what makes the layer's complete coverage observable (FR-016) + * without the layer carrying it: the sequence begins at offset 0, ends at `text.length`, and has + * no gap and no overlap anywhere between. + * + * The second is the line split. No yielded token crosses a line boundary (FR-047), while a stored + * `value` region may, because the format lets a field be written across two lines and real files + * do it. Every token encoding in use, the Language Server Protocol's included, expresses a token + * as a length on a single line, so a token spanning one cannot be encoded at all. The split adds + * tokens and moves no boundary, so the tiling holds exactly as before, and the stored region stays + * whole for everything that is not being drawn. + * + * A generator, so a consumer colouring a viewport stops where it stops rather than materialising + * four hundred thousand tokens to read the first fifty. + */ +export function* classify(layer: SyntaxLayer): Iterable { + const text = layer.text; + const tokens = layer.tokens; + const starts = tokens.starts; + const ends = tokens.ends; + // The layer's own line index, built once and shared with every position query made against it. + const lineStarts = lineStartsOf(layer); + + /** The line the walk is on. It only ever moves forward, so the whole pass stays linear. */ + let line = 0; + + /** One token per line the span touches, split at each line start strictly inside it. */ + function* perLine(from: number, to: number, kind: TokenKind): Generator { + while (line + 1 < lineStarts.length && lineStarts[line + 1]! <= from) line += 1; + let at = from; + while (line + 1 < lineStarts.length && lineStarts[line + 1]! < to) { + line += 1; + const boundary = lineStarts[line]!; + yield { start: at, end: boundary, kind }; + at = boundary; + } + if (at < to) yield { start: at, end: to, kind }; + } + + /** One past the last character yielded so far, which is where the next gap would begin. */ + let covered = 0; + for (let index = 0; index < tokens.length; index += 1) { + const start = starts[index]!; + if (start > covered) yield* perLine(covered, start, 'trivia'); + const end = ends[index]!; + yield* perLine(start, end, tokens.kindAt(index)); + covered = end; + } + if (covered < text.length) yield* perLine(covered, text.length, 'trivia'); +} diff --git a/packages/core/src/syntax/layer.ts b/packages/core/src/syntax/layer.ts new file mode 100644 index 0000000..f5de28a --- /dev/null +++ b/packages/core/src/syntax/layer.ts @@ -0,0 +1,200 @@ +import { scan } from '../parse/scan.js'; +import type { Region } from './region.js'; +import { TokenStore, type TokenKind } from './tokens.js'; + +/** + * One statement as the text writes it, terminated or not. + * + * Not an object in the model and carrying no schema meaning (FR-006): a statement whose type the + * schema never heard of, or which carries twice the fields its type defines, is still a statement. + * Deciding what any of it means is the reader's job, and the layer is what a reader positions + * against. + */ +export interface Statement { + /** The whole statement, from its first non-blank character through its terminator. */ + readonly region: Region; + /** Just the type name. Empty, at the offset one would have begun, when none was written. */ + readonly typeName: Region; + /** Type name text, exactly as written and not case-folded. */ + readonly typeNameText: string; + /** + * Fields after the type name, in positional order, one region each. + * + * Index 0 is the first field after the type name, matching `RawObject.values`, which has already + * had the type name shifted off. A field written empty still gets a region, an empty one + * positioned where its value would have begun, so that positional indexing never shifts and a + * blank slot in the middle of an extensible group can still be pointed at. + * + * The count is what was written rather than what the type defines. A statement carrying more or + * fewer fields than its type is a finding, not a representation problem. + */ + readonly fields: readonly Region[]; + /** True when no terminator was found, meaning the statement runs to end of input. */ + readonly unterminated: boolean; +} + +/** + * Everything the text contains and where it was. + * + * The layer holds the text it was built from, which is what makes byte-identical reconstruction a + * consequence rather than a feature: a reconstruction defined as concatenating slices of that text + * returns the text by construction. What can actually break is the tiling, so that is what the + * corpus test asserts and what {@link classify} makes observable. + * + * Nothing builds a layer implicitly (FR-005). `lex` and `parseIdf` read the same characters + * through the same scan and construct none of this, so a caller who never names `scanIdf` pays + * neither its time nor its memory. + */ +export interface SyntaxLayer { + /** The text this layer was built from. Held so regions can be resolved. */ + readonly text: string; + /** Statements in source order. */ + readonly statements: readonly Statement[]; + /** Every meaningful token in source order, packed. Whitespace is not among them. */ + readonly tokens: TokenStore; +} + +/** + * Scan IDF text into a syntax layer. + * + * Text and nothing else: no schema, because the layer records what the text says and never what it + * means (FR-006). It never throws, for any input, and text that violates the grammar is + * represented rather than stopped at (FR-004): an unterminated final statement runs to end of + * input and says so, a statement written without a type name still tiles, and empty text produces + * a layer with no statements and no tokens, which satisfies the tiling invariant vacuously. + * + * One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a + * quarter over a plain read of the same text; the work above the scan is a push per token and a + * record per statement. + */ +export function scanIdf(text: string): SyntaxLayer { + const tokens = new TokenStore(initialCapacity(text.length)); + const statements: Statement[] = []; + + /** + * Comments seen since the last field closed, as flat `[start, end]` pairs in source order. + * + * They are buffered rather than pushed on sight because a comment can interrupt a field, in + * which case the scan reports it before the field it sits inside: the field's own bounds are + * only known once the field closes. Emitting them at that point, ordered against the value, + * is what keeps the store in source order. + */ + const comments: number[] = []; + /** Index into {@link comments} of the first pair not yet pushed. */ + let nextComment = 0; + /** Raw text runs of the field being read, as flat `[start, end]` pairs in source order. */ + const runs: number[] = []; + + /** Offset the current statement opened at. */ + let openedAt = 0; + /** The current statement's type name, replaced when its field closes. */ + let typeName: Region = EMPTY_REGION; + /** The current statement's fields. Reassigned per statement, so a pushed record keeps its own. */ + let fields: Region[] = []; + + /** Push every buffered comment that begins before `offset`, keeping the store in source order. */ + const flushComments = (offset: number): void => { + while (nextComment < comments.length && comments[nextComment]! < offset) { + tokens.push(comments[nextComment]!, comments[nextComment + 1]!, 'comment'); + nextComment += 2; + } + }; + + scan(text, { + statementStart(offset) { + openedAt = offset; + fields = []; + }, + + /** + * The runs are recorded only for the sake of a field written on both sides of a comment that + * interrupts it. Every other reader of this scan wants the field's value, which arrives whole + * below. + */ + fieldText(start, end) { + runs.push(start, end); + }, + + fieldEnd(index, start, end) { + const region: Region = { start, end }; + if (index === 0) typeName = region; + else fields.push(region); + + const kind: TokenKind = index === 0 ? 'typeName' : 'value'; + flushComments(start); + // A field written empty has an empty region, which is a position rather than a span, so it + // is recorded above and is no token. + if (end > start) tokens.push(start, end, kind); + + // Only a comment can separate two runs of one field, so a run beginning at or after the + // value ends is text written after an interrupting comment: `A !- why\n B,` writes both `A` + // and `B` into one field. The scan bounds the value at `A` deliberately, so that a value + // region and a comment region can never overlap, which leaves `B` reached by no region at + // all, and a gap holding something other than whitespace is what the tiling invariant + // forbids. Every other run holds the value itself or the whitespace before it, and both + // begin before it ends, so a well-formed field never enters the body and never allocates. + for (let at = 0; at < runs.length; at += 2) { + const runStart = runs[at]!; + if (runStart < end) continue; + const runEnd = runs[at + 1]!; + const raw = text.slice(runStart, runEnd); + const from = runStart + (raw.length - raw.trimStart().length); + const to = runEnd - (raw.length - raw.trimEnd().length); + if (from >= to) continue; + flushComments(from); + tokens.push(from, to, kind); + } + + flushComments(Infinity); + + comments.length = 0; + nextComment = 0; + runs.length = 0; + }, + + separator(offset) { + tokens.push(offset, offset + 1, 'separator'); + }, + + terminator(offset) { + tokens.push(offset, offset + 1, 'terminator'); + }, + + comment(start, end) { + comments.push(start, end); + }, + + statementEnd(end, unterminated) { + statements.push({ + region: { start: openedAt, end }, + typeName, + typeNameText: text.slice(typeName.start, typeName.end), + fields, + unterminated, + }); + }, + }); + + // Comments after the last terminator close no field, so nothing has flushed them. Text that is + // only comments reaches here having reported no statement at all. + flushComments(Infinity); + + return { text, statements, tokens }; +} + +/** + * Stands in until a statement's first field closes, which it always does before the statement + * ends, so no statement is ever recorded holding this. + */ +const EMPTY_REGION: Region = { start: 0, end: 0 }; + +/** + * A starting size for the token store, so a large file does not grow it a dozen times. + * + * A commented IDF line runs to about a dozen characters per token, so one per sixteen undershoots + * slightly and costs at most one doubling, which is the safer side to be wrong on: a file that is + * mostly whitespace or mostly comment would otherwise pay for capacity it never fills. + */ +function initialCapacity(length: number): number { + return length < 1024 ? 64 : length >> 4; +} diff --git a/packages/core/src/syntax/region.ts b/packages/core/src/syntax/region.ts new file mode 100644 index 0000000..1ed334d --- /dev/null +++ b/packages/core/src/syntax/region.ts @@ -0,0 +1,154 @@ +/** + * A span of source text, half-open. + * + * `end` is one past the last character, so `text.slice(start, end)` is exactly the text the + * region selects, an empty region is `start === end`, and two adjacent regions share a number + * rather than straddling one. An empty region is what a cursor between two characters is. + * + * Offsets are indices into the JavaScript string, which are UTF-16 code units. Nothing converts + * them, because no consumer wants them converted: the Language Server Protocol's default + * `positionEncoding` is UTF-16, Monaco measures columns in UTF-16 code units, and CodeMirror + * addresses its document by the same offsets. The Python library counts code points instead, + * because Python string indices are code points; the two agree for every character below the + * astral planes and differ for anything above them, which in practice means an emoji in a + * comment. That divergence is registered rather than papered over. + */ +export interface Region { + /** Offset of the first character, into the source string. */ + readonly start: number; + /** Offset one past the last character. */ + readonly end: number; +} + +/** + * A position as a human reads it, derived from an offset rather than stored. + * + * Storing line and column on every region would double its size to hold two numbers that are a + * function of one. Both count from one, which is what the existing findings already report; a + * consumer whose own convention counts a column from zero subtracts one, and that subtraction is + * the consumer's, because a service that guessed which convention its caller wanted would be + * modelling a protocol. + */ +export interface LineColumn { + /** 1-based, counting from the start of the text. */ + readonly line: number; + /** 1-based, in UTF-16 code units, from the start of the line. */ + readonly column: number; +} + +/** + * The minimum a position query needs: the text its offsets index into. + * + * Declared structurally rather than by importing `SyntaxLayer`, so that positions stay usable + * without one and this module depends on nothing. A `SyntaxLayer` holds its text and therefore + * satisfies this, which is what lets `lineColumnAt(layer, offset)` read as the contract writes it. + */ +export interface TextSource { + /** The text offsets are measured into. */ + readonly text: string; +} + +/** + * Line index per source, built once and reused. + * + * Keyed by the source object rather than by its text, because a `SyntaxLayer` is immutable and + * lives as long as the document does, while keying on the string itself would hold megabytes of + * text alive for the sake of a few kilobytes of offsets. A caller that passes a fresh object + * literal each time gets a fresh index each time, which is the documented cost of doing that. + */ +const lineIndexes = new WeakMap(); + +/** + * Offsets at which each line begins, one entry per line, ascending. + * + * A line break is a line feed, and nothing else. A carriage return before it belongs to the line + * it ends, so a column at the end of a CRLF line is one greater than the same column in a file + * using line feeds alone, which is what every editor reports. A lone carriage return is *not* a + * break: the existing lexer does not treat it as one and Python's `_line_and_column` counts line + * feeds, and the conformance corpus compares a finding on its line, so a third opinion here would + * put the two libraries one line apart on a file no gate would explain. + * + * @internal + */ +export function lineStartsOf(source: TextSource): Int32Array { + const cached = lineIndexes.get(source); + if (cached !== undefined) return cached; + const starts = buildLineStarts(source.text); + lineIndexes.set(source, starts); + return starts; +} + +/** + * The line and column an offset falls on. + * + * Binary search over the line index, so a query costs the logarithm of the line count rather than + * a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than + * throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer + * was built from, and refusing to answer is worse than answering about the nearest character. + * + * The column of a *statement* is the column of its first non-blank character, not of the + * whitespace indenting it. This function does not enforce that rule, because a statement's region + * already starts at that character; it is stated here because every column this feature reports + * obeys it, and because the conformance corpus compares findings on their line and type name and + * would notice the number moving. + */ +export function lineColumnAt(source: TextSource, offset: number): LineColumn { + const clamped = clampOffset(offset, source.text.length); + const starts = lineStartsOf(source); + const line = lineIndexAt(starts, clamped); + return { line: line + 1, column: clamped - starts[line]! + 1 }; +} + +/** + * The offset a line and column names. + * + * The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one + * that does not: a line past the end resolves on the last line, a column past the end of its line + * resolves at the line's last position, which is the offset of the break that ends it. Round + * tripping an out-of-range position therefore returns the clamped position rather than the one + * asked for, which is the only honest answer available. + */ +export function offsetAt(source: TextSource, position: LineColumn): number { + const text = source.text; + const starts = lineStartsOf(source); + const line = clampOffset(position.line - 1, starts.length - 1); + const lineStart = starts[line]!; + // One past the last position on this line is where the next line starts, so the last position on + // it is one before that: the offset of the line feed, at which a cursor is still on this line. + const lineEnd = line + 1 < starts.length ? starts[line + 1]! - 1 : text.length; + const offset = lineStart + clampOffset(position.column - 1, text.length); + return offset > lineEnd ? lineEnd : offset; +} + +/** Two passes so the array is allocated once at its exact size; `indexOf` does the scanning. */ +function buildLineStarts(text: string): Int32Array { + let count = 1; + for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1)) count += 1; + const starts = new Int32Array(count); + let line = 1; + for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1)) { + starts[line] = at + 1; + line += 1; + } + return starts; +} + +/** Index of the greatest line start that is at or before `offset`. */ +function lineIndexAt(starts: Int32Array, offset: number): number { + let low = 0; + let high = starts.length - 1; + while (low < high) { + const middle = (low + high + 1) >> 1; + if (starts[middle]! <= offset) low = middle; + else high = middle - 1; + } + return low; +} + +/** Into `[0, max]`, whole. `NaN` lands at 0, since no position is nearer than another. */ +function clampOffset(value: number, max: number): number { + if (Number.isNaN(value)) return 0; + const whole = Math.trunc(value); + if (whole < 0) return 0; + return whole > max ? max : whole; +} diff --git a/packages/core/src/syntax/tokens.ts b/packages/core/src/syntax/tokens.ts new file mode 100644 index 0000000..c4c77d6 --- /dev/null +++ b/packages/core/src/syntax/tokens.ts @@ -0,0 +1,183 @@ +import type { Region } from './region.js'; + +/** + * What a span of text is, grammatically. Nothing here needs a schema: the layer records what the + * text says and never what it means, so a `value` is a value whether or not the field exists. + */ +export type TokenKind = + | 'typeName' // the statement's first field + | 'value' // any other field's text + | 'separator' // a comma + | 'terminator' // a semicolon + | 'comment' // an exclamation mark to end of line, the mark included + | 'trivia'; // whitespace between meaningful tokens; never stored, only yielded + +/** + * One token, materialised. + * + * A `Token` *is* a `Region` rather than carrying one, which keeps a classified stream to one + * object per token instead of two. On a file of forty thousand lines that difference is the + * difference between an editor that repaints and one that stutters, and a token still passes + * anywhere a region is expected. + * + * A materialised token never crosses a line boundary; a stored `value` region may, because the + * format lets a field be written across two lines, and the classification view splits it. + */ +export interface Token extends Region { + readonly kind: TokenKind; +} + +/** Kind codes, in the order the union declares them. The index is what `kinds` stores. */ +const KIND_BY_CODE: readonly TokenKind[] = [ + 'typeName', + 'value', + 'separator', + 'terminator', + 'comment', + 'trivia', +]; + +const CODE_BY_KIND: Readonly> = { + typeName: 0, + value: 1, + separator: 2, + terminator: 3, + comment: 4, + trivia: 5, +}; + +/** Where growth starts. Small enough for a one-statement file, large enough to skip early copies. */ +const INITIAL_CAPACITY = 64; + +/** + * Every token of one text, packed into three parallel arrays. + * + * Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code, + * held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand + * statements that is roughly 3.6 MB rather than upwards of 16 MB. + * + * This is a contract rather than an implementation note. The intended use is an editor holding + * several large files open at once, and a layer that allocated an object per token would pass + * every correctness test while being unusable there. So `Token` objects are materialised only when + * a caller asks for one, which for classification means only for the tokens actually drawn. + * + * The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing + * of the text either: it holds positions, and the text they index stays with the layer. + */ +export class TokenStore implements Iterable { + #starts: Int32Array; + #ends: Int32Array; + #kinds: Uint8Array; + #length = 0; + + constructor(capacity: number = INITIAL_CAPACITY) { + const initial = Math.max(0, Math.trunc(capacity)); + this.#starts = new Int32Array(initial); + this.#ends = new Int32Array(initial); + this.#kinds = new Uint8Array(initial); + } + + /** How many tokens are stored. The arrays below are longer than this; they carry slack. */ + get length(): number { + return this.#length; + } + + /** + * Start offsets, in source order. + * + * A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the + * positions without materialising anything. It is invalidated by the next {@link push}, which in + * practice means it is safe for as long as the layer is: nothing appends to a built layer. + */ + get starts(): Int32Array { + return this.#starts.subarray(0, this.#length); + } + + /** End offsets, exclusive, in source order. A live view, like {@link starts}. */ + get ends(): Int32Array { + return this.#ends.subarray(0, this.#length); + } + + /** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */ + get kinds(): Uint8Array { + return this.#kinds.subarray(0, this.#length); + } + + /** Append one token and return its index. Grows geometrically, so a scan stays linear. */ + push(start: number, end: number, kind: TokenKind): number { + const index = this.#length; + if (index === this.#starts.length) this.#grow(); + this.#starts[index] = start; + this.#ends[index] = end; + this.#kinds[index] = CODE_BY_KIND[kind]; + this.#length = index + 1; + return index; + } + + /** The start offset of one token. */ + startAt(index: number): number { + return this.#starts[this.#checked(index)]!; + } + + /** The end offset of one token, exclusive. */ + endAt(index: number): number { + return this.#ends[this.#checked(index)]!; + } + + /** The kind of one token. */ + kindAt(index: number): TokenKind { + return KIND_BY_CODE[this.#kinds[this.#checked(index)]!]!; + } + + /** The stored kind code of one token, for a consumer encoding kinds numerically. */ + kindCodeAt(index: number): number { + return this.#kinds[this.#checked(index)]!; + } + + /** + * Build a `Token` object for one stored token. + * + * The only place a token becomes an object. Called for the tokens a consumer actually reads, and + * never on the way in. + */ + materialise(index: number): Token { + const at = this.#checked(index); + return { + start: this.#starts[at]!, + end: this.#ends[at]!, + kind: KIND_BY_CODE[this.#kinds[at]!]!, + }; + } + + /** + * Every stored token, materialised one at a time. + * + * Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads + * the arrays or goes through `classify`, which yields only what it is asked for. Note that this + * yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored + * and is computed as the complement. + */ + *[Symbol.iterator](): IterableIterator { + for (let index = 0; index < this.#length; index += 1) yield this.materialise(index); + } + + #checked(index: number): number { + if (!Number.isInteger(index) || index < 0 || index >= this.#length) { + throw new RangeError(`No token at index ${index} (${this.#length} stored)`); + } + return index; + } + + #grow(): void { + const capacity = this.#starts.length === 0 ? INITIAL_CAPACITY : this.#starts.length * 2; + const starts = new Int32Array(capacity); + const ends = new Int32Array(capacity); + const kinds = new Uint8Array(capacity); + starts.set(this.#starts); + ends.set(this.#ends); + kinds.set(this.#kinds); + this.#starts = starts; + this.#ends = ends; + this.#kinds = kinds; + } +} diff --git a/packages/core/tests/__snapshots__/parse.test.ts.snap b/packages/core/tests/__snapshots__/parse.test.ts.snap new file mode 100644 index 0000000..a1b473a --- /dev/null +++ b/packages/core/tests/__snapshots__/parse.test.ts.snap @@ -0,0 +1,363 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`reading is unchanged by positioning > produces the same document and the same diagnostics for every syntax fixture 1`] = ` +"--- comma-inside-trailing-comment +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": 0 + } + } +} + +--- comment-between-separator-and-value +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": 0 + } + } +} + +--- comments-only +diagnostics: [] +document: {} + +--- duplicate-object-name +diagnostics: [ + { + "message": "A Zone named \\"Zone One\\" already exists", + "line": 7, + "column": 1, + "typeName": "Zone", + "objectName": "Zone One", + "code": "ParseError" + } +] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0 + } + } +} + +--- empty +diagnostics: [] +document: {} + +--- line-endings-crlf +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": 0 + } + }, + "Timestep": { + "Timestep 1": { + "number_of_timesteps_per_hour": 6 + } + } +} + +--- line-endings-lf +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": 0 + } + }, + "Timestep": { + "Timestep 1": { + "number_of_timesteps_per_hour": 6 + } + } +} + +--- line-endings-mixed +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": 0 + } + }, + "Timestep": { + "Timestep 1": { + "number_of_timesteps_per_hour": 6 + } + } +} + +--- missing-terminator-swallows-next +diagnostics: [ + { + "message": "Field \\"x_origin\\" expects a number, got \\"0.0\\n\\nZone\\"", + "line": 6, + "typeName": "Zone", + "objectName": "Zone One", + "code": "InvalidField" + }, + { + "message": "Field \\"y_origin\\" expects a number, got \\"Zone Two\\"", + "line": 9, + "typeName": "Zone", + "objectName": "Zone One", + "code": "InvalidField" + } +] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": "0.0\\n\\nZone", + "y_origin": "Zone Two", + "z_origin": 0, + "type": 0 + } + }, + "Timestep": { + "Timestep 1": { + "number_of_timesteps_per_hour": 6 + } + } +} + +--- no-version-declared +diagnostics: [] +document: { + "Zone": { + "Zone One": { + "direction_of_relative_north": 0 + } + } +} + +--- single-unterminated-word +diagnostics: [ + { + "message": "Unterminated object near \\"Zone\\" (missing \\";\\")", + "line": 1, + "code": "ParseError", + "column": 1 + } +] +document: {} + +--- surface-bad-ninth-vertex +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Material:NoMass": { + "R13": { + "roughness": "Rough", + "thermal_resistance": 2.29, + "thermal_absorptance": 0.9, + "solar_absorptance": 0.75, + "visible_absorptance": 0.75 + } + }, + "Construction": { + "ExtWall": { + "outside_layer": "R13" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0, + "x_origin": 0, + "y_origin": 0, + "z_origin": 0 + } + }, + "BuildingSurface:Detailed": { + "South Wall": { + "surface_type": "Wall", + "construction_name": "ExtWall", + "zone_name": "Zone One", + "outside_boundary_condition": "Outdoors", + "sun_exposure": "SunExposed", + "wind_exposure": "WindExposed", + "view_factor_to_ground": 0.5, + "number_of_vertices": 12, + "vertices": [ + { + "vertex_x_coordinate": 0, + "vertex_y_coordinate": 0, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 4, + "vertex_y_coordinate": 0, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 8, + "vertex_y_coordinate": 0, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 12, + "vertex_y_coordinate": 0, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 12, + "vertex_y_coordinate": 4, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 12, + "vertex_y_coordinate": 8, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 8, + "vertex_y_coordinate": 8, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 4, + "vertex_y_coordinate": 8, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": "not-a-number", + "vertex_y_coordinate": 8, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 0, + "vertex_y_coordinate": 4, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 0, + "vertex_y_coordinate": 2, + "vertex_z_coordinate": 3 + }, + { + "vertex_x_coordinate": 0, + "vertex_y_coordinate": 0, + "vertex_z_coordinate": 3 + } + ] + } + } +} + +--- unknown-object-type +diagnostics: [ + { + "message": "Unknown object type \\"NotAnObject:AtAll\\" in EnergyPlus 26.1.0", + "line": 3, + "column": 1, + "typeName": "NotAnObject:AtAll", + "objectName": "Whatever", + "code": "UnknownObjectType" + } +] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + }, + "Timestep": { + "Timestep 1": { + "number_of_timesteps_per_hour": 6 + } + } +} + +--- unsupported-version +diagnostics: [] +document: { + "Version": { + "Version 1": { + "version_identifier": "7.0" + } + }, + "Zone": { + "Zone One": { + "direction_of_relative_north": 0 + } + } +} + +--- unterminated-final-statement +diagnostics: [ + { + "message": "Unterminated object near \\"0.0\\" (missing \\";\\")", + "line": 3, + "code": "ParseError", + "column": 1, + "typeName": "Zone" + } +] +document: { + "Version": { + "Version 1": { + "version_identifier": "26.1" + } + } +} + +--- value-across-two-lines +diagnostics: [] +document: { + "Zone": { + "My\\n Zone": { + "direction_of_relative_north": 0 + } + } +}" +`; diff --git a/packages/core/tests/__snapshots__/validate.test.ts.snap b/packages/core/tests/__snapshots__/validate.test.ts.snap new file mode 100644 index 0000000..31745ac --- /dev/null +++ b/packages/core/tests/__snapshots__/validate.test.ts.snap @@ -0,0 +1,172 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`validating is unchanged by positioning > produces the same findings for every syntax fixture 1`] = ` +"--- comma-inside-trailing-comment +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- comment-between-separator-and-value +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- comments-only +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- duplicate-object-name +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- empty +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- line-endings-crlf +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- line-endings-lf +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- line-endings-mixed +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- missing-terminator-swallows-next +{ + "errors": [ + { + "severity": "error", + "objType": "Zone", + "objName": "Zone One", + "field": "x_origin", + "message": "Expected number, got string", + "code": "E003" + }, + { + "severity": "error", + "objType": "Zone", + "objName": "Zone One", + "field": "y_origin", + "message": "Expected number, got string", + "code": "E003" + }, + { + "severity": "error", + "objType": "Zone", + "objName": "Zone One", + "field": "type", + "message": "Value 0 is below minimum 1", + "code": "E005" + } + ], + "warnings": [], + "info": [], + "isValid": false, + "totalIssues": 3 +} + +--- no-version-declared +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- single-unterminated-word +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- surface-bad-ninth-vertex +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- unknown-object-type +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- unsupported-version +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- unterminated-final-statement +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +} + +--- value-across-two-lines +{ + "errors": [], + "warnings": [], + "info": [], + "isValid": true, + "totalIssues": 0 +}" +`; diff --git a/packages/core/tests/classify.test.ts b/packages/core/tests/classify.test.ts new file mode 100644 index 0000000..8acbea8 --- /dev/null +++ b/packages/core/tests/classify.test.ts @@ -0,0 +1,393 @@ +import { describe, expect, it } from 'vitest'; + +import { + classify, + lex, + lineColumnAt, + scanIdf, + type LexDiagnostic, + type SyntaxLayer, + type Token, + type TokenKind, +} from '@idfkit/core'; + +import { syntaxFixture, syntaxFixtures } from './helpers.js'; + +/** + * What the kinds mean. + * + * The tiling of the *stored* tokens is asserted in `scan.test.ts`; this file is about the + * classified stream, which is a different thing: it fills the gaps with `trivia`, splits at line + * boundaries, and is the only place a consumer sees a kind at all. So coverage is re-checked here + * on the yielded stream, and everything else asked of it is about what each kind selects. + */ + +/** Every kind the layer stores. `trivia` is the complement and is never stored, only yielded. */ +const MEANINGFUL_KINDS: readonly TokenKind[] = [ + 'typeName', + 'value', + 'separator', + 'terminator', + 'comment', +]; + +/** + * Where a classified stream stops covering the text exactly once, described rather than counted. + * + * Empty, and only empty, when the stream begins at 0, ends at `text.length`, and has no gap and no + * overlap between. A failure names the offset and the clause, so a regression reads as "gap at + * [38,40)" rather than as two long strings differing somewhere. + */ +function coverageBreaks(layer: SyntaxLayer, tokens: readonly Token[]): string[] { + const breaks: string[] = []; + let covered = 0; + for (const token of tokens) { + if (token.end <= token.start) { + breaks.push(`empty or inverted token [${token.start},${token.end})`); + } + if (token.start < covered) breaks.push(`overlap at ${token.start}, covered to ${covered}`); + else if (token.start > covered) breaks.push(`gap [${covered},${token.start})`); + covered = Math.max(covered, token.end); + } + if (covered !== layer.text.length) { + breaks.push(`stream ends at ${covered}, text is ${layer.text.length} long`); + } + return breaks; +} + +/** + * Whether a token spans a line boundary, measured on its last character rather than on `end`. + * + * A region is half-open, so the last character it selects is at `end - 1`. That distinction is the + * whole of this predicate: a value split at a newline ends at the *following* line's first offset, + * and measuring the line of `end` would report every split token as a crossing when none of them + * is. A token may therefore contain the line feed that ends its own line, which is inside that + * line rather than across it. + */ +function crossesLine(layer: SyntaxLayer, token: Token): boolean { + return lineColumnAt(layer, token.start).line !== lineColumnAt(layer, token.end - 1).line; +} + +/** Whether any non-whitespace precedes a token on its own line, which is what "trails" means. */ +function trails(text: string, token: Token): boolean { + const lineStart = text.lastIndexOf('\n', token.start - 1) + 1; + return text.slice(lineStart, token.start).trim() !== ''; +} + +/** The tokens of one kind, in source order. */ +function of(tokens: readonly Token[], kind: TokenKind): readonly Token[] { + return tokens.filter((token) => token.kind === kind); +} + +/** One fixture, scanned and classified, with everything the assertions below want to look at. */ +function classified(name: string): { + readonly text: string; + readonly layer: SyntaxLayer; + readonly tokens: readonly Token[]; +} { + const text = syntaxFixture(name); + const layer = scanIdf(text); + return { text, layer, tokens: [...classify(layer)] }; +} + +describe('classify', () => { + describe('coverage and kinds', () => { + it('covers every character of every fixture exactly once', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + expect(coverageBreaks(layer, [...classify(layer)]), fixture.name).toEqual([]); + } + }); + + it('distinguishes a type name, a value, a separator, a terminator and a comment', () => { + // One file carrying all five, so the claim is that they are told apart within the same text + // rather than that five files each managed one. + const { tokens } = classified('comma-inside-trailing-comment'); + const kinds = tokens.map((token) => token.kind); + for (const kind of MEANINGFUL_KINDS) { + expect(kinds, `no ${kind} token`).toContain(kind); + } + expect(kinds).toContain('trivia'); + }); + + it('selects the type name each statement declares, and nothing else', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + const drawn = of([...classify(layer)], 'typeName'); + + // A type name that was written is drawn in full: the tokens inside its stored region + // reassemble the text the statement reports for it. + for (const statement of layer.statements) { + if (statement.typeName.end === statement.typeName.start) continue; + const parts = drawn.filter( + (token) => + token.start >= statement.typeName.start && token.end <= statement.typeName.end + ); + const text = parts.map((token) => fixture.text.slice(token.start, token.end)).join(''); + expect(text, `${fixture.name} at ${statement.typeName.start}`).toBe( + statement.typeNameText + ); + } + + // And nothing that is not one is drawn as one. + for (const token of drawn) { + const owner = layer.statements.find( + (statement) => + token.start >= statement.typeName.start && token.end <= statement.typeName.end + ); + expect( + owner, + `${fixture.name}: typeName [${token.start},${token.end}) belongs to no statement` + ).toBeDefined(); + } + } + }); + + it('selects each field value in full, delimiters excluded', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + const drawn = of([...classify(layer)], 'value'); + + for (const statement of layer.statements) { + for (const field of statement.fields) { + if (field.end === field.start) continue; + const parts = drawn.filter( + (token) => token.start >= field.start && token.end <= field.end + ); + const text = parts.map((token) => fixture.text.slice(token.start, token.end)).join(''); + expect(text, `${fixture.name} at ${field.start}`).toBe( + fixture.text.slice(field.start, field.end) + ); + } + } + + // No value carries a delimiter or a comment mark, whichever field it belongs to: those + // three characters are exactly what ends a field, so one inside a value would mean the + // value had swallowed a token of another kind. + const fields = layer.statements.flatMap((statement) => statement.fields); + for (const token of drawn) { + const text = fixture.text.slice(token.start, token.end); + expect(text, `${fixture.name} at ${token.start}`).not.toMatch(/[,;!]/); + + // A value token can be blank, but only in the middle of a field the text wrote across + // several lines: `missing-terminator-swallows-next` has a field running through a blank + // line, and the line split hands that line back as a value with nothing on it. What it + // must never be is a blank token standing on its own where trivia belongs. + if (text.trim() !== '') continue; + const spanning = fields.some( + (field) => field.start < token.start && field.end > token.end + ); + expect(spanning, `${fixture.name}: blank value at ${token.start} spans no field`).toBe( + true + ); + } + } + }); + + it('selects one comma per separator and one semicolon per terminator', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + const tokens = [...classify(layer)]; + + for (const token of of(tokens, 'separator')) { + expect(fixture.text.slice(token.start, token.end), fixture.name).toBe(','); + } + for (const token of of(tokens, 'terminator')) { + expect(fixture.text.slice(token.start, token.end), fixture.name).toBe(';'); + } + + // A terminator is what makes a statement terminated, so the two counts are one fact. + expect(of(tokens, 'terminator').length, fixture.name).toBe( + layer.statements.filter((statement) => !statement.unterminated).length + ); + } + }); + + it('selects a comment from its mark to the end of its line', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + for (const token of of([...classify(layer)], 'comment')) { + const text = fixture.text.slice(token.start, token.end); + expect(text, `${fixture.name} at ${token.start}`).toMatch(/^!/); + // The line feed itself is trivia. A carriage return before it is the last character + // before the newline and so belongs to the comment, which is why this looks for a line + // feed rather than for trailing whitespace generally. + expect(text, `${fixture.name} at ${token.start}`).not.toContain('\n'); + } + } + }); + + it('selects only whitespace as trivia', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + for (const token of of([...classify(layer)], 'trivia')) { + const text = fixture.text.slice(token.start, token.end); + expect(text, `${fixture.name} at ${token.start}`).toMatch(/^\s+$/); + } + } + }); + }); + + describe('comments', () => { + it('keeps a comment trailing a value out of that value', () => { + const { text, tokens } = classified('comma-inside-trailing-comment'); + const name = of(tokens, 'value')[1]; + const comment = of(tokens, 'comment')[0]; + + expect(name && text.slice(name.start, name.end)).toBe('Zone One'); + expect(comment && text.slice(comment.start, comment.end)).toBe( + '!- Name, which is followed here by a comma; and a semicolon' + ); + // Its own region, opening after the value closes: an underline drawn on the value lands on + // `Zone One` and stops there rather than running through the annotation beside it. + expect(comment!.start).toBeGreaterThan(name!.end); + expect(text.slice(name!.start, name!.end)).not.toContain('!'); + }); + + it('tells a comment that trails a value from one that has its line to itself', () => { + const trailing = classified('comma-inside-trailing-comment'); + const alone = classified('comment-between-separator-and-value'); + + // The rule is whether any non-whitespace precedes the mark on the same line, and it is + // decidable from the token stream alone, with no schema and no parse. + const trailingComments = of(trailing.tokens, 'comment'); + expect(trailingComments).toHaveLength(3); + expect(trailingComments.map((token) => trails(trailing.text, token))).toEqual([ + true, + true, + true, + ]); + + const loneComments = of(alone.tokens, 'comment'); + expect(loneComments).toHaveLength(2); + expect(loneComments.map((token) => trails(alone.text, token))).toEqual([false, false]); + + // And a comment sitting between a separator and its value is part of neither: the separator + // closes before it, and the value opens after it. + const separatorBefore = of(alone.tokens, 'separator').find( + (token) => token.end <= loneComments[0]!.start + ); + const valueAfter = of(alone.tokens, 'value').find( + (token) => token.start >= loneComments[1]!.end + ); + expect(separatorBefore).toBeDefined(); + expect(valueAfter && alone.text.slice(valueAfter.start, valueAfter.end)).toBe('0.0'); + }); + + it('does not read a comma or a semicolon inside a comment as a delimiter', () => { + const { text, tokens } = classified('comma-inside-trailing-comment'); + const comment = of(tokens, 'comment')[0]!; + const inside = text.slice(comment.start, comment.end); + + // The fixture exists to put both delimiters inside a comment. If it ever stopped containing + // them this test would keep passing while proving nothing, so it says so. + expect(inside).toContain(','); + expect(inside).toContain(';'); + + for (const token of tokens) { + if (token.kind !== 'separator' && token.kind !== 'terminator') continue; + expect( + token.start >= comment.end || token.end <= comment.start, + `delimiter [${token.start},${token.end}) is inside the comment` + ).toBe(true); + } + + // Four commas and two semicolons are written as delimiters in this file; the pair inside the + // comment adds to neither count. + expect(of(tokens, 'separator')).toHaveLength(4); + expect(of(tokens, 'terminator')).toHaveLength(2); + }); + }); + + describe('without a parse and without a schema', () => { + it('classifies text that does not parse, completely', () => { + for (const name of ['single-unterminated-word', 'unterminated-final-statement']) { + const { text, layer, tokens } = classified(name); + + // "Does not parse" is shown rather than asserted by naming: `lex` reports the grammar + // violation and drops the statement it could not terminate, so the reader ends up with + // strictly fewer objects than the text wrote statements. + const diagnostics: LexDiagnostic[] = []; + const objects = lex(text, { onDiagnostic: (d) => diagnostics.push(d) }); + expect(diagnostics.length, name).toBeGreaterThan(0); + expect(objects.length, name).toBeLessThan(layer.statements.length); + + // The layer represents the violation instead of stopping at it, so every character is + // still classified: colouring a file somebody is halfway through typing is the point. + expect(coverageBreaks(layer, tokens), name).toEqual([]); + expect(layer.statements.at(-1)?.unterminated, name).toBe(true); + } + }); + + it('classifies with no schema available at all', () => { + // There is no schema in the call because there is no parameter for one: `scanIdf` takes text + // and nothing else, and `classify` takes the layer and nothing else. That is what FR-006 + // claims, and an arity is how the claim is observable from outside. + expect(scanIdf).toHaveLength(1); + expect(classify).toHaveLength(1); + + // Two files no schema resolves for: one declaring no version at all, one declaring a version + // this repository ships nothing for. Both classify, and both distinguish all five kinds. + for (const name of ['no-version-declared', 'unsupported-version']) { + const { layer, tokens } = classified(name); + expect(coverageBreaks(layer, tokens), name).toEqual([]); + const kinds = tokens.map((token) => token.kind); + for (const kind of MEANINGFUL_KINDS) { + expect(kinds, `${name}: no ${kind} token`).toContain(kind); + } + } + }); + }); + + describe('line boundaries', () => { + // `Zone,\n My\n Zone,\n 0.0;\n`: one field written across two lines. The two assertions + // below are the point of that fixture, and they are kept apart deliberately. A regression that + // stopped splitting fails the second alone; a regression that stopped storing whole regions, + // cutting the field at the newline on the way in, fails the first alone. + + it('stores a value region that does span a line boundary', () => { + const { text, layer } = classified('value-across-two-lines'); + const name = layer.statements[0]?.fields[0]; + + // Stated here so that a fixture quietly reformatted, which the corpus README warns against, + // fails as itself rather than as a puzzling assertion about offsets further down. + expect(text).toBe('Zone,\n My\n Zone,\n 0.0;\n'); + expect(name).toBeDefined(); + expect(text.slice(name!.start, name!.end)).toBe('My\n Zone'); + expect(lineColumnAt(layer, name!.start).line).toBe(2); + expect(lineColumnAt(layer, name!.end - 1).line).toBe(3); + }); + + it('yields no token that does, while coverage stays exact', () => { + const { text, layer, tokens } = classified('value-across-two-lines'); + const stored = layer.statements[0]!.fields[0]!; + + const crossing = tokens + .filter((token) => crossesLine(layer, token)) + .map((token) => `${token.kind} [${token.start},${token.end})`); + expect(crossing).toEqual([]); + expect(coverageBreaks(layer, tokens)).toEqual([]); + + // The split adds tokens and moves no boundary, so the halves of the stored region are still + // exactly the stored region. + const halves = of(tokens, 'value').filter( + (token) => token.start >= stored.start && token.end <= stored.end + ); + expect(halves).toHaveLength(2); + expect(halves.map((token) => text.slice(token.start, token.end)).join('')).toBe('My\n Zone'); + }); + + it('yields no token that crosses a line boundary, anywhere in the corpus (SC-017)', () => { + for (const fixture of syntaxFixtures()) { + const layer = scanIdf(fixture.text); + const tokens = [...classify(layer)]; + const crossing = tokens + .filter((token) => crossesLine(layer, token)) + .map((token) => `${token.kind} [${token.start},${token.end})`); + + expect(crossing, fixture.name).toEqual([]); + expect(coverageBreaks(layer, tokens), fixture.name).toEqual([]); + } + }); + }); +}); diff --git a/packages/core/tests/document.test.ts b/packages/core/tests/document.test.ts index c94dd4b..2955064 100644 --- a/packages/core/tests/document.test.ts +++ b/packages/core/tests/document.test.ts @@ -1,6 +1,6 @@ import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import { IdfDocument, shapeOf } from '@idfkit/core'; +import { IdfDocument, parseIdf, shapeOf } from '@idfkit/core'; import type { Schema } from '@idfkit/schemas'; import { schema } from './helpers.js'; @@ -259,3 +259,71 @@ describe('type-name lookup', () => { expect(fresh.types()).toEqual(['Zone']); }); }); + +/** + * The two facts the language service's correlation rests on (research R6). + * + * `@idfkit/language` positions a validation finding by matching the object it names against the + * statements in the text: by folded type name and folded object name for a named object, and by + * folded type name and ordinal for an anonymous one. Neither key is sound on its own; each is sound + * only because of behaviour these two tests pin down. Both are asserted rather than trusted because + * a change to either breaks correlation *silently*: findings keep arriving, they just start + * underlining the wrong object, and no other test in either repository would notice. + */ +describe('what positioning a finding depends on', () => { + it('never lets a document parsed from text hold two objects of one type under one name', () => { + // Without this the name key would be ambiguous, and duplicate names are common in real files. + // `addRaw` throws on the second one, `parseIdf` catches that, records a `ParseError` and skips + // the object, so the duplicate reaches a reader as a reading finding positioned by the scanner + // that saw it, and never reaches correlation at all. + doc.addRaw('Zone', 'Zone One'); + expect(() => doc.addRaw('Zone', 'Zone One')).toThrow(/already exists/); + // Folded, because that is the key correlation uses and the key a collection stores under. + expect(() => doc.addRaw('Zone', 'ZONE ONE')).toThrow(/already exists/); + + const text = [ + 'Version, 26.1;', + '', + 'Zone,', + ' Zone One,', + ' 0.0;', + '', + 'Zone,', + ' Zone One,', + ' 90.0;', + '', + ].join('\n'); + const parsed = parseIdf(text, v26, { strict: false }); + + expect(parsed.diagnostics.map((d) => d.code)).toEqual(['ParseError']); + expect(parsed.diagnostics[0]?.line).toBe(7); + // The first statement is the one the document kept, so a finding about this type and this name + // is a finding about the first statement in the text. + expect(parsed.document.all('Zone').size).toBe(1); + expect(parsed.document.require('Zone', 'Zone One').get('direction_of_relative_north')).toBe(0); + }); + + it('preserves insertion order, so the Nth object of a type is the Nth statement', () => { + // The ordinal key serves anonymous objects, whose findings carry an empty `objName` and so + // cannot be correlated by name at all. It is sound only while `parseIdf` adds in source order + // and `IdfCollection` hands the objects back in the order they were inserted. + for (const name of ['C', 'A', 'B']) doc.add('Zone', name); + expect([...doc.all('Zone')].map((zone) => zone.name)).toEqual(['C', 'A', 'B']); + + const text = [ + 'Version, 26.1;', + '', + 'Output:Variable, *, Zone Air Temperature, Hourly;', + 'Output:Variable, *, Site Outdoor Air Drybulb Temperature, Hourly;', + 'Output:Variable, *, Zone Mean Air Temperature, Daily;', + '', + ].join('\n'); + const { document } = parseIdf(text, v26, { strict: false }); + + expect([...document.all('Output:Variable')].map((obj) => obj.get('variable_name'))).toEqual([ + 'Zone Air Temperature', + 'Site Outdoor Air Drybulb Temperature', + 'Zone Mean Air Temperature', + ]); + }); +}); diff --git a/packages/core/tests/fixtures/syntax/README.md b/packages/core/tests/fixtures/syntax/README.md new file mode 100644 index 0000000..291e49b --- /dev/null +++ b/packages/core/tests/fixtures/syntax/README.md @@ -0,0 +1,37 @@ +# Syntax fixtures + +One file per edge case named in the specification for the IDF language service +(feature 005, task T002). Every one of them is text a real file can contain, and +several of them are text that does not parse, which is the point: the syntax +layer is required to be produced for text that violates the grammar and to +represent the violation rather than stopping at it. + +Read them with `syntaxFixture` and `syntaxFixtures` from `tests/helpers.ts`, +which read the bytes as they are on disk. Do not read them through anything that +normalises line endings, and do not reformat them: three of them exist only to +carry a particular line ending, and an editor that helpfully converts one of +those has destroyed the fixture rather than tidied it. + +| File | What it is for | +| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `empty.idf` | Zero bytes. The tiling invariant has to hold vacuously. | +| `comments-only.idf` | Comments and nothing else, and no version either. | +| `single-unterminated-word.idf` | The four bytes `Zone`, no separator, no terminator, no trailing newline. | +| `unterminated-final-statement.idf` | A last statement with no `;`, which runs to end of input. | +| `missing-terminator-swallows-next.idf` | A missing `;` that swallows the statement below it, so one object extends far past where it looks like it ends. | +| `line-endings-lf.idf` | Line feed only. | +| `line-endings-crlf.idf` | Carriage return and line feed only, on every line. | +| `line-endings-mixed.idf` | Both conventions in one file, alternating. | +| `value-across-two-lines.idf` | A field value written across two lines, so its stored region crosses a line boundary while no drawn token may. | +| `comment-between-separator-and-value.idf` | A comment sitting between a separator and the value that follows it. | +| `comma-inside-trailing-comment.idf` | A comma and a semicolon inside a comment trailing a value, neither of which is a delimiter. | +| `surface-bad-ninth-vertex.idf` | A surface with twelve vertices whose ninth carries a value that is not a number. | +| `duplicate-object-name.idf` | Two objects of one type declaring the same name, which the schema forbids and real files contain. | +| `unknown-object-type.idf` | An object of a type the schema does not define. | +| `no-version-declared.idf` | No `Version` statement, so no schema resolves. | +| `unsupported-version.idf` | A version this repository ships no schema for. | + +The line-ending fixtures carry the same statements deliberately, so a test that +finds them classifying differently has found a line-ending bug rather than a +content difference. Their byte counts are 135, 143 and 139: identical text, four +or eight extra carriage returns. diff --git a/packages/core/tests/fixtures/syntax/comma-inside-trailing-comment.idf b/packages/core/tests/fixtures/syntax/comma-inside-trailing-comment.idf new file mode 100644 index 0000000..3ed84cd --- /dev/null +++ b/packages/core/tests/fixtures/syntax/comma-inside-trailing-comment.idf @@ -0,0 +1,6 @@ +Version, 26.1; + +Zone, + Zone One, !- Name, which is followed here by a comma; and a semicolon + 0.0, !- Direction of Relative North + 0.0; !- X Origin diff --git a/packages/core/tests/fixtures/syntax/comment-between-separator-and-value.idf b/packages/core/tests/fixtures/syntax/comment-between-separator-and-value.idf new file mode 100644 index 0000000..195d1bb --- /dev/null +++ b/packages/core/tests/fixtures/syntax/comment-between-separator-and-value.idf @@ -0,0 +1,8 @@ +Version, 26.1; + +Zone, + Zone One, + !- the separator above and the value below are two lines apart, + !- and this comment sits between them + 0.0, + 0.0; diff --git a/packages/core/tests/fixtures/syntax/comments-only.idf b/packages/core/tests/fixtures/syntax/comments-only.idf new file mode 100644 index 0000000..95fce8e --- /dev/null +++ b/packages/core/tests/fixtures/syntax/comments-only.idf @@ -0,0 +1,6 @@ +! A model somebody started and never wrote a statement into. +!-Generator IDFEditor 1.51 +!-Option SortedOrder +! +! There is no Version statement here either, so nothing below can be +! resolved against a schema. Classification must still cover every byte. diff --git a/packages/core/tests/fixtures/syntax/duplicate-object-name.idf b/packages/core/tests/fixtures/syntax/duplicate-object-name.idf new file mode 100644 index 0000000..c4d8965 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/duplicate-object-name.idf @@ -0,0 +1,9 @@ +Version, 26.1; + +Zone, + Zone One, !- Name + 0.0; !- Direction of Relative North + +Zone, + Zone One, !- Name + 90.0; !- Direction of Relative North diff --git a/packages/core/tests/fixtures/syntax/empty.idf b/packages/core/tests/fixtures/syntax/empty.idf new file mode 100644 index 0000000..e69de29 diff --git a/packages/core/tests/fixtures/syntax/line-endings-crlf.idf b/packages/core/tests/fixtures/syntax/line-endings-crlf.idf new file mode 100644 index 0000000..70ac6f4 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/line-endings-crlf.idf @@ -0,0 +1,8 @@ +Version, 26.1; + +Zone, + Zone One, !- Name + 0.0, !- Direction of Relative North + 0.0; !- X Origin + +Timestep, 6; diff --git a/packages/core/tests/fixtures/syntax/line-endings-lf.idf b/packages/core/tests/fixtures/syntax/line-endings-lf.idf new file mode 100644 index 0000000..a9ca655 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/line-endings-lf.idf @@ -0,0 +1,8 @@ +Version, 26.1; + +Zone, + Zone One, !- Name + 0.0, !- Direction of Relative North + 0.0; !- X Origin + +Timestep, 6; diff --git a/packages/core/tests/fixtures/syntax/line-endings-mixed.idf b/packages/core/tests/fixtures/syntax/line-endings-mixed.idf new file mode 100644 index 0000000..35b3e66 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/line-endings-mixed.idf @@ -0,0 +1,8 @@ +Version, 26.1; + +Zone, + Zone One, !- Name + 0.0, !- Direction of Relative North + 0.0; !- X Origin + +Timestep, 6; diff --git a/packages/core/tests/fixtures/syntax/missing-terminator-swallows-next.idf b/packages/core/tests/fixtures/syntax/missing-terminator-swallows-next.idf new file mode 100644 index 0000000..2d15b48 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/missing-terminator-swallows-next.idf @@ -0,0 +1,13 @@ +Version, 26.1; + +Zone, + Zone One, + 0.0, + 0.0 + +Zone, + Zone Two, + 0.0, + 0.0; + +Timestep, 6; diff --git a/packages/core/tests/fixtures/syntax/no-version-declared.idf b/packages/core/tests/fixtures/syntax/no-version-declared.idf new file mode 100644 index 0000000..1465905 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/no-version-declared.idf @@ -0,0 +1,5 @@ +! No Version statement, so no schema can be resolved for this text. + +Zone, + Zone One, !- Name + 0.0; !- Direction of Relative North diff --git a/packages/core/tests/fixtures/syntax/single-unterminated-word.idf b/packages/core/tests/fixtures/syntax/single-unterminated-word.idf new file mode 100644 index 0000000..74c7417 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/single-unterminated-word.idf @@ -0,0 +1 @@ +Zone \ No newline at end of file diff --git a/packages/core/tests/fixtures/syntax/surface-bad-ninth-vertex.idf b/packages/core/tests/fixtures/syntax/surface-bad-ninth-vertex.idf new file mode 100644 index 0000000..d24fef0 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/surface-bad-ninth-vertex.idf @@ -0,0 +1,45 @@ +Version, 26.1; + +Material:NoMass, + R13, !- Name + Rough, !- Roughness + 2.29, !- Thermal Resistance {m2-K/W} + 0.9, !- Thermal Absorptance + 0.75, !- Solar Absorptance + 0.75; !- Visible Absorptance + +Construction, + ExtWall, !- Name + R13; !- Outside Layer + +Zone, + Zone One, !- Name + 0.0, !- Direction of Relative North + 0.0, !- X Origin + 0.0, !- Y Origin + 0.0; !- Z Origin + +BuildingSurface:Detailed, + South Wall, !- Name + Wall, !- Surface Type + ExtWall, !- Construction Name + Zone One, !- Zone Name + , !- Space Name + Outdoors, !- Outside Boundary Condition + , !- Outside Boundary Condition Object + SunExposed, !- Sun Exposure + WindExposed, !- Wind Exposure + 0.5, !- View Factor to Ground + 12, !- Number of Vertices + 0.0, 0.0, 3.0, !- Vertex 1 + 4.0, 0.0, 3.0, !- Vertex 2 + 8.0, 0.0, 3.0, !- Vertex 3 + 12.0, 0.0, 3.0, !- Vertex 4 + 12.0, 4.0, 3.0, !- Vertex 5 + 12.0, 8.0, 3.0, !- Vertex 6 + 8.0, 8.0, 3.0, !- Vertex 7 + 4.0, 8.0, 3.0, !- Vertex 8 + not-a-number, 8.0, 3.0, !- Vertex 9 + 0.0, 4.0, 3.0, !- Vertex 10 + 0.0, 2.0, 3.0, !- Vertex 11 + 0.0, 0.0, 3.0; !- Vertex 12 diff --git a/packages/core/tests/fixtures/syntax/unknown-object-type.idf b/packages/core/tests/fixtures/syntax/unknown-object-type.idf new file mode 100644 index 0000000..34b549e --- /dev/null +++ b/packages/core/tests/fixtures/syntax/unknown-object-type.idf @@ -0,0 +1,7 @@ +Version, 26.1; + +NotAnObject:AtAll, + Whatever, !- Name + 1.0; !- Some Field + +Timestep, 6; diff --git a/packages/core/tests/fixtures/syntax/unsupported-version.idf b/packages/core/tests/fixtures/syntax/unsupported-version.idf new file mode 100644 index 0000000..292216e --- /dev/null +++ b/packages/core/tests/fixtures/syntax/unsupported-version.idf @@ -0,0 +1,5 @@ +Version, 7.0; + +Zone, + Zone One, !- Name + 0.0; !- Direction of Relative North diff --git a/packages/core/tests/fixtures/syntax/unterminated-final-statement.idf b/packages/core/tests/fixtures/syntax/unterminated-final-statement.idf new file mode 100644 index 0000000..8443890 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/unterminated-final-statement.idf @@ -0,0 +1,6 @@ +Version, 26.1; + +Zone, + Zone One, + 0.0, + 0.0 diff --git a/packages/core/tests/fixtures/syntax/value-across-two-lines.idf b/packages/core/tests/fixtures/syntax/value-across-two-lines.idf new file mode 100644 index 0000000..c37b518 --- /dev/null +++ b/packages/core/tests/fixtures/syntax/value-across-two-lines.idf @@ -0,0 +1,4 @@ +Zone, + My + Zone, + 0.0; diff --git a/packages/core/tests/helpers.ts b/packages/core/tests/helpers.ts index 20c3799..517a671 100644 --- a/packages/core/tests/helpers.ts +++ b/packages/core/tests/helpers.ts @@ -1,4 +1,5 @@ -import { existsSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import type { Schema } from '@idfkit/schemas'; import { localBundle, nodeSource } from '@idfkit/schemas/node'; @@ -49,3 +50,61 @@ export function prose(): Promise { .then((value) => value as readonly string[]); return prosePromise; } + +const syntaxFixtureDir = fileURLToPath(new URL('./fixtures/syntax/', import.meta.url)); + +/** One file from the syntax fixture corpus, and where it came from. */ +export interface SyntaxFixture { + /** Its name without the extension, such as `line-endings-crlf`. */ + readonly name: string; + /** Its path on disk, so a failure can say which file broke rather than which index. */ + readonly path: string; + /** Its bytes, decoded as UTF-8 and otherwise unaltered. */ + readonly text: string; +} + +/** + * One syntax fixture's text, exactly as it is on disk. + * + * The whole value of this corpus is that the bytes are the bytes. Three of the + * fixtures differ from each other only in their line endings, and a reader that + * translated those would leave the tests passing against text that no longer + * contains the case they were written for. `readFileSync` performs no such + * translation and neither does decoding UTF-8, which is why the read is spelled + * out here once rather than left to each test to get right. + * + * @param name the fixture's name, without the `.idf` extension + */ +export function syntaxFixture(name: string): string { + const found = syntaxFixtures().find((fixture) => fixture.name === name); + if (found === undefined) { + const available = syntaxFixtures().map((fixture) => fixture.name); + throw new Error(`no syntax fixture named "${name}". There are: ${available.join(', ')}`); + } + return found.text; +} + +/** + * Every syntax fixture, in name order. + * + * Several tests assert a property over the whole corpus rather than over a file + * they chose, which is the point of having one: a fixture added for one case is + * then held to every invariant already proven, without anybody remembering to + * add it to a list. So this enumerates the directory rather than naming its + * contents. + */ +let syntaxFixtureCache: readonly SyntaxFixture[] | undefined; +export function syntaxFixtures(): readonly SyntaxFixture[] { + syntaxFixtureCache ??= readdirSync(syntaxFixtureDir) + .filter((entry) => entry.endsWith('.idf')) + .sort() + .map((entry) => { + const path = `${syntaxFixtureDir}${entry}`; + return { + name: entry.slice(0, -'.idf'.length), + path, + text: readFileSync(path, 'utf8'), + }; + }); + return syntaxFixtureCache; +} diff --git a/packages/core/tests/parse.test.ts b/packages/core/tests/parse.test.ts index cbcac8f..28e4cde 100644 --- a/packages/core/tests/parse.test.ts +++ b/packages/core/tests/parse.test.ts @@ -10,7 +10,7 @@ import { } from '@idfkit/core'; import type { Schema } from '@idfkit/schemas'; -import { schema } from './helpers.js'; +import { schema, syntaxFixtures } from './helpers.js'; let v26: Schema; beforeAll(async () => { @@ -312,3 +312,58 @@ describe('field position with comments between the fields', () => { expect(invalid[0]?.line).toBe(7); }); }); + +/** + * FR-014 and FR-015: reading returns what it returned before the language service existed. + * + * Positions are attached afterwards, by correlating findings against a syntax layer in a separate + * package, so `parseIdf` was never edited and "additive" ought to hold by construction. This is the + * check that makes it hold rather than intend to. The snapshot is every syntax fixture read in full, + * document and diagnostics both, serialised deterministically, so a change to either shows up as a + * diff a reviewer can read instead of as a number nobody was watching. + * + * It cannot testify about a past it did not observe; what it can do is fail the moment reading + * starts producing something different, which is the property FR-015 actually needs. + */ +describe('reading is unchanged by positioning', () => { + /** Every fixture read, in name order, as text a snapshot can diff line by line. */ + function readCorpus(): string { + return syntaxFixtures() + .map(({ name, text }) => { + const { document, diagnostics } = parseIdf(text, v26, { strict: false }); + return [ + `--- ${name}`, + `diagnostics: ${JSON.stringify(diagnostics, null, 2)}`, + `document: ${JSON.stringify(document.toJSON(), null, 2)}`, + ].join('\n'); + }) + .join('\n\n'); + } + + it('produces the same document and the same diagnostics for every syntax fixture', () => { + expect(readCorpus()).toMatchSnapshot(); + }); + + it('attaches nothing to a diagnostic, so an existing caller receives exactly what it did', () => { + // `region` and `precision` belong to a `PositionedFinding`, which is a separate value built by + // `@idfkit/language` from this one. Finding either here would mean the position had been merged + // into the source after all, which is the shape of the change this design exists to avoid. + const declared = new Set([ + 'message', + 'line', + 'column', + 'code', + 'filepath', + 'typeName', + 'objectName', + ]); + + for (const { name, text } of syntaxFixtures()) { + for (const diagnostic of parseIdf(text, v26, { strict: false }).diagnostics) { + for (const key of Object.keys(diagnostic)) { + expect(declared.has(key), `${name}.idf carries "${key}" on a ParseDiagnostic`).toBe(true); + } + } + } + }); +}); diff --git a/packages/core/tests/scan.test.ts b/packages/core/tests/scan.test.ts new file mode 100644 index 0000000..395a69f --- /dev/null +++ b/packages/core/tests/scan.test.ts @@ -0,0 +1,486 @@ +import { beforeAll, describe, expect, it } from 'vitest'; + +import { classify, lex, parseIdf, scanIdf, type RawObject, type SyntaxLayer } from '@idfkit/core'; +import type { Schema } from '@idfkit/schemas'; + +import { schema, syntaxFixture, syntaxFixtures, type SyntaxFixture } from './helpers.js'; + +let v26: Schema; +beforeAll(async () => { + v26 = await schema('26.1.0'); +}); + +const corpus = syntaxFixtures(); + +/** + * The six clauses of the tiling invariant, verbatim from `contracts/syntax-layer.md`. + * + * Numbered here so a failure can name the clause a reader can then go and read, rather than + * describing it in whatever words the assertion happened to use. + */ +const CLAUSES = [ + 'tokens are in source order', + 'no token is empty or inverted', + 'no two tokens overlap', + 'every token lies within [0, text.length]', + 'every character in a gap between two tokens is whitespace', + 'with gaps filled as trivia, the sequence begins at 0 and ends at text.length', +] as const; + +/** + * Fail one clause, at one token, in one fixture. + * + * The whole point of this file is that a breakage is legible. `expect(rebuilt).toBe(text)` on a + * 4 KB fixture reports that two strings differ and leaves a reader to diff them; the invariant is + * six separate properties and exactly one of them will have broken, so the message says which one, + * at which token index, and with the offsets involved. + */ +function fail(clause: number, index: number, detail: string, fixture: string): never { + throw new Error( + `clause ${clause} (${CLAUSES[clause - 1]}) failed at token ${index}: ${detail} in fixture ${fixture}.idf` + ); +} + +/** `[start,end)`, the notation the contract writes regions in. */ +function span(start: number, end: number): string { + return `[${start},${end})`; +} + +/** Offset of the first non-whitespace character in `text`, or -1. `\s` is what `trim` trims. */ +function firstNonSpace(text: string): number { + return text.search(/\S/); +} + +/** + * The positions of a token stream, and nothing else. + * + * The clause walk needs three numbers per token and no behaviour, so it asks for exactly that. A + * `TokenStore` satisfies this, which is what lets the corpus pass one straight in, and so does a + * hand-built stream, which is what lets the negative control below break a clause on purpose + * without having to construct a store in an invalid state. + */ +interface TokenSpans { + readonly length: number; + readonly starts: Int32Array; + readonly ends: Int32Array; +} + +/** + * Clauses 1 to 5, over the stored tokens, in one forward walk. + * + * One pass rather than five, because each clause is a comparison against the token before and the + * order they are checked in matters: an offset outside the text (clause 4) makes every message the + * other four could produce nonsense, and an out-of-order token (clause 1) makes "overlap" and + * "gap" undefined. So the checks run in the order that keeps the message true. + */ +function assertStoredTiling(fixture: SyntaxFixture, text: string, tokens: TokenSpans): void { + const starts = tokens.starts; + const ends = tokens.ends; + + let previousStart = 0; + let previousEnd = 0; + + for (let index = 0; index < tokens.length; index += 1) { + const start = starts[index]!; + const end = ends[index]!; + + if (start < 0 || end > text.length) { + fail(4, index, `${span(start, end)} is outside [0,${text.length}]`, fixture.name); + } + + if (end < start) fail(2, index, `${span(start, end)} is inverted`, fixture.name); + if (end === start) fail(2, index, `${span(start, end)} is empty`, fixture.name); + + if (index > 0) { + if (start < previousStart) { + fail( + 1, + index, + `${span(start, end)} begins before token ${index - 1} at ${span(previousStart, previousEnd)}`, + fixture.name + ); + } + if (start < previousEnd) { + fail( + 3, + index, + `${span(previousStart, previousEnd)} overlaps ${span(start, end)}`, + fixture.name + ); + } + if (start > previousEnd) { + const gap = text.slice(previousEnd, start); + const at = firstNonSpace(gap); + if (at >= 0) { + fail( + 5, + index, + `the gap ${span(previousEnd, start)} before it holds ${JSON.stringify(gap[at])} at offset ${previousEnd + at}`, + fixture.name + ); + } + } + } + + previousStart = start; + previousEnd = end; + } +} + +/** + * Clause 6, and the head and tail's share of clause 5. + * + * Clause 6 is a statement about the sequence with its gaps filled, so it is checked against + * `classify`, which is the thing that fills them, rather than against arithmetic of this file's + * own. Walking that sequence also covers the two regions clause 5 does not reach, the text before + * the first token and after the last: those are not gaps *between* two tokens, but `classify` + * hands them back as `trivia`, and a trivia token holding a letter would mean the fill was not + * trivia at all. So every yielded trivia token is checked to be whitespace here. + */ +function assertFilledTiling(fixture: SyntaxFixture, layer: SyntaxLayer): void { + const { text } = layer; + + let covered = 0; + let index = 0; + for (const token of classify(layer)) { + if (token.start !== covered) { + const detail = + index === 0 + ? `the sequence begins at ${token.start} rather than at 0` + : `${span(token.start, token.end)} leaves ${span(covered, token.start)} covered by nothing`; + fail(6, index, detail, fixture.name); + } + if (token.kind === 'trivia') { + const filled = text.slice(token.start, token.end); + const at = firstNonSpace(filled); + if (at >= 0) { + fail( + 5, + index, + `the trivia filling ${span(token.start, token.end)} holds ${JSON.stringify(filled[at])} at offset ${token.start + at}`, + fixture.name + ); + } + } + covered = token.end; + index += 1; + } + + if (covered !== text.length) { + fail( + 6, + index - 1, + `the sequence ends at ${covered} rather than at ${text.length}`, + fixture.name + ); + } +} + +describe('the tiling invariant', () => { + it('has a corpus to hold it against', () => { + // A property asserted over an empty list passes. Naming the count here is what stops a broken + // fixture loader from turning every test below into a green no-op. + expect(corpus.length).toBeGreaterThanOrEqual(16); + }); + + it.each(corpus)('holds over $name', (fixture) => { + const layer = scanIdf(fixture.text); + assertStoredTiling(fixture, layer.text, layer.tokens); + assertFilledTiling(fixture, layer); + }); + + it('holds vacuously for empty text', () => { + const layer = scanIdf(''); + + expect(layer.statements).toEqual([]); + expect(layer.tokens.length).toBe(0); + expect([...classify(layer)]).toEqual([]); + }); + + it('reports the clause and the token index rather than a string difference', () => { + // The assertion about the assertions. A test whose failure reads "expected 'Zone,\n Zone + // One,...' to be 'Zone,\n Zone One,...'" costs a reader the afternoon this file exists to save + // them, so the shape of the message is itself checked. Clause 3 is broken deliberately, on a + // real fixture's real token positions, and the message the walker produces is read back. + const fixture = corpus.find((entry) => entry.name === 'line-endings-lf')!; + const layer = scanIdf(fixture.text); + const ends = layer.tokens.ends.slice(); + // The first token widened by one character, so it runs into the separator that follows it. + ends[0] = layer.tokens.starts[1]! + 1; + const overlapping: TokenSpans = { + length: layer.tokens.length, + starts: layer.tokens.starts, + ends, + }; + + expect(() => assertStoredTiling(fixture, layer.text, overlapping)).toThrow( + /^clause 3 \(no two tokens overlap\) failed at token 1: \[\d+,\d+\) overlaps \[\d+,\d+\) in fixture line-endings-lf\.idf$/ + ); + }); +}); + +/** + * Byte-identical reconstruction (SC-003, FR-010). + * + * This follows from the invariant above rather than standing alone. The layer holds the text, so a + * reconstruction defined as concatenating slices of that text returns the text by construction: + * what it can prove is that the traversal is total, not that slicing works. It is asserted anyway + * because it is the property the specification names and because it fails loudly if `classify` + * ever stops early, and it sits below the clause walk so that a reader who sees both fail reaches + * for the clause message first. + */ +describe('reconstruction from classify', () => { + it.each(corpus)('rebuilds $name byte for byte', (fixture) => { + const layer = scanIdf(fixture.text); + const rebuilt = [...classify(layer)] + .map((token) => fixture.text.slice(token.start, token.end)) + .join(''); + + expect(rebuilt).toBe(fixture.text); + }); + + it.each(['line-endings-lf', 'line-endings-crlf', 'line-endings-mixed'])( + 'keeps every line ending of %s', + (name) => { + const text = syntaxFixture(name); + const rebuilt = [...classify(scanIdf(text))] + .map((token) => text.slice(token.start, token.end)) + .join(''); + + // Compared on the counts as well as on the text, because a reader looking at a failure of + // this test wants to know whether a carriage return went missing or a whole line did. + const carriageReturns = (source: string): number => source.split('\r').length - 1; + const lineFeeds = (source: string): number => source.split('\n').length - 1; + expect(carriageReturns(rebuilt)).toBe(carriageReturns(text)); + expect(lineFeeds(rebuilt)).toBe(lineFeeds(text)); + expect(rebuilt).toBe(text); + } + ); + + it('rebuilds text that does not parse', () => { + // Named rather than taken from the loop so the claim "including files that do not parse" is + // checked rather than asserted: each of these four is confirmed to produce a diagnostic before + // its reconstruction is checked, so the case cannot quietly become a well-formed file. + const broken = [ + 'single-unterminated-word', + 'unterminated-final-statement', + 'missing-terminator-swallows-next', + 'unknown-object-type', + ]; + + for (const name of broken) { + const text = syntaxFixture(name); + const { diagnostics } = parseIdf(text, v26, { strict: false }); + expect( + diagnostics.map((diagnostic) => diagnostic.code), + `${name} was expected not to parse` + ).not.toEqual([]); + + const rebuilt = [...classify(scanIdf(text))] + .map((token) => text.slice(token.start, token.end)) + .join(''); + expect(rebuilt).toBe(text); + } + }); +}); + +/** + * The statements `lex` reports for one text, paired with the statements `scanIdf` reports. + * + * `lex` yields an object only for a statement that terminated and carried a type name: an + * unterminated one becomes a diagnostic and is dropped, and one written with no type name becomes + * a different diagnostic and is dropped. The layer represents both, because representing what was + * written is its job. So the correspondence is over the statements that survive that filter, and + * the filter is spelled out here rather than left implicit, because a test that quietly compared + * two lists of different lengths would prove nothing. + */ +function pairStatements(text: string): { statements: SyntaxLayer['statements']; raw: RawObject[] } { + const layer = scanIdf(text); + return { + statements: layer.statements.filter( + (statement) => !statement.unterminated && statement.typeNameText !== '' + ), + raw: lex(text), + }; +} + +/** + * The shared scanner's two modes, held against each other (contract: "The shared scanner"). + * + * This is the test T018 exists for. `lex` and `scanIdf` read the same characters through one scan, + * and if they ever drift by one character about where a comment ends, findings land on the wrong + * field and nothing else notices until a file puts a comment somewhere unusual. Comparing the text + * a field's region selects against the value `lex` assembled is what makes that drift a failure. + */ +describe('scanIdf and lex agree on where every field is', () => { + it('has statements and fields to compare', () => { + // Same guard as the corpus count above, one level down: the comparison below is a loop over + // two lists, and two empty lists agree about everything. + let statements = 0; + let fields = 0; + for (const fixture of corpus) { + const paired = pairStatements(fixture.text); + statements += paired.statements.length; + for (const statement of paired.statements) fields += statement.fields.length; + } + + // A floor rather than the exact count, because the corpus is meant to grow: a fixture added + // for one case should be held to every property already proven, not break the guard on them. + expect(statements).toBeGreaterThanOrEqual(30); + expect(fields).toBeGreaterThanOrEqual(100); + }); + + it.each(corpus)('positions every field of $name identically', (fixture) => { + const { text } = fixture; + const { statements, raw } = pairStatements(text); + + expect( + statements.length, + `${fixture.name}: scanIdf reports ${statements.length} terminated, named statements and lex reports ${raw.length} objects` + ).toBe(raw.length); + + for (let index = 0; index < statements.length; index += 1) { + const statement = statements[index]!; + const object = raw[index]!; + const where = `${fixture.name}, statement ${index}`; + + expect( + text.slice(statement.typeName.start, statement.typeName.end), + `${where}: type name` + ).toBe(object.typeName); + expect(statement.typeNameText, `${where}: type name text`).toBe(object.typeName); + expect(statement.region.start, `${where}: statement offset`).toBe(object.offset); + + expect( + statement.fields.length, + `${where}: scanIdf reports ${statement.fields.length} fields and lex reports ${object.values.length} values` + ).toBe(object.values.length); + + for (let field = 0; field < statement.fields.length; field += 1) { + const region = statement.fields[field]!; + const selected = text.slice(region.start, region.end); + expect( + selected, + `${where}, field ${field}: scanIdf selects ${JSON.stringify(selected)} at ${span(region.start, region.end)} and lex reports ${JSON.stringify(object.values[field])}` + ).toBe(object.values[field]); + } + } + }); + + it('positions a field written across two lines on both of its lines', () => { + // The one shape where a stored region crosses a line boundary, so the two modes have the most + // room to disagree: `lex` joins the runs and trims, the layer bounds the region. + const text = syntaxFixture('value-across-two-lines'); + const { statements, raw } = pairStatements(text); + + const region = statements[0]!.fields[0]!; + expect(text.slice(region.start, region.end)).toBe(raw[0]!.values[0]); + expect(text.slice(region.start, region.end)).toContain('\n'); + }); + + it('positions a value separated from its comma by a comment', () => { + const text = syntaxFixture('comment-between-separator-and-value'); + const { statements, raw } = pairStatements(text); + + const zone = statements[1]!; + for (let field = 0; field < zone.fields.length; field += 1) { + const region = zone.fields[field]!; + expect(text.slice(region.start, region.end)).toBe(raw[1]!.values[field]); + } + }); +}); + +/** How many of the layer's two backing array types were constructed while `body` ran. */ +interface LayerAllocations { + readonly int32: number; + readonly uint8: number; +} + +/** + * Count `Int32Array` and `Uint8Array` constructions during one synchronous call. + * + * The layer's storage is three typed arrays and nothing else: two `Int32Array` for starts and ends + * and one `Uint8Array` for kinds, allocated in `TokenStore`'s constructor, plus the `Int32Array` + * line index a position query builds. No other code in this package constructs a typed array at + * all, so counting these two constructors counts the layer exactly. + * + * Structural rather than timed, which is the point: a timing comparison would be measuring the + * machine, and would still pass on a day the layer was built and thrown away. Swapping the global + * binding for a counting `Proxy` is sound because the implementation resolves `Int32Array` from + * the global at construction time, like every other module in this package, and because the + * assertion below runs a positive control through the same probe. An instrument that catches + * nothing proves nothing; one that catches `scanIdf` and not `parseIdf` has measured something. + */ +function countLayerAllocations(body: () => void): LayerAllocations { + const realInt32 = globalThis.Int32Array; + const realUint8 = globalThis.Uint8Array; + let int32 = 0; + let uint8 = 0; + + globalThis.Int32Array = new Proxy(realInt32, { + construct(target, args, newTarget) { + int32 += 1; + return Reflect.construct(target, args, newTarget) as object; + }, + }); + globalThis.Uint8Array = new Proxy(realUint8, { + construct(target, args, newTarget) { + uint8 += 1; + return Reflect.construct(target, args, newTarget) as object; + }, + }); + + try { + body(); + } finally { + globalThis.Int32Array = realInt32; + globalThis.Uint8Array = realUint8; + } + + return { int32, uint8 }; +} + +describe('nothing builds a layer implicitly (FR-005)', () => { + it('allocates none of the layer while parseIdf and lex read the corpus', () => { + // Warm first, outside the probe, so a one-time lazy computation inside the schema is not + // mistaken for the read path allocating. Everything measured below is then a repeat call. + for (const fixture of corpus) { + lex(fixture.text); + parseIdf(fixture.text, v26, { strict: false }); + } + + const allocations = countLayerAllocations(() => { + for (const fixture of corpus) { + lex(fixture.text); + parseIdf(fixture.text, v26, { strict: false }); + } + }); + + expect(allocations).toEqual({ int32: 0, uint8: 0 }); + }); + + it('allocates the layer when, and only when, scanIdf is named', () => { + // The positive control for the assertion above. Same probe, same text, one call difference. + const text = syntaxFixture('surface-bad-ninth-vertex'); + + const withLayer = countLayerAllocations(() => { + scanIdf(text); + }); + const withoutLayer = countLayerAllocations(() => { + parseIdf(text, v26, { strict: false }); + }); + + expect(withLayer.int32).toBeGreaterThan(0); + expect(withLayer.uint8).toBeGreaterThan(0); + expect(withoutLayer).toEqual({ int32: 0, uint8: 0 }); + }); + + it('leaves a parse result carrying no layer', () => { + // The memory half of FR-005 read from the other end: whatever a caller holds onto after a + // parse, none of it is a token store, so nothing keeps the layer alive by reference either. + const text = syntaxFixture('line-endings-lf'); + const result = parseIdf(text, v26, { strict: false }); + + expect(Object.keys(result).sort()).toEqual(['diagnostics', 'document']); + }); +}); diff --git a/packages/core/tests/validate.test.ts b/packages/core/tests/validate.test.ts index e0ab931..f4fc27b 100644 --- a/packages/core/tests/validate.test.ts +++ b/packages/core/tests/validate.test.ts @@ -7,7 +7,7 @@ import { DATA } from '../src/internal.js'; import { Severity, validateDocument, validateObject } from '../src/validate/index.js'; import type { ValidationError } from '../src/validate/index.js'; -import { schema } from './helpers.js'; +import { schema, syntaxFixtures } from './helpers.js'; let v26: Schema; let v94: Schema; @@ -675,3 +675,43 @@ describe('validateDocument', () => { expect(codes(result.warnings)).toEqual(['W002']); }); }); + +/** + * FR-014 and FR-015: validating returns what it returned before the language service existed. + * + * The companion of the same assertion in `parse.test.ts`, and it is here for the same reason: the + * conformance corpus compares findings produced by this exact code path, so a finding that gained a + * field, changed a message, or moved between severities would be a cross-language difference before + * it was anything else. Positioning happens afterwards and elsewhere; this is what keeps that true. + */ +describe('validating is unchanged by positioning', () => { + /** Every fixture read then validated, in name order, as text a snapshot can diff line by line. */ + function validateCorpus(): string { + return syntaxFixtures() + .map(({ name, text }) => { + const { document } = parseIdf(text, v26, { strict: false }); + return `--- ${name}\n${JSON.stringify(validateDocument(document), null, 2)}`; + }) + .join('\n\n'); + } + + it('produces the same findings for every syntax fixture', () => { + expect(validateCorpus()).toMatchSnapshot(); + }); + + it('attaches nothing to a finding, so an existing caller receives exactly what it did', () => { + // `region` and `precision` belong to a `PositionedFinding`, which `@idfkit/language` builds from + // this value rather than inside it. Finding either here would mean a validator had been edited. + const declared = new Set(['severity', 'objType', 'objName', 'field', 'message', 'code']); + + for (const { name, text } of syntaxFixtures()) { + const { document } = parseIdf(text, v26, { strict: false }); + const result = validateDocument(document); + for (const finding of [...result.errors, ...result.warnings, ...result.info]) { + for (const key of Object.keys(finding)) { + expect(declared.has(key), `${name}.idf carries "${key}" on a ValidationError`).toBe(true); + } + } + } + }); +}); diff --git a/packages/idfkit/language.d.ts b/packages/idfkit/language.d.ts new file mode 100644 index 0000000..1e246d6 --- /dev/null +++ b/packages/idfkit/language.d.ts @@ -0,0 +1,12 @@ +// The honest declaration: `idfkit/language` is `@idfkit/language`, whole. +// +// @idfkit/language is an OPTIONAL peer dependency. Installing `idfkit` does not +// install it, so this re-export is unresolvable until it is added: +// +// npm install @idfkit/language +// +// TypeScript only reads this file when something imports `idfkit/language`, so a +// project that never touches the subpath type-checks clean with the peer absent +// (FR-046). A project that does import it and has not installed the peer gets +// TS2307 naming @idfkit/language, on the line below. +export * from '@idfkit/language'; diff --git a/packages/idfkit/language.js b/packages/idfkit/language.js new file mode 100644 index 0000000..b84d202 --- /dev/null +++ b/packages/idfkit/language.js @@ -0,0 +1,114 @@ +/** + * `idfkit/language`, which is `@idfkit/language` behind a named-install guard. + * + * WHY THIS FILE IS NOT `export * from '@idfkit/language'` + * + * @idfkit/language is an optional peer dependency: `npm install idfkit` does not + * install it, which is what keeps the language service off disk for the readers + * who only read and write models (SC-015). `check-install-size.mjs` reports + * 94.5% of the 1.75 MiB budget used with 98.3 KB free, and the service emits + * more than that, so this is arithmetic rather than taste. The cost is that this + * subpath can be imported while the package behind it is absent, and FR-046 + * requires that failure to name the component to install rather than surface as + * a bare unresolved-module error. + * + * A static `export * from '@idfkit/language'` cannot do that. Static re-exports + * are resolved and linked before any module in the graph is evaluated, so there + * is no point at which this file's own code runs first: Node fails the link with + * + * ERR_MODULE_NOT_FOUND: Cannot find package '@idfkit/language' imported from + * .../node_modules/idfkit/language.js + * + * and nothing here is ever reached. `weather.js` next to this file documents the + * same reasoning and the `imports` fallback array that looks like the mechanism + * for it and is not. + * + * WHAT THIS COSTS, AND WHAT IT DELIBERATELY DOES NOT + * + * A dynamic import can be caught, so the guard below is a top-level `await`. + * That makes this module asynchronous in the module graph. It does not make the + * API asynchronous: every name below is an ordinary synchronous binding, and the + * service's whole point is that it is synchronous and free of input and output + * (FR-024). + * + * import { contextAt } from 'idfkit/language'; + * const context = contextAt(text, offset, schema); // no await, ever + * + * The awaited module graph is the whole price. Concretely: `require()` of this + * subpath cannot work, which costs nothing because every package here is ESM + * only and has no CommonJS entry point; and a bundler must support top-level + * await, which Node >= 20, esbuild, Rollup, Vite and webpack >= 5.83 all do. + * + * The second cost is that a dynamic import cannot be spread with `export *`, so + * the re-exported names are written out. That list can drift from the real + * surface of @idfkit/language with nothing noticing, which is why it does not + * drift silently: `npm run check:facade` reads both and fails on any difference. + * + * Types are not affected, and are not listed here. `language.d.ts` next to this + * file is the plain `export * from '@idfkit/language'`, so the declared surface + * is the peer's own, whole, including the many names that exist only as types: + * `CursorContext`, `CompletionResult`, `Offer`, `PositionedFinding` and the rest + * carry no runtime value and so have nothing to re-export here. + * + * `scanIdf` and `classify` are deliberately absent. They live in @idfkit/core + * and reach a reader as `idfkit`, because the syntax layer serves reading and + * writing too. Re-exporting them here would give one function two names. + */ + +/** Every form of "that package is not installed" worth translating. */ +const NOT_FOUND = new Set(['ERR_MODULE_NOT_FOUND', 'MODULE_NOT_FOUND']); + +/** + * The specifier a resolution failure was actually about. + * + * Node's ERR_MODULE_NOT_FOUND carries no structured field for this on 22.12: + * the error's own properties are `stack`, `code` and `message`, and `url` is + * undefined. The specifier is only in the message, in one place, quoted: + * + * Cannot find package '@idfkit/language' imported from .../idfkit/language.js + * Cannot find package '@some/dep' imported from .../@idfkit/language/dist/index.js + * + * Both of those messages CONTAIN the string `@idfkit/language`, the second one + * only because the peer's own file path is in it. So the test has to be the + * quoted position and not a substring search: the second failure means the peer + * is installed and one of its own dependencies is not, and answering that with + * "npm install @idfkit/language" would send a reader to reinstall a package + * they already have while the real fault went unmentioned. + */ +const UNRESOLVED = /Cannot find (?:package|module) '([^']+)'/; + +/** + * The peer, or a failure that says how to get it. + * + * Only a resolution failure naming @idfkit/language itself is translated. + * Anything else, including an error thrown from inside @idfkit/language, is + * re-thrown untouched. The bias is deliberate: a message Node phrases + * differently in some later version falls through to the raw error rather than + * to a confident wrong instruction, and `check-absent-component.mjs` fails on a + * bare ERR_MODULE_NOT_FOUND reaching a reader, so that regression is loud. + */ +let language; +try { + language = await import('@idfkit/language'); +} catch (error) { + const unresolved = UNRESOLVED.exec(String(error?.message ?? ''))?.[1]; + const absent = NOT_FOUND.has(error?.code) && unresolved === '@idfkit/language'; + if (!absent) throw error; + throw new Error( + "idfkit/language requires the optional component '@idfkit/language', which is not installed.\n" + + '\n' + + ' npm install @idfkit/language\n' + + '\n' + + 'It is an optional peer dependency, so installing idfkit deliberately leaves it out: the ' + + 'language service stays off disk for everyone who reads and writes models without an ' + + 'editor in front of them. Everything else in idfkit works without it.', + { cause: error } + ); +} + +export const completionsAt = language.completionsAt; +export const contextAt = language.contextAt; +export const declarationAt = language.declarationAt; +export const explainAt = language.explainAt; +export const findingsIn = language.findingsIn; +export const position = language.position; diff --git a/packages/idfkit/package.json b/packages/idfkit/package.json index c3e85bb..b9b2a61 100644 --- a/packages/idfkit/package.json +++ b/packages/idfkit/package.json @@ -1,7 +1,7 @@ { "name": "idfkit", "version": "0.0.0", - "description": "The shared install name for idfkit in JavaScript: subpath re-exports of @idfkit/core, @idfkit/schemas and the optional @idfkit/weather", + "description": "The shared install name for idfkit in JavaScript: subpath re-exports of @idfkit/core, @idfkit/schemas and the optional @idfkit/weather and @idfkit/language", "type": "module", "license": "MIT", "author": "Samuel Letellier-Duchesne ", @@ -26,6 +26,10 @@ "types": "./index.d.ts", "default": "./index.js" }, + "./language": { + "types": "./language.d.ts", + "default": "./language.js" + }, "./node": { "types": "./node.d.ts", "default": "./node.js" @@ -42,6 +46,8 @@ "files": [ "index.js", "index.d.ts", + "language.js", + "language.d.ts", "node.js", "node.d.ts", "schemas.js", @@ -61,9 +67,13 @@ "@idfkit/schemas": "0.0.0" }, "peerDependencies": { + "@idfkit/language": "0.0.0", "@idfkit/weather": "0.0.0" }, "peerDependenciesMeta": { + "@idfkit/language": { + "optional": true + }, "@idfkit/weather": { "optional": true } diff --git a/packages/language/LICENSE b/packages/language/LICENSE new file mode 100644 index 0000000..0f92ae7 --- /dev/null +++ b/packages/language/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Samuel Letellier-Duchesne + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/language/README.md b/packages/language/README.md new file mode 100644 index 0000000..a140ca5 --- /dev/null +++ b/packages/language/README.md @@ -0,0 +1,47 @@ +# @idfkit/language + +The opt-in language service for IDF text: what completes here, what this means, +what this points at, and where a finding sits in the characters the reader is +looking at. + +Peer-depends on [`@idfkit/core`](../core) and on nothing else. Everything +exported here is synchronous and free of I/O, so it runs unchanged in Node, a +browser, a worker, or behind an editor server. + +**[Documentation](https://js.idfkit.com/)** · +[API reference](https://js.idfkit.com/reference/language/) + +```bash +npm install @idfkit/language +``` + +## Reaching it from the shared name + +A project that installs `idfkit` reaches this package through a subpath: + +```ts +import { completionsAt } from 'idfkit/language'; +``` + +The subpath stays in the export map whether or not this package is installed. +Importing it without this package names the install to run rather than failing +with a bare module-resolution error, which is why the service costs a reader who +never asks for it zero bytes on disk and zero bytes in a bundle. + +## No protocol + +This package imports, depends on, and names nothing from any editor protocol +library. It answers in its own vocabulary and a consumer translates: `envelop` +to its own editor's shape, `idfkit-lsp` to the Language Server Protocol. The +regions it reports never cross a line boundary, because no editor token encoding +can express one that does, so the translation stays a rename rather than +arithmetic. + +## Versioning + +This package joins the repository's release lockstep. `@idfkit/core`, +`@idfkit/schemas`, `@idfkit/weather` and this package are versioned and released +together, and the dependency on `@idfkit/core` is an exact peer range rather than +a caret one. A service paired with a syntax layer it disagrees with would put +findings on the wrong characters, silently; the lockstep is what makes that +install unsupported rather than merely unlikely. diff --git a/packages/language/package.json b/packages/language/package.json new file mode 100644 index 0000000..33c6e16 --- /dev/null +++ b/packages/language/package.json @@ -0,0 +1,48 @@ +{ + "name": "@idfkit/language", + "version": "0.0.0", + "description": "The opt-in IDF language service: cursor answers, positioned findings, synchronous and free of I/O, with no editor protocol", + "type": "module", + "license": "MIT", + "author": "Samuel Letellier-Duchesne ", + "repository": { + "type": "git", + "url": "git+https://github.com/idfkit/idfkit-js.git", + "directory": "packages/language" + }, + "homepage": "https://js.idfkit.com/", + "bugs": { + "url": "https://github.com/idfkit/idfkit-js/issues" + }, + "keywords": [ + "energyplus", + "idf", + "editor", + "completion", + "diagnostics", + "building-energy", + "simulation" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "LICENSE" + ], + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@idfkit/core": "0.0.0" + }, + "scripts": { + "build": "tsc --build" + } +} diff --git a/packages/language/src/complete.ts b/packages/language/src/complete.ts new file mode 100644 index 0000000..326f444 --- /dev/null +++ b/packages/language/src/complete.ts @@ -0,0 +1,336 @@ +import { + describeObjectType, + type IdfDocument, + type ProsePool, + type Region, + type Schema, + type SlimField, + type SlimType, +} from '@idfkit/core'; + +import { contextAt, fieldNameAt, type CursorContext } from './cursor.js'; + +/** + * One thing that may go where the cursor is. + * + * Everything a consumer needs to render the offer and to apply it, so that a list can be shown and + * accepted without a second call and without the consumer measuring anything itself. + */ +export interface Offer { + /** The text to insert. */ + readonly value: string; + /** + * The region this offer would replace. + * + * Carried rather than left to the consumer (FR-048), and this is not optional politeness. An + * editor's own word rules break on both halves of this format: type names contain colons, so + * `BuildingSurface:Detailed` is two words to most of them, and values contain spaces, so + * `Office Zone 1` is three. A consumer left to derive the replaced span would get it wrong on + * the majority of real completions. Working it out needs the format's rules, which live here. + * + * The region is empty where nothing has been written yet, which is an insertion at that point. + */ + readonly replaces: Region; + /** What kind of thing this is. */ + readonly kind: 'objectType' | 'enumValue' | 'referenceTarget'; + /** Whether the schema marks the field required. Undefined for object types. */ + readonly required: boolean | undefined; + /** + * The schema's own prose, when the caller supplied the pool. + * + * The type's memo for a type-name offer and the field's note for a value offer, because those + * are the two things the pool holds. It carries no sentence about an individual permitted value + * and neither does this: deriving one from the value's spelling is the one thing FR-022 forbids + * by name. + */ + readonly prose: string | undefined; +} + +/** + * What completes here, or why nothing does. + * + * A discriminated union rather than a list that is sometimes empty, because "the schema permits + * anything here" and "I could not consult a schema" are different states and an editor that + * rendered them identically would teach the reader that the tool is broken in the first case and + * silently wrong in the second (FR-020, FR-031). A consumer that only wants the happy path matches + * `'ok'` and ignores the rest; a consumer that wants to tell a reader why there is nothing has the + * reason. + * + * `'ok'` with no offers is a legitimate state and a different one from all four below: it says the + * schema constrains this field to names the model has not declared yet. + */ +export type CompletionResult = + | { readonly status: 'ok'; readonly offers: readonly Offer[] } + | { readonly status: 'unconstrained' } + | { readonly status: 'noSchema' } + | { readonly status: 'unknownType'; readonly typeName: string } + | { readonly status: 'notApplicable' }; + +/** What a caller can supply beyond the text, the offset and the schema. */ +export interface CompletionOptions { + /** Supplies candidate names for reference fields. Omit and none are offered. */ + readonly document?: IdfDocument; + /** Supplies prose for the offers. Omit and offers carry none. */ + readonly prose?: ProsePool; +} + +const NOT_APPLICABLE: CompletionResult = { status: 'notApplicable' }; +const NO_SCHEMA: CompletionResult = { status: 'noSchema' }; +const UNCONSTRAINED: CompletionResult = { status: 'unconstrained' }; + +/** + * What may be written where the cursor is. + * + * Bounded local work: the cursor is placed by {@link contextAt}, which scans one statement, and + * the answer then comes from the schema, whose cost is the type rather than the file. The one + * exception is a reference field, whose candidates are the names other objects declare, which is a + * whole-document question by nature; it is answered from the document the caller already holds and + * never by parsing one (research R9). A service that quietly parsed a document when the argument + * was omitted would make one function on the keystroke path eighty milliseconds slower depending + * on an argument nobody passed, which is the worst failure mode available. + * + * The whole schema's type list is offered where a statement begins, unfiltered by what has been + * typed so far. Filtering is the consumer's, and it has what it needs to do it: `replaces` says + * exactly which characters the offer stands in for, which is the span an editor's own word rules + * get wrong on this format. + * + * `schema` is written as possibly absent rather than required, because `'noSchema'` is a state + * FR-031 requires this to report and a signature that forbade the input would make it unreachable + * from typed code. + * + * Nothing throws, for any input. + */ +export function completionsAt( + text: string, + offset: number, + schema: Schema | undefined, + options: CompletionOptions = {} +): CompletionResult { + const context = contextAt(text, offset, schema); + + // Inside a comment nothing completes, with a schema or without one. Reporting `'noSchema'` here + // would send a caller off to load one that would change this answer not at all. + if (context.at === 'comment') return NOT_APPLICABLE; + if (schema === undefined) return NO_SCHEMA; + + // `'typeName'` and `'betweenStatements'` are the two states that carry no field index, and both + // are a statement beginning: half a type name written, or nothing written yet. + if (context.fieldIndex === undefined) { + return { status: 'ok', offers: typeNameOffers(schema, context, options.prose) }; + } + + const typeName = context.typeName; + if (typeName === undefined) { + return { status: 'unknownType', typeName: context.statement.typeNameText }; + } + const type = schema.get(typeName); + if (type === undefined) return { status: 'unknownType', typeName }; + + const facts = fieldFactsAt(schema, typeName, type, context.fieldIndex, options.prose); + // Past the type's last field, with no extensible group to repeat. The schema constrains a field + // it does not define in no way at all; that the field should not be there is a finding, and + // saying so is that finding's job rather than this one's. + if (facts === undefined) return UNCONSTRAINED; + + const written = context.statement.fields[context.fieldIndex]; + // A field index is counted from the separators of this same statement, so it always names a + // written field. An insertion at the statement's end is the harmless answer if that ever stops + // being true; replacing the statement's whole region would not be. + const replaces: Region = written ?? { + start: context.statement.region.end, + end: context.statement.region.end, + }; + + if (facts.values !== undefined && facts.values.length > 0) { + return { + status: 'ok', + offers: facts.values.map((value) => ({ + // Numeric on the handful of fields that express a choice numerically, and the offer is + // text to insert, so it is spelled the way it would be written. + value: String(value), + replaces, + kind: 'enumValue', + required: facts.required, + prose: facts.prose, + })), + }; + } + + if (facts.objectList !== undefined && facts.objectList.length > 0) { + const document = options.document; + // No document, so no candidates can exist. Saying so is the point: an empty `'ok'` list here + // would be indistinguishable from a model that has declared nothing yet. + if (document === undefined) return NOT_APPLICABLE; + return { + status: 'ok', + offers: declaredNames(document, schema, facts.objectList).map((value) => ({ + value, + replaces, + kind: 'referenceTarget', + required: facts.required, + prose: facts.prose, + })), + }; + } + + return UNCONSTRAINED; +} + +/** + * Every object type the schema defines, as offers replacing the type name as written. + * + * The replaced region is the statement's type name, which is empty where nothing has been typed + * yet, so accepting an offer between two statements inserts and accepting one over a half-written + * name replaces the whole of it, colons included. + */ +function typeNameOffers( + schema: Schema, + context: CursorContext, + prose: ProsePool | undefined +): Offer[] { + const replaces = context.statement.typeName; + return schema.typeNames.map((value) => ({ + value, + replaces, + kind: 'objectType', + // A type is not required or optional; only a field is. + required: undefined, + prose: memoOf(schema, value, prose), + })); +} + +/** + * The names objects in this document declare into any of `lists`. + * + * A walk of the document the caller handed in, which is a document cost rather than a text cost: + * nothing is parsed, nothing is scanned, and a document a keystroke behind the text is the correct + * input rather than a stale one, because the statement being typed is incomplete by definition and + * names harvested from it would be garbage (research R9). + * + * A name reaches a list two ways, and both are read here so that no third way to resolve a name is + * invented (FR-029): the object's own name, when its type's `nref` contributes to the list, and an + * ordinary field's value, when that field's `ref` does. The second is how anonymous types such as + * `FluidProperties:Name` carry their identity, and treating those as nameless would offer nothing + * where the model plainly declares something. + */ +function declaredNames(document: IdfDocument, schema: Schema, lists: readonly string[]): string[] { + const wanted = new Set(lists); + const seen = new Set(); + const names: string[] = []; + + const keep = (value: string | undefined): void => { + if (value === undefined || value === '') return; + // Deduplicated case-insensitively, the way EnergyPlus resolves a name, but offered in the + // casing the model wrote it in, which is what a reader expects to see inserted. + const key = value.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + names.push(value); + }; + + for (const object of document.objects()) { + const type = schema.get(object.typeName); + if (type === undefined) continue; + if (contributes(type.nref, wanted)) keep(object.name); + for (const [field, definition] of Object.entries(type.p)) { + if (!contributes(definition.ref, wanted)) continue; + const value = object.get(field); + if (typeof value === 'string') keep(value); + } + } + return names; +} + +/** Whether any list this record contributes to is one the field points into. */ +function contributes( + declared: readonly string[] | undefined, + wanted: ReadonlySet +): boolean { + if (declared === undefined) return false; + return declared.some((list) => wanted.has(list)); +} + +/** The facts an offer needs about the field the cursor is in. */ +interface FieldFacts { + /** Permitted values, when the field is a choice. */ + readonly values: readonly (string | number)[] | undefined; + /** Reference lists this field points into. */ + readonly objectList: readonly string[] | undefined; + /** Whether the schema marks the field required. */ + readonly required: boolean; + /** The field's own note, resolved when a pool was supplied. */ + readonly prose: string | undefined; +} + +/** + * What the schema says about the field at a positional index. + * + * Taken from `describeObjectType`, which is the one place a field's facts are read (FR-029), so + * that a completion offers what the reference page documents rather than a second opinion about + * the same bundle. + * + * With one hole, and it is worth naming rather than hiding. `describeObjectType` drops the type's + * positional first field, because Python's `get_field_names` drops it on the assumption that it is + * always the name. On a named type it is, and the name is free text, so `'unconstrained'` is the + * right answer there anyway. On the 164 anonymous types it is a real field, and 41 of those are + * choice fields: `GlobalGeometryRules.starting_vertex_position` is one an author edits by hand. + * Reporting nothing for those would be silently wrong, so this reads that one field's record + * directly. Closing the hole properly means changing `describeObjectType` in `@idfkit/core`, which + * this package does not own. + */ +function fieldFactsAt( + schema: Schema, + typeName: string, + type: SlimType, + index: number, + prose: ProsePool | undefined +): FieldFacts | undefined { + const name = fieldNameAt(type, index); + if (name === undefined) return undefined; + + const described = describeObjectType(schema, typeName, prose).fields.find( + (field) => field.name === name + ); + if (described !== undefined) { + return { + values: described.enumValues, + objectList: described.objectList, + required: described.required, + prose: described.note, + }; + } + + const definition = type.p[name] ?? type.x?.p[name]; + if (definition === undefined) return undefined; + return { + values: permittedValues(definition), + objectList: definition.ol, + required: (type.r ?? []).includes(name), + prose: definition.n === undefined ? undefined : prose?.[definition.n], + }; +} + +/** + * The values a field accepts, for the one field `describeObjectType` cannot describe. + * + * Reproduces that function's `acceptedValues` exactly, and only exists because it is not reachable + * from here. `se`, the collapsed `anyOf` string branch, carries the sizing sentinels and wins when + * it is present; `e`, the choice list, has had the empty string filtered out by the bundle because + * `e` is what validation checks against, and `eb` records that it was there. + */ +function permittedValues(field: SlimField): (string | number)[] | undefined { + if (field.se !== undefined) return [...field.se]; + if (field.e === undefined) return undefined; + return field.eb === 1 ? ['', ...field.e] : [...field.e]; +} + +/** A type's own prose, resolved against the pool. Nothing is hydrated when no pool was supplied. */ +function memoOf( + schema: Schema, + typeName: string, + prose: ProsePool | undefined +): string | undefined { + if (prose === undefined) return undefined; + const memo = schema.get(typeName)?.m; + return memo === undefined ? undefined : prose[memo]; +} diff --git a/packages/language/src/cursor.ts b/packages/language/src/cursor.ts new file mode 100644 index 0000000..b89a634 --- /dev/null +++ b/packages/language/src/cursor.ts @@ -0,0 +1,334 @@ +import type { Region, Schema, SlimType, Statement } from '@idfkit/core'; + +/** + * Which statement an offset falls in, which field, and which part. + * + * Everything a cursor answer needs, and nothing a whole file would have to be read to know. + * `statement` is the one statement the offset sits in, scanned locally; `at` says which part of it + * the offset is on; `fieldIndex` positions that part; and `typeName` and `fieldName` are what the + * schema calls them, when a schema was supplied and knows. + */ +export interface CursorContext { + /** The statement the offset falls in, scanned locally. */ + readonly statement: Statement; + /** Where in the statement the offset is. */ + readonly at: 'typeName' | 'field' | 'comment' | 'betweenStatements'; + /** + * Positional index of the field, when `at` is `'field'`. + * + * Counted the same way `Statement.fields` is indexed, so it maps onto schema field order + * directly: index 0 is the first field after the type name, which on a named type is the name. + */ + readonly fieldIndex: number | undefined; + /** The canonical type name, when the schema defines it. */ + readonly typeName: string | undefined; + /** The schema field name, when the type is known and the index is in range. */ + readonly fieldName: string | undefined; +} + +/** + * Where the cursor is, found by scanning outwards from it. + * + * **This never builds or consults a `SyntaxLayer`, and it must not start.** Building a layer to + * answer a cursor is the reparse the whole design exists to avoid: it costs the file, on a + * keystroke, for an answer that concerns one statement. The committed measurement asks the same + * question of a large model and of a file a hundredth its size and requires the two to come out + * within a small factor, which is a check a reparse cannot pass on any machine. + * + * The method is a backward scan to the nearest semicolon that is not inside a comment, then a + * forward scan over that one statement. Both halves are sound only because of what this grammar + * lacks: IDF has no nesting, no string literals and no escape sequences, so a semicolon terminates + * a statement unconditionally unless a comment swallowed it, and a comment is an exclamation mark + * running to the end of its line. Deciding whether a candidate semicolon is real therefore costs + * the length of its line, and finding the statement costs the distance back to the previous real + * terminator. Neither number grows with the file. Almost no other grammar would permit this, and + * a language that gained a string literal would break it silently, so the property is stated here + * rather than left for a reader to rediscover. + * + * The honest bound is the distance back to the previous real terminator rather than the length of + * the statement. Those are the same number in ordinary files and differ in one real case: a cursor + * in the first statement of a file that opens with a large comment header scans back through the + * header. Headers are small, and the case is still linear with a tiny constant. + * + * Nothing throws, for any input. An offset outside `[0, text.length]` is clamped into range rather + * than refused (FR-032): a cursor arrives from an editor that may be a keystroke ahead of the text + * it was measured against, and answering about the nearest character beats answering nothing. + * + * The schema is optional. Without one the context still reports the statement, the part, and the + * field index, which is what positions a finding about a type no schema defines. + */ +export function contextAt(text: string, offset: number, schema?: Schema): CursorContext { + const at = clampOffset(offset, text.length); + const scanned = scanStatement(text, statementScanStart(text, at), at); + const statement = scanned.statement ?? emptyStatementAt(at); + + const part = partOf(scanned); + const fieldIndex = part === 'field' ? scanned.slot - 1 : undefined; + + const typeName = schema?.resolve(statement.typeNameText); + const type = typeName === undefined ? undefined : schema?.get(typeName); + const fieldName = + type === undefined || fieldIndex === undefined ? undefined : fieldNameAt(type, fieldIndex); + + return { statement, at: part, fieldIndex, typeName, fieldName }; +} + +/** + * The schema field name at a positional index, or `undefined` past the end of the type. + * + * Positional order is the schema's own: the fixed fields in `f`, whose index 0 is the name on a + * named type, and then the extensible group repeating its `fields` for as long as values were + * written. This mirrors how `parseIdf` maps positional values onto named fields, deliberately and + * exactly, because a cursor that disagreed with the reader about which field it was in would offer + * the neighbouring field's values. + * + * @internal + */ +export function fieldNameAt(type: SlimType, index: number): string | undefined { + if (index < 0) return undefined; + const fixed = type.f; + if (index < fixed.length) return fixed[index]; + const extensible = type.x; + const width = extensible?.fields.length ?? 0; + if (extensible === undefined || width === 0) return undefined; + return extensible.fields[(index - fixed.length) % width]; +} + +/** Which part of the statement the scan put the offset on. */ +function partOf(scanned: Scanned): CursorContext['at'] { + // A comment wins over everything, including the field it interrupts: nothing completes and + // nothing explains inside one, whether it sits between two statements or in the middle of a + // field's value. + if (scanned.inComment) return 'comment'; + // No statement had opened by the time the scan passed the offset, so the offset is in the + // whitespace after a terminator and before whatever comes next, which includes trailing + // whitespace at end of file. That is the state in which a new statement is beginning. + if (scanned.statement === undefined) return 'betweenStatements'; + return scanned.slot === 0 ? 'typeName' : 'field'; +} + +/** + * The statement a cursor is beginning to write, standing in for one that is not there yet. + * + * `CursorContext.statement` is not optional, and making it optional to describe an empty line + * would push a branch onto every consumer to serve the one state in which there is nothing to + * describe. An empty statement at the cursor is the truthful alternative: it selects nothing, + * which is exactly the region an offer inserted here would replace, and it reports + * `unterminated`, because a statement nobody has written carries no terminator. + */ +function emptyStatementAt(offset: number): Statement { + const region: Region = { start: offset, end: offset }; + return { region, typeName: region, typeNameText: '', fields: [], unterminated: true }; +} + +/** + * Offset to begin the forward scan at: one past the nearest real terminator before `offset`. + * + * Backwards through the semicolons with `lastIndexOf`, which is a native scan rather than a + * character loop, testing each candidate for comment membership and skipping the ones a comment + * swallowed. A file with no terminator before the offset starts at zero, which is the only other + * place a statement can begin. + */ +function statementScanStart(text: string, offset: number): number { + for (let at = offset - 1; at >= 0; at -= 1) { + const found = text.lastIndexOf(';', at); + if (found < 0) return 0; + if (!insideComment(text, found)) return found + 1; + // Step past the one a comment swallowed and keep looking. `at -= 1` runs next, so a candidate + // at offset zero leaves the loop rather than finding itself again. + at = found; + } + return 0; +} + +/** + * Whether the character at `index` sits inside a comment. + * + * Back to the start of its line, then forward for an exclamation mark before it. There are no + * string literals and no escapes, so an exclamation mark on the line before this character always + * opened a comment that is still open here, and one line is all it costs to know. + * + * The forward search is what keeps that promise. `lastIndexOf('!', index)` answers the same + * question and reads the whole prefix to do it whenever the file holds no exclamation mark before + * `index`, which is every position in a machine-written file that carries no comments at all: the + * cost then grows with how far into the file the cursor is, which is precisely what this module + * exists not to do. Searching forward from the line's own start stops at the line's end instead. + */ +function insideComment(text: string, index: number): boolean { + const lineStart = index === 0 ? 0 : text.lastIndexOf('\n', index - 1) + 1; + const opened = text.indexOf('!', lineStart); + return opened !== -1 && opened <= index; +} + +/** What one forward pass over a single statement found. */ +interface Scanned { + /** The statement, or `undefined` when none had opened by the time the scan passed the offset. */ + readonly statement: Statement | undefined; + /** Whether the offset falls inside a comment, the exclamation mark itself included. */ + readonly inComment: boolean; + /** Separators seen strictly before the offset: 0 is the type name, 1 the first field after it. */ + readonly slot: number; +} + +const EXCLAMATION = 0x21; +const COMMA = 0x2c; +const SEMICOLON = 0x3b; + +/** + * Read one statement forward from `from`, stopping as soon as the offset is placed. + * + * A second implementation of the character rules `@idfkit/core`'s internal scan already holds, and + * that is a deliberate cost rather than an oversight. The scan is internal to core on purpose, so + * that the syntax layer and the model-building read cannot drift apart; reaching it from here + * would mean publishing it, which is the hazard that decision exists to prevent. The rules + * reproduced here are the four that matter, and the field bounds are computed exactly as the layer + * computes them, so that a region reported by a cursor and a region reported by a finding select + * the same characters. + * + * The pass stops at the statement's terminator, or at end of input, or as soon as it is past the + * offset with nothing open, which is what keeps the cost the statement's rather than the file's. + */ +function scanStatement(text: string, from: number, offset: number): Scanned { + const length = text.length; + + /** 0 for the type name, 1 for the first field after it. */ + let fieldIndex = 0; + /** True once the statement's first non-blank, non-comment character has been seen. */ + let open = false; + /** Where that character was. */ + let openedAt = 0; + /** First non-blank character of the current field's value, or -1 while it has none. */ + let valueStart = -1; + /** One past the last non-blank character of the run `valueStart` falls in. */ + let valueEnd = -1; + /** True once a comment has ended the run holding `valueStart`. */ + let valueClosed = false; + + let typeName: Region = { start: from, end: from }; + const fields: Region[] = []; + let inComment = false; + let slot = 0; + let statement: Statement | undefined; + + /** + * Record the field the scan is inside, closed at `at`. + * + * A field with no value text gets an empty region at `at`, the separator or terminator or end of + * input that closed it, because everything before it was whitespace or comment. That is what + * keeps positional indexing sound through a blank slot in the middle of an extensible group. + * + * A field written on both sides of a comment that interrupts it keeps the first run alone, which + * is what the layer stores, so a value region and a comment region can never overlap. + */ + const closeField = (at: number): void => { + const region: Region = + valueStart < 0 ? { start: at, end: at } : { start: valueStart, end: valueEnd }; + if (fieldIndex === 0) typeName = region; + else fields.push(region); + valueStart = -1; + valueEnd = -1; + valueClosed = false; + }; + + const finish = (end: number, unterminated: boolean): Statement => ({ + region: { start: openedAt, end }, + typeName, + typeNameText: text.slice(typeName.start, typeName.end), + fields, + unterminated, + }); + + let index = from; + while (index < length) { + // Nothing has opened and the offset is behind us, so no statement contains it and none that + // opens later can. Every comment that could hold the offset began at or before it and has + // already been tested. + if (!open && index > offset) break; + + const code = text.charCodeAt(index); + + if (code === EXCLAMATION) { + const feed = text.indexOf('\n', index); + const end = feed === -1 ? length : feed; + // Inclusive at both ends. A cursor on the exclamation mark is in the comment it opens, and a + // cursor one past its last character is at the end of the comment's text rather than in the + // whitespace of the next line: typing there extends the comment. + if (offset >= index && offset <= end) inComment = true; + // The comment ends the run holding the value but not the field, so `Zone1, !- Name` leaves + // the value at `Zone1` and text after the line feed still belongs to the same field. + if (valueStart >= 0) valueClosed = true; + if (feed === -1) break; + index = feed + 1; + continue; + } + + const blank = isSpace(code); + if (!open && !blank) { + open = true; + openedAt = index; + } + + if (code === COMMA || code === SEMICOLON) { + closeField(index); + if (code === SEMICOLON) { + statement = finish(index + 1, false); + break; + } + if (index < offset) slot += 1; + fieldIndex += 1; + index += 1; + continue; + } + + if (!blank) { + if (valueStart < 0) { + valueStart = index; + valueEnd = index + 1; + } else if (!valueClosed) { + valueEnd = index + 1; + } + } + + index += 1; + } + + // Input ran out inside a statement. Its last field is closed like any other, so the cursor reads + // what was written rather than reconstructing it from the leftovers, and the statement says it + // runs to the end. + if (statement === undefined && open) { + closeField(length); + statement = finish(length, true); + } + + return { statement, inComment, slot }; +} + +/** + * Whether a character is whitespace, by the definition `String.prototype.trim` uses. + * + * The same set the core scan tests, character for character. A field's value is trimmed, so a + * disagreement about one character would put a region beside a value rather than on it, and the + * cursor and the syntax layer would then report different bounds for the same field. + */ +function isSpace(code: number): boolean { + if (code < 0x80) return code === 0x20 || (code >= 0x09 && code <= 0x0d); + return ( + code === 0xa0 || + code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || + code === 0x2028 || + code === 0x2029 || + code === 0x202f || + code === 0x205f || + code === 0x3000 || + code === 0xfeff + ); +} + +/** Into `[0, max]`, whole. `NaN` lands at 0, since no position is nearer than another. */ +function clampOffset(value: number, max: number): number { + if (Number.isNaN(value)) return 0; + const whole = Math.trunc(value); + if (whole < 0) return 0; + return whole > max ? max : whole; +} diff --git a/packages/language/src/declaration.ts b/packages/language/src/declaration.ts new file mode 100644 index 0000000..1f3b4ca --- /dev/null +++ b/packages/language/src/declaration.ts @@ -0,0 +1,257 @@ +import { + describeObjectType, + type IdfDocument, + type Region, + type Schema, + type SlimType, +} from '@idfkit/core'; + +import { contextAt } from './cursor.js'; + +/** Where one object declares the name the cursor is on. */ +export interface Declaration { + /** + * The region where the name is declared, meaning the name field of the declaring statement. + * + * A region in the text this was asked about, not in the document: the document holds no + * positions, and it is the text the reader is looking at. It identifies a thing rather than + * something to draw, so it stays whole and may cross a line boundary (FR-050). + */ + readonly region: Region; + /** The declaring statement's canonical type. */ + readonly typeName: string; +} + +/** + * What the value under the cursor names, or why it names nothing. + * + * The same discriminated shape as `CompletionResult`, for the same reason: "this value names + * nothing that exists" and "I could not consult a schema" are different states, and a consumer + * that wants to tell a reader why nothing happened has the reason (FR-031). + * + * `'ok'` with no declarations is a legitimate state and a different one from all three below: it + * says the field points into a reference list and nothing in the document declares this name. + * Where the name is declared nowhere, none is reported and none is guessed at; the + * dangling-reference finding is what tells the reader why. + */ +export type DeclarationResult = + | { readonly status: 'ok'; readonly declarations: readonly Declaration[] } + | { readonly status: 'noSchema' } + | { readonly status: 'unknownType'; readonly typeName: string } + | { readonly status: 'notApplicable' }; + +const NOT_APPLICABLE: DeclarationResult = { status: 'notApplicable' }; +const NO_SCHEMA: DeclarationResult = { status: 'noSchema' }; +const NONE: DeclarationResult = { status: 'ok', declarations: [] }; + +/** + * Where the name under the cursor is declared. + * + * Two steps, and keeping them apart is what keeps this honest. Which objects declare the name is + * resolved against the caller's document, by the two rules the model itself uses and no third one + * (FR-029): the object's own name, when its type's `nref` contributes to a list this field points + * into, and an ordinary field's value, when that field's `ref` does. Only then is the declaring + * statement located in the text, by looking for the name the document holds and confirming with a + * cursor context that the match really is that type's declaring field. So the text is never + * searched for a string that happens to match: it is searched for a name the document has already + * said is declared, and a match that turns out to be a value somewhere else is discarded. + * + * The document is the caller's, exactly as it is for reference completion, and for the same reason + * (research R9): declarations are a whole-document fact, every consumer already holds a document, + * and parsing one here would put the eighty milliseconds this design exists to avoid back on the + * keystroke path. A document a keystroke behind the text is the correct input rather than a stale + * one. + * + * **Honest bound.** Locating a declaration reads the text rather than one statement of it, because + * "every declaration" is a question about the whole file and cannot be answered from a bounded + * local scan. The reading is `String.prototype.indexOf`, a native scan of a few hundred kilobytes + * costing a small fraction of a millisecond, plus one bounded cursor context per candidate match, + * and no layer is built and no document is parsed. It is stated here rather than left for a reader + * to discover in a profile. + * + * `schema` is written as possibly absent rather than required, because `'noSchema'` is a state + * FR-031 requires this to report and a signature that forbade the input would make it unreachable + * from typed code. + * + * Nothing throws, for any input. + */ +export function declarationAt( + text: string, + offset: number, + schema: Schema | undefined, + document: IdfDocument +): DeclarationResult { + const context = contextAt(text, offset, schema); + + // A type name declares nothing and names nothing, a comment holds no values, and between two + // statements there is no value to follow. Reporting `'noSchema'` in a comment would send a + // caller off to load one that would change this answer not at all. + if (context.at !== 'field') return NOT_APPLICABLE; + if (schema === undefined) return NO_SCHEMA; + + const typeName = context.typeName; + if (typeName === undefined) { + return { status: 'unknownType', typeName: context.statement.typeNameText }; + } + const type = schema.get(typeName); + // Unreachable through `resolve`, which only returns a name the schema holds, and kept because + // `describeObjectType` throws on a type the schema lacks and nothing here may throw (FR-030). + if (type === undefined) return { status: 'unknownType', typeName }; + + const index = context.fieldIndex; + const fieldName = context.fieldName; + // No field index on a `'field'` context is unreachable, and a field the type does not define is + // ordinary: a statement may carry more fields than its type has. + if (index === undefined || fieldName === undefined) return NOT_APPLICABLE; + + const lists = listsPointedInto(schema, typeName, type, fieldName); + // The field points at nothing. Searching the document for an object that happens to be called + // whatever is written here would find one often enough to be believed and would be a guess. + if (lists.length === 0) return NOT_APPLICABLE; + + const written = context.statement.fields[index]; + const target = written === undefined ? '' : text.slice(written.start, written.end); + // A field left empty names nothing, which is the same answer as a name nothing declares. + if (target === '') return NONE; + + const declarations: Declaration[] = []; + const seen = new Set(); + for (const declared of declaringObjects(document, schema, new Set(lists), fold(target))) { + const region = locate(text, schema, declared); + if (region === undefined || seen.has(region.start)) continue; + seen.add(region.start); + declarations.push({ region, typeName: declared.typeName }); + } + return { status: 'ok', declarations }; +} + +/** + * The reference lists a field points into, taken from `describeObjectType` (FR-029). + * + * With the one hole `completionsAt` documents at length: `describeObjectType` drops the type's + * positional first field, which on an anonymous type is a real field, so that one is read from its + * own record. Reading `ol` is the same key the description reports and not a second opinion about + * it. + */ +function listsPointedInto( + schema: Schema, + typeName: string, + type: SlimType, + fieldName: string +): readonly string[] { + const described = describeObjectType(schema, typeName).fields.find( + (field) => field.name === fieldName + ); + if (described !== undefined) return described.objectList ?? []; + return type.p[fieldName]?.ol ?? []; +} + +/** One object that declares the name, and where in its statement the declaration is written. */ +interface DeclaringObject { + /** The declaring object's canonical type name. */ + readonly typeName: string; + /** Positional index of the field that declares the name, counted as `CursorContext` counts. */ + readonly fieldIndex: number; + /** The name as the document holds it, which is the casing the text wrote it in. */ + readonly text: string; +} + +/** + * The objects declaring `folded` into one of `wanted`. + * + * The two ways a name reaches a reference list, and no third one (FR-029). Both are read, because + * the second is how anonymous types such as `FluidProperties:Name` carry their identity, and + * treating those as nameless would report nothing where the model plainly declares something. + * + * Membership of the lists the field points into is what makes a match a declaration rather than a + * coincidence. A `Zone` and a `Construction` may both be called `Office`, and following a + * construction name to the zone would be the guess this function exists to avoid. It is why a name + * declared only into some other list reports none here while the validator's dangling check, which + * asks the coarser question of whether any object declares the name at all, stays quiet: this + * reports what the field can name, not what the file happens to contain. + * + * Names are compared case-insensitively, the way EnergyPlus resolves them and the way every other + * name comparison in this library does. + */ +function declaringObjects( + document: IdfDocument, + schema: Schema, + wanted: ReadonlySet, + folded: string +): DeclaringObject[] { + const out: DeclaringObject[] = []; + + for (const object of document.objects()) { + const canonical = schema.resolve(object.typeName) ?? object.typeName; + const type = schema.get(canonical); + if (type === undefined) continue; + + const nameIndex = type.f.indexOf('name'); + if ( + nameIndex >= 0 && + object.name !== '' && + fold(object.name) === folded && + contributes(type.nref, wanted) + ) { + out.push({ typeName: canonical, fieldIndex: nameIndex, text: object.name }); + } + + for (const [field, definition] of Object.entries(type.p)) { + if (!contributes(definition.ref, wanted)) continue; + const value = object.get(field); + if (typeof value !== 'string' || value === '' || fold(value) !== folded) continue; + const fieldIndex = type.f.indexOf(field); + if (fieldIndex < 0) continue; + out.push({ typeName: canonical, fieldIndex, text: value }); + } + } + return out; +} + +/** Whether any list this record contributes to is one the field points into. */ +function contributes( + declared: readonly string[] | undefined, + wanted: ReadonlySet +): boolean { + if (declared === undefined) return false; + return declared.some((list) => wanted.has(list)); +} + +/** + * The region in the text where this object writes the name it declares. + * + * Every occurrence of the name is a candidate and a cursor context decides each one: the match + * must be a field, of the declaring type, at the declaring field's index, and the field's whole + * text must be the name rather than a value containing it, so `Office` never resolves to + * `Office Zone 2`. The first survivor is the answer, because a document cannot hold two objects of + * one type sharing a name: `IdfDocument.addRaw` throws on the second and `parseIdf` records a + * `ParseError` and skips it, which is the same fact the finding correlation rests on (research + * R6). + * + * The comparison is exact rather than folded because the string being looked for came out of the + * document, which stores a value as the text wrote it, trimmed and with comments stripped. The one + * case that misses is a name interrupted by a comment between two of its words, which the reader + * joins and the syntax layer does not; that declaration is then reported as unlocatable rather + * than as a region on the wrong characters. + */ +function locate(text: string, schema: Schema, declared: DeclaringObject): Region | undefined { + for ( + let from = text.indexOf(declared.text); + from >= 0; + from = text.indexOf(declared.text, from + 1) + ) { + const context = contextAt(text, from, schema); + if (context.at !== 'field' || context.fieldIndex !== declared.fieldIndex) continue; + if (context.typeName !== declared.typeName) continue; + const region = context.statement.fields[declared.fieldIndex]; + if (region === undefined) continue; + if (text.slice(region.start, region.end) !== declared.text) continue; + return region; + } + return undefined; +} + +/** Case-insensitive name comparison, as EnergyPlus resolves names. */ +function fold(value: string): string { + return value.toLowerCase(); +} diff --git a/packages/language/src/explain.ts b/packages/language/src/explain.ts new file mode 100644 index 0000000..60bf07b --- /dev/null +++ b/packages/language/src/explain.ts @@ -0,0 +1,219 @@ +import { + describeObjectType, + docsUrlForObject, + type DocsUrl, + type FieldDescription, + type ProsePool, + type Region, + type Schema, +} from '@idfkit/core'; + +import { contextAt, type CursorContext } from './cursor.js'; + +/** + * What the schema says about whatever the offset is on. + * + * Every member is reported rather than composed. The facts come from + * `describeObjectType`, the prose from the pool the caller supplied, and the manual location from + * `docsUrlForObject`; nothing here is derived from a field's name, which is the one thing FR-022 + * forbids by name. + */ +export interface Explanation { + /** + * The region this explanation describes (FR-049). + * + * What a consumer highlights while the explanation is shown, so that the reader can see which + * characters the words are about. Carried rather than left to the consumer for the reason offers + * carry theirs: an editor's own word rules split `BuildingSurface:Detailed` at the colon and + * `Office Zone 1` at the spaces, and would highlight a fragment of each. + * + * It identifies a thing rather than something to draw, so it stays whole and may cross a line + * boundary (FR-050): a field written across two lines is ordinary in this format. + */ + readonly region: Region; + /** What the offset is on. */ + readonly of: 'objectType' | 'field'; + /** The canonical type name. */ + readonly typeName: string; + /** The schema field name, when `of` is `'field'`. */ + readonly fieldName: string | undefined; + /** + * The schema's own prose, when the caller supplied the pool. + * + * The type's memo for a type name and the field's note for a field. `undefined` where no pool + * was supplied and where the pool carries none for this record: absence is reported as absence, + * never filled with a sentence made up from the name. + */ + readonly prose: string | undefined; + /** + * The field's facts, exactly as `describeObjectType` produces them. Undefined for a type name. + * + * Also undefined for one field this package cannot describe, and it is worth naming rather than + * hiding. `describeObjectType` drops the type's positional first field, because Python's + * `get_field_names` drops it on the assumption that it is always the name. On the anonymous + * types it is a real field, and `GlobalGeometryRules.starting_vertex_position` is one an author + * edits by hand. Rebuilding a `FieldDescription` here would be a second copy of the schema's + * facts, which FR-029 forbids and which would drift from the reference page the reader is + * comparing against, so the structural facts are reported as absent and the field's own prose is + * still resolved. Closing the hole properly means changing `describeObjectType` in + * `@idfkit/core`, which this package does not own. + */ + readonly field: FieldDescription | undefined; + /** + * Where the manual documents this, from `docsUrlForObject`. + * + * The type's page in both cases, because that is where the manual documents a field: it has no + * page of its own. `undefined` for a version the documentation site does not carry. + */ + readonly docs: DocsUrl | undefined; +} + +/** + * What the schema says here, or why it says nothing. + * + * The same discriminated shape as `CompletionResult`, for the same reason: "there is nothing here + * to explain" and "I could not consult a schema" are different states, and an editor that rendered + * them identically would teach the reader that the tool is broken in the first case and silently + * wrong in the second (FR-031). + */ +export type ExplanationResult = + | { readonly status: 'ok'; readonly explanation: Explanation } + | { readonly status: 'noSchema' } + | { readonly status: 'unknownType'; readonly typeName: string } + | { readonly status: 'notApplicable' }; + +const NOT_APPLICABLE: ExplanationResult = { status: 'notApplicable' }; +const NO_SCHEMA: ExplanationResult = { status: 'noSchema' }; + +/** + * What the thing under the cursor means, in the schema's own words. + * + * Bounded local work, like every other cursor answer: the cursor is placed by {@link contextAt}, + * which scans one statement, and the answer then comes from the type, whose cost is the type + * rather than the file. + * + * The prose pool stays the caller's to load (FR-028). It is a parameter rather than something + * reached for, and this module imports it as a type alone, so a caller who never asks for prose + * never pays for it: passing nothing yields the structural facts with `prose: undefined`, which is + * a truthful absence rather than a sentence derived from the field's name. + * + * `schema` is written as possibly absent rather than required, because `'noSchema'` is a state + * FR-031 requires this to report and a signature that forbade the input would make it unreachable + * from typed code. + * + * Nothing throws, for any input. + */ +export function explainAt( + text: string, + offset: number, + schema: Schema | undefined, + prose?: ProsePool +): ExplanationResult { + const context = contextAt(text, offset, schema); + + // Inside a comment, and in the whitespace between two statements, there is nothing the schema + // has anything to say about. Reporting `'noSchema'` here would send a caller off to load one + // that would change this answer not at all. + if (context.at === 'comment' || context.at === 'betweenStatements') return NOT_APPLICABLE; + if (schema === undefined) return NO_SCHEMA; + + // The region first, and the schema afterwards. An offset on a separator, on a terminator, or in + // the whitespace beside a value falls outside every region this statement holds, and answering + // it with the nearest field is the behaviour that makes a hover feel haunted (FR-049). The test + // is containment rather than a character test, because a region already ends where the value + // ends: it begins at the value's first non-blank character and stops after its last, so + // everything between two values, the comma included, is outside both. + const at = clampOffset(offset, text.length); + const region = describedRegion(context, at); + if (region === undefined) return NOT_APPLICABLE; + + const typeName = context.typeName; + if (typeName === undefined) { + return { status: 'unknownType', typeName: context.statement.typeNameText }; + } + const type = schema.get(typeName); + // Unreachable through `resolve`, which only returns a name the schema holds, and kept because + // `describeObjectType` throws on a type the schema lacks and nothing here may throw (FR-030). + if (type === undefined) return { status: 'unknownType', typeName }; + + const described = describeObjectType(schema, typeName, prose); + const docs = docsUrlForObject(typeName, schema.version, schema); + + if (context.at === 'typeName') { + return { + status: 'ok', + explanation: { + region, + of: 'objectType', + typeName, + fieldName: undefined, + prose: described.memo, + field: undefined, + docs, + }, + }; + } + + const fieldName = context.fieldName; + // Past the type's last field, with no extensible group to repeat. The schema defines no field + // here and so says nothing about one; that the field should not be there is a finding, and + // saying so is that finding's job rather than this one's. + if (fieldName === undefined) return NOT_APPLICABLE; + + const field = described.fields.find((candidate) => candidate.name === fieldName); + return { + status: 'ok', + explanation: { + region, + of: 'field', + typeName, + fieldName, + // `field.note` is the same lookup, and taking it from the description keeps one source for + // it. The fallback is the pool lookup for the one field the description cannot carry, which + // resolves an index the schema already holds rather than reproducing anything. + prose: field === undefined ? proseFor(type.p[fieldName]?.n, prose) : field.note, + field, + docs, + }, + }; +} + +/** + * The region the explanation is about, or `undefined` when the offset is not on one. + * + * `at` is `'typeName'` or `'field'` by the time this is called, so the region is the statement's + * type name or the written field at the cursor's index. A field written empty has an empty region + * positioned between its separators, which contains no offset at all, and that is the right answer + * for it: there is no text there to explain. + */ +function describedRegion(context: CursorContext, offset: number): Region | undefined { + const region = + context.at === 'typeName' + ? context.statement.typeName + : // `fieldIndex` is defined for every `'field'` context; the guard is what the compiler + // needs, and an offset on no field is not on a region either way. + context.statement.fields[context.fieldIndex ?? -1]; + if (region === undefined) return undefined; + return offset >= region.start && offset < region.end ? region : undefined; +} + +/** Resolve a prose index against the pool. Nothing is hydrated when no pool was supplied. */ +function proseFor(index: number | undefined, prose: ProsePool | undefined): string | undefined { + if (index === undefined || prose === undefined) return undefined; + return prose[index]; +} + +/** + * Into `[0, max]`, whole. `NaN` lands at 0, since no position is nearer than another. + * + * The clamp `contextAt` applies (FR-032), repeated because it is private to the cursor module. A + * containment test measured against an unclamped offset would disagree with the context it is + * testing, and would report nothing for an offset a keystroke past the end of the text where the + * cursor answer describes the last character. + */ +function clampOffset(value: number, max: number): number { + if (Number.isNaN(value)) return 0; + const whole = Math.trunc(value); + if (whole < 0) return 0; + return whole > max ? max : whole; +} diff --git a/packages/language/src/findings.ts b/packages/language/src/findings.ts new file mode 100644 index 0000000..f02b7b6 --- /dev/null +++ b/packages/language/src/findings.ts @@ -0,0 +1,436 @@ +import { offsetAt, parseIdf, scanIdf, validateDocument } from '@idfkit/core'; +import type { + ParseDiagnostic, + Region, + Schema, + SlimType, + Statement, + SyntaxLayer, + ValidationError, +} from '@idfkit/core'; + +/** + * An existing finding with the region it concerns attached. Not a new finding. + * + * `F` stays whatever it was: a `ParseDiagnostic` or a `ValidationError`, unchanged, with two + * properties added beside it. The generic is what lets both travel through one path without either + * type being touched, which is the whole point of correlating here rather than at the source. FR-014 + * forbids filtering or rewording a finding and FR-015 forbids changing what an existing caller + * receives; both hold structurally, because nothing on the read path knows this module exists. + */ +export type PositionedFinding = F & { + /** The region the finding concerns. */ + readonly region: Region; + /** Whether the region selects the field or falls back to the statement. */ + readonly precision: 'field' | 'statement'; +}; + +/** + * Read text, validate what it produced, and position every finding from both. + * + * One parse, one validation, one scan, in that order, and then the correlation below. A consumer + * that already holds findings from its own run should call {@link position} instead and keep its + * parse; this exists for the consumer that has only text. + * + * Reading findings come first and validation findings after, which is the order in which the two + * runs produced them. Neither list is filtered: a value that is both unreadable and invalid is two + * findings here because it was two findings before. + * + * @example + * ```ts + * for (const finding of findingsIn(text, schema)) { + * const { line, column } = lineColumnAt({ text }, finding.region.start); + * console.log(`${line}:${column}`, finding.message); + * } + * ``` + */ +export function findingsIn( + text: string, + schema: Schema +): readonly PositionedFinding[] { + // Never strict: a strict read throws on the first finding, and a caller asking for every finding + // in a file is asking for the file to be described rather than rejected. + const { document, diagnostics } = parseIdf(text, schema, { strict: false }); + const validation = validateDocument(document); + const findings: (ParseDiagnostic | ValidationError)[] = [ + ...diagnostics, + ...validation.errors, + ...validation.warnings, + ...validation.info, + ]; + return position(findings, scanIdf(text), schema); +} + +/** + * Attach a region to each of a set of findings already produced. + * + * Exposed separately from {@link findingsIn} so a consumer holding findings from its own parse and + * its own validation pays for one scan rather than for a second read of the same text. + * + * Two families of finding arrive here and they are positioned by different routes, because they + * know different things. A reading finding was made by the scanner and already names a line, and + * often a column, so it is placed directly at what the scanner saw. A validation finding was made + * from a document and names an object by type and name, never a place, so it is correlated against + * the statement index below. + * + * A finding of neither shape, or one naming something this text does not contain, still comes back + * with a region: an empty one at the start of the text, saying as little as is actually known. SC-004 + * asks for a region on every finding and zero omitted, and dropping the ones that were hard would + * satisfy the count while defeating the point. + * + * Findings are returned in the order they were given, so a caller can zip the result against its + * own list. + */ +export function position( + findings: readonly F[], + layer: SyntaxLayer, + schema: Schema +): readonly PositionedFinding[] { + // Built per call rather than cached against the layer: it costs one pass over the statements and + // a couple of string operations each, against a parse and a validation run that have already + // happened. A cache here would be state, and this package holds none. + const index = indexStatements(layer, schema); + return findings.map((finding) => { + const placed = isValidationShaped(finding) + ? placeValidation(finding, occurrenceOf(finding), schema, index) + : isParseShaped(finding) + ? placeParse(finding, layer) + : undefined; + return { + ...finding, + region: placed?.region ?? NOWHERE, + precision: placed?.precision ?? 'statement', + }; + }); +} + +/** A region and how precisely it answers, before it is attached to a finding. */ +interface Placement { + readonly region: Region; + readonly precision: 'field' | 'statement'; +} + +/** + * Where a finding lands when nothing in the text answers to it. + * + * Empty and at the start, which is what an editor renders as a file-level diagnostic. A region + * covering the whole text would be a claim about where the problem is, and there is no such claim + * to make. + */ +const NOWHERE: Region = { start: 0, end: 0 }; + +/** + * What a finding may carry to say WHICH occurrence it concerns, when its own type cannot. + * + * Neither property exists on `ValidationError` today and nothing in `@idfkit/core` produces one, so + * neither is required and neither is exported: a caller that knows more than the finding does + * passes `ValidationError & { readonly index: number }` and this reads it. They are declared rather + * than read out of thin air because both are answers to real gaps, and both are spelled the way the + * library already spells them. + * + * `index` is `ReferenceEdge.index`, the repeat within an extensible group, and it is the only way + * to tell the ninth vertex of a surface from the first: a `ValidationError` naming + * `vertex_x_coordinate` names the field and not the repeat. `ordinal` is the position of an + * anonymous object among the statements of its type, which `objName` cannot carry because it is + * documented as empty for exactly those objects. + */ +interface Occurrence { + /** Position of the object among the statements of its type, counting from zero. */ + readonly ordinal?: number; + /** Repeat within the extensible group, counting from zero, when the field lives in one. */ + readonly index?: number; +} + +/** + * Statements indexed twice, both in source order, which is how a finding reaches a place. + * + * The name key is unambiguous, and that is not obvious: duplicate names are common in real files, + * so keying on one looks unsafe. It is safe because `IdfDocument.addRaw` throws when a second + * object of a type carries an existing name, `parseIdf` catches that, records a `ParseError`, and + * skips the object. A document parsed from text therefore never holds two objects sharing a folded + * type and a folded name, so no validation finding can be about the second one. The duplicate + * reaches the reader as a reading finding instead, positioned by the scanner that saw it, and never + * arrives here at all. `packages/core/tests/document.test.ts` asserts that rather than trusting it. + * + * The ordinal key is sound because `parseIdf` adds in source order and `IdfCollection` preserves + * insertion order and documents that it does, so the Nth object of a type in the document is the + * Nth statement of that type in the text. That is asserted in the same file, for the same reason. + */ +interface StatementIndex { + /** Folded type name and folded object name, for named objects. */ + readonly byName: ReadonlyMap; + /** Folded type name to the statements of that type, in source order, for anonymous ones. */ + readonly byOrdinal: ReadonlyMap; +} + +function indexStatements(layer: SyntaxLayer, schema: Schema): StatementIndex { + const byName = new Map(); + const byOrdinal = new Map(); + // The schema hydrates a definition on every `get`, and a file holds thousands of statements of a + // few hundred types, so the answer to "is this type named?" is worth keeping. + const named = new Map(); + + for (const statement of layer.statements) { + const type = fold(statement.typeNameText); + + let ofType = byOrdinal.get(type); + if (ofType === undefined) { + ofType = []; + byOrdinal.set(type, ofType); + } + ofType.push(statement); + + let isNamed = named.get(type); + if (isNamed === undefined) { + isNamed = schema.get(statement.typeNameText)?.anon !== 1; + named.set(type, isNamed); + } + if (!isNamed) continue; + + // The name is the first field after the type name, which is what `Statement.fields` indexes + // from zero and what `parseIdf` reads the object's name out of. + const written = statement.fields[0]; + if (written === undefined) continue; + const key = nameKey(type, fold(layer.text.slice(written.start, written.end).trim())); + // First occurrence wins, matching `addRaw`: the object the document kept is the first one + // written, so a finding about this type and name is a finding about this statement. + if (!byName.has(key)) byName.set(key, statement); + } + + return { byName, byOrdinal }; +} + +// --------------------------------------------------------------------------- +// Validation findings, which name an object and are correlated +// --------------------------------------------------------------------------- + +function placeValidation( + finding: ValidationError, + occurrence: Occurrence, + schema: Schema, + index: StatementIndex +): Placement | undefined { + const statement = correlate(finding, occurrence, index); + if (statement === undefined) return undefined; + + // A finding with no field concerns the object itself: an unknown type, a singleton written twice. + // It selects the type name, which is the part of the statement that identifies it and the only + // part short enough to underline. + if (finding.field === undefined) { + return { region: statement.typeName, precision: 'statement' }; + } + + const type = schema.get(finding.objType); + const at = type === undefined ? undefined : fieldIndexOf(type, finding.field, occurrence.index); + // A field the schema does not define, or one whose repeat the finding could not say, falls back + // to the whole statement and says so. Guessing a position for it would put an underline under a + // value that is not the one complained about, which reads as correct and is not. + if (at === undefined) return { region: statement.region, precision: 'statement' }; + + const region = statement.fields[at]; + // Past what was written: the field is absent from the text, which is exactly what a missing + // required field is. There is nothing to select, so the statement stands in. + if (region === undefined) return { region: statement.region, precision: 'statement' }; + + // A field written blank keeps an empty region between its two commas, and that is still the right + // place: it is where the value would have gone, and where an editor puts the caret to type it. + return { region, precision: 'field' }; +} + +/** The statement a validation finding is about, by name when it has one and by ordinal when it does not. */ +function correlate( + finding: ValidationError, + occurrence: Occurrence, + index: StatementIndex +): Statement | undefined { + const type = fold(finding.objType); + if (finding.objName !== '') { + const named = index.byName.get(nameKey(type, fold(finding.objName))); + if (named !== undefined) return named; + } + const ofType = index.byOrdinal.get(type); + if (ofType === undefined) return undefined; + // `objName` is documented as empty for anonymous objects, which is the signal to use the ordinal. + // The finding does not carry one, so a caller that knows it supplies it and everything else lands + // on the first statement of the type. That is exact for the singletons nearly every anonymous + // type is, and it is the closest available answer for the few that are not. + return ofType[occurrence.ordinal ?? 0]; +} + +/** + * The positional index a schema field name occupies, or `undefined` when the schema has no answer. + * + * Positional order is `SlimType.f`, which is the IDD order and includes the name at index 0 for a + * named type. `Statement.fields` is indexed from the first field after the type name, so a fixed + * field's index into it is simply its position in `f` for both named and anonymous types: on a + * named type index 0 is the name, and on an anonymous one index 0 is the first real field, which is + * what `f` holds in each case. + * + * An extensible field lives in the repeat group instead, which begins where the fixed fields end. + * The arithmetic is the one place this can go wrong quietly: a finding about the ninth vertex of a + * surface positioned at the first looks plausible in a screenshot and is useless in an editor. Worked + * on `BuildingSurface:Detailed`, whose eleven fixed fields are followed by repeats three wide, the + * x coordinate of the ninth vertex is `11 + 8 * 3 + 0`, which is field 35 of the statement. + */ +function fieldIndexOf( + type: SlimType, + field: string, + repeat: number | undefined +): number | undefined { + const fixed = type.f.indexOf(field); + if (fixed !== -1) return fixed; + + const extensible = type.x; + if (extensible === undefined) return undefined; + const offsetWithinGroup = extensible.fields.indexOf(field); + if (offsetWithinGroup === -1) return undefined; + // Which repeat is not something a `ValidationError` can say, and a wrong repeat is worse than no + // position at all, so without one this declines to answer and the caller falls back to the + // statement. + if (repeat === undefined) return undefined; + + const fixedCount = type.f.length; + const groupWidth = extensible.fields.length; + return fixedCount + repeat * groupWidth + offsetWithinGroup; +} + +// --------------------------------------------------------------------------- +// Reading findings, which name a place and are used as they are +// --------------------------------------------------------------------------- + +/** + * Where a reading finding sits, from the line and column the scanner recorded. + * + * Correlation is not used and must not be: a duplicate-name finding names a type and a name that + * belong to the statement ABOVE the one it is about, so correlating it would underline the wrong + * object. The scanner saw the offending statement and said where it was, and that answer is better + * than any reconstruction of it. + * + * A finding whose column resolves to the statement's own first character is about the statement, and + * every finding `lex` and `parseIdf` produce about a statement carries one. A finding carrying only + * a line is about a field on that line, which is what the `InvalidField` diagnostic is; it becomes + * field-precise when exactly one field begins on that line, and stays statement-precise when several + * do, because then which one is meant is not recoverable. + */ +function placeParse(finding: ParseDiagnostic, layer: SyntaxLayer): Placement | undefined { + const span = lineSpan(layer, finding.line); + const statement = statementNear(layer, span); + if (statement === undefined) return undefined; + + if (finding.column !== undefined) { + const at = offsetAt(layer, { line: finding.line, column: finding.column }); + if (at === statement.region.start) { + return { region: statement.typeName, precision: 'statement' }; + } + } + + const field = soleFieldOn(statement, span); + return field === undefined + ? { region: statement.region, precision: 'statement' } + : { region: field, precision: 'field' }; +} + +/** The statement a line falls in, or the one beginning later on it when the line falls between two. */ +function statementNear(layer: SyntaxLayer, span: Region): Statement | undefined { + const statements = layer.statements; + const at = lastStartingAtOrBefore(statements, span.start); + const containing = at === -1 ? undefined : statements[at]; + if (containing !== undefined && span.start < containing.region.end) return containing; + // Between two statements, which is where a finding on the blank line above a statement lands. + const next = statements[at + 1]; + return next !== undefined && next.region.start <= span.end ? next : containing; +} + +/** The one field beginning on this line, or `undefined` when none or several do. */ +function soleFieldOn(statement: Statement, span: Region): Region | undefined { + let found: Region | undefined; + for (const field of statement.fields) { + if (field.start < span.start || field.start > span.end) continue; + if (found !== undefined) return undefined; + found = field; + } + return found; +} + +/** Index of the last statement beginning at or before `offset`, or -1 when none does. */ +function lastStartingAtOrBefore(statements: readonly Statement[], offset: number): number { + let low = 0; + let high = statements.length - 1; + let found = -1; + while (low <= high) { + const middle = (low + high) >> 1; + if (statements[middle]!.region.start <= offset) { + found = middle; + low = middle + 1; + } else { + high = middle - 1; + } + } + return found; +} + +/** + * The offsets a 1-based line begins and ends at, the ending one being the line break itself. + * + * `offsetAt` clamps a column past the end of its line to that break, so asking for an impossible + * column is how the end is found without a second index of the text. + */ +function lineSpan(layer: SyntaxLayer, line: number): Region { + return { + start: offsetAt(layer, { line, column: 1 }), + end: offsetAt(layer, { line, column: Number.MAX_SAFE_INTEGER }), + }; +} + +// --------------------------------------------------------------------------- +// Telling the two shapes apart +// --------------------------------------------------------------------------- + +/** + * A `ValidationError` names an object type; nothing else this positions does. + * + * Structural rather than nominal because `position` is generic over the finding, which is what keeps + * both types unmodified. The two shapes are disjoint on this property in both libraries. + */ +function isValidationShaped(finding: object): finding is ValidationError { + return typeof (finding as { objType?: unknown }).objType === 'string'; +} + +/** A `ParseDiagnostic` names a line. Checked after the above, since a validation finding names none. */ +function isParseShaped(finding: object): finding is ParseDiagnostic { + return typeof (finding as { line?: unknown }).line === 'number'; +} + +/** Whatever a caller attached to say which occurrence a finding is about. See {@link Occurrence}. */ +function occurrenceOf(finding: object): Occurrence { + const carried = finding as Occurrence; + return { + ordinal: typeof carried.ordinal === 'number' ? carried.ordinal : undefined, + index: typeof carried.index === 'number' ? carried.index : undefined, + }; +} + +/** Case folding, as EnergyPlus resolves a type name and as `IdfCollection` keys a name. */ +function fold(value: string): string { + return value.toLowerCase(); +} + +/** + * The two halves of the name key, joined by a character neither of them can contain. + * + * An object name routinely holds spaces, commas and punctuation, so the separator is the one + * character IDF text cannot carry at all. Joining on anything a name may hold would let two + * different pairs produce one key. + */ +function nameKey(type: string, name: string): string { + return `${type}${SEPARATOR}${name}`; +} + +/** + * The NUL character itself, written as a constant rather than as an escape inside the template. + * + * `\u0000` inside a template literal needs one backslash, and a second one turns it into the + * six printable characters `\u0000`, which a name may perfectly well contain: the join would then + * be ambiguous in exactly the way the doc comment above says it is not. + */ +const SEPARATOR = '\u0000'; diff --git a/packages/language/src/index.ts b/packages/language/src/index.ts new file mode 100644 index 0000000..923a58a --- /dev/null +++ b/packages/language/src/index.ts @@ -0,0 +1,37 @@ +/** + * `@idfkit/language` — the opt-in language service for IDF text. + * + * Answers a cursor: what completes here, what this means, what this points at. + * Positions the findings a parse and a validation already produced. Everything + * exported here is synchronous and free of I/O, so the same code runs unchanged + * in Node, a browser, a browser worker, and behind an editor server. + * + * Every answer takes the text itself and never a path (FR-026): an editor's + * buffer differs from the file on disk whenever there are unsaved changes, + * which is most of the time an editor is interesting. Nothing here reads a + * file, opens a socket, consults a clock, or returns a promise, and there is no + * service object to construct, because a service object is where state would + * accumulate. + * + * The syntax layer this builds on is not re-exported. `scanIdf` and `classify` + * come from `@idfkit/core`, which this package peer-depends on, because the + * layer serves reading and writing too and one function deserves one name. + * + * Nothing here imports, depends on, or names a type from any editor protocol + * library, and nothing here ever will. A consumer translates. + */ + +export { contextAt } from './cursor.js'; +export type { CursorContext } from './cursor.js'; + +export { completionsAt } from './complete.js'; +export type { CompletionOptions, CompletionResult, Offer } from './complete.js'; + +export { explainAt } from './explain.js'; +export type { Explanation, ExplanationResult } from './explain.js'; + +export { declarationAt } from './declaration.js'; +export type { Declaration, DeclarationResult } from './declaration.js'; + +export { findingsIn, position } from './findings.js'; +export type { PositionedFinding } from './findings.js'; diff --git a/packages/language/tests/answers.test.ts b/packages/language/tests/answers.test.ts new file mode 100644 index 0000000..2e92c55 --- /dev/null +++ b/packages/language/tests/answers.test.ts @@ -0,0 +1,591 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +import { beforeAll, describe, expect, it } from 'vitest'; + +import { describeObjectType, docsUrlForObject, parseIdf } from '@idfkit/core'; +import type { IdfDocument, ProsePool, Schema } from '@idfkit/core'; + +import { prose as loadProse, schema } from '../../core/tests/helpers.js'; +import { completionsAt } from '../src/complete.js'; +import { declarationAt } from '../src/declaration.js'; +import { explainAt } from '../src/explain.js'; + +let v26: Schema; +beforeAll(async () => { + v26 = await schema('26.1.0'); +}); + +/** + * One model carrying every shape the three answers have to distinguish. + * + * Written out here rather than taken from the syntax fixtures because those exist to carry + * malformations, and these tests are about what the schema says when the text is ordinary. It + * carries, deliberately: a choice field, a reference field pointing into a list two types declare + * into, a value the model declares nowhere, a type name containing a colon, values containing + * spaces, and a free-text field holding a string that is a declared name somewhere else. + */ +const model = [ + 'Version, 26.1;', + '', + 'Material,', + ' Insulation Board, !- Name', + ' VeryRough, !- Roughness', + ' 0.1, !- Thickness {m}', + ' 0.5, !- Conductivity {W/m-K}', + ' 100, !- Density {kg/m3}', + ' 900; !- Specific Heat {J/kg-K}', + '', + 'Material:NoMass,', + ' Air Gap, !- Name', + ' Smooth, !- Roughness', + ' 0.15; !- Thermal Resistance {m2-K/W}', + '', + 'Construction,', + ' Exterior Wall, !- Name', + ' Insulation Board, !- Outside Layer', + ' Air Gap; !- Layer 2', + '', + 'Construction,', + ' Ghost Wall, !- Name', + ' No Such Material; !- Outside Layer', + '', + 'Zone,', + ' Office Zone 1, !- Name', + ' 0.0, !- Direction of Relative North', + ' 0.0, !- X Origin', + ' 0.0, !- Y Origin', + ' 0.0; !- Z Origin', + '', + 'BuildingSurface:Detailed,', + ' South Wall, !- Name', + ' Wall, !- Surface Type', + ' Exterior Wall, !- Construction Name', + ' Office Zone 1, !- Zone Name', + ' , !- Space Name', + ' Outdoors, !- Outside Boundary Condition', + ' , !- Outside Boundary Condition Object', + ' SunExposed, !- Sun Exposure', + ' WindExposed, !- Wind Exposure', + ' 0.5, !- View Factor to Ground', + ' 4, !- Number of Vertices', + ' 0.0, 0.0, 3.0, !- Vertex 1', + ' 4.0, 0.0, 3.0, !- Vertex 2', + ' 4.0, 0.0, 0.0, !- Vertex 3', + ' 0.0, 0.0, 0.0; !- Vertex 4', + '', + 'Output:Variable,', + ' Exterior Wall, !- Key Value', + ' Zone Mean Air Temperature, !- Variable Name', + ' Hourly; !- Reporting Frequency', + '', +].join('\n'); + +let document: IdfDocument; +beforeAll(() => { + document = parseIdf(model, v26, { strict: false }).document; +}); + +/** + * Offset of the first character of the value written on the line carrying `comment`. + * + * Every test here is about a position, and `text.indexOf('0.1') + 1` hides which field was meant. + * Naming the line by its trailing comment is how a reader of the fixture finds the same place. + */ +function valueOn(text: string, comment: string, from = 0): number { + const marker = text.indexOf(comment, from); + if (marker < 0) throw new Error(`no line carries the comment ${JSON.stringify(comment)}`); + const lineStart = text.lastIndexOf('\n', marker) + 1; + const value = /\S/.exec(text.slice(lineStart, marker)); + if (value === null) throw new Error(`nothing is written on the ${comment} line`); + return lineStart + value.index; +} + +/** Offset one character into the type name of the statement that opens with `written`. */ +function typeNameIn(text: string, written: string, from = 0): number { + const found = text.indexOf(written, from); + if (found < 0) throw new Error(`fixture does not contain ${JSON.stringify(written)}`); + return found + 1; +} + +/** What the text says between two offsets, which is what a region is asserted through. */ +function sliced(text: string, region: { readonly start: number; readonly end: number }): string { + return text.slice(region.start, region.end); +} + +// --------------------------------------------------------------------------- +// completionsAt +// --------------------------------------------------------------------------- + +describe('completionsAt, on a choice field', () => { + it('offers exactly the schema list for that field, and nothing else', () => { + // SC-006, and the reason the expectation is read out of the schema rather than written here: a + // hand-written list is a second copy of schema knowledge, and it would keep passing while the + // code offered the neighbouring field's values or a stale set from an older version. + const roughness = v26.get('Material')?.p['roughness']; + expect(roughness?.e).toBeDefined(); + // No blank branch on this field, so the schema's list is `e` exactly. A field carrying `eb` + // declares a blank the bundle filtered out, and the comparison would then need it back. + expect(roughness?.eb).toBeUndefined(); + + const result = completionsAt(model, valueOn(model, '!- Roughness'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.offers.map((offer) => offer.value)).toEqual([...roughness!.e!]); + for (const offer of result.offers) { + expect(offer.kind).toBe('enumValue'); + expect(offer.required).toBe((v26.get('Material')?.r ?? []).includes('roughness')); + // No pool was supplied, so no prose is hydrated. FR-028, asserted in full below. + expect(offer.prose).toBeUndefined(); + } + }); +}); + +describe('completionsAt, on a reference field', () => { + it('offers the names declared into the lists the field points into', () => { + // The list comes from the schema, and the names come from the caller's document: two types + // declare into `MaterialName` here, and both are offered, because a `Material:NoMass` is as + // good a layer as a `Material`. + expect(v26.get('Construction')?.p['outside_layer']?.ol).toEqual(['MaterialName']); + + const result = completionsAt(model, valueOn(model, '!- Outside Layer'), v26, { document }); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.offers.map((offer) => offer.value)).toEqual(['Insulation Board', 'Air Gap']); + expect(result.offers.every((offer) => offer.kind === 'referenceTarget')).toBe(true); + }); + + it('offers the zone names for a zone field, and not every name in the model', () => { + const result = completionsAt(model, valueOn(model, '!- Zone Name'), v26, { document }); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.offers.map((offer) => offer.value)).toEqual(['Office Zone 1']); + }); + + it('offers none and says so when no document was supplied, rather than parsing one', () => { + // Research R9: a service that quietly parsed a document when the argument was omitted would + // make one function on the keystroke path eighty milliseconds slower depending on an argument + // nobody passed. `'notApplicable'` says the answer is unavailable; an empty `'ok'` list would + // say the model has declared nothing, which is false here and is the state below. + const result = completionsAt(model, valueOn(model, '!- Outside Layer'), v26); + + expect(result).toEqual({ status: 'notApplicable' }); + }); + + it('reports an empty offer list when the document really declares nothing', () => { + const empty = parseIdf('Version, 26.1;\n', v26, { strict: false }).document; + const result = completionsAt(model, valueOn(model, '!- Outside Layer'), v26, { + document: empty, + }); + + expect(result).toEqual({ status: 'ok', offers: [] }); + }); +}); + +describe('completionsAt, where the schema constrains nothing', () => { + it('returns unconstrained for a numeric field rather than an empty list', () => { + // FR-020 and FR-031: "the schema permits anything here" is a different answer from "there is + // nothing to offer", and an editor that rendered an empty list for both would teach the reader + // that the tool is broken. + const result = completionsAt(model, valueOn(model, '!- Thickness'), v26); + + expect(result).toEqual({ status: 'unconstrained' }); + }); + + it('returns unconstrained for a free-text name field', () => { + const result = completionsAt(model, valueOn(model, '!- Name'), v26); + + expect(result).toEqual({ status: 'unconstrained' }); + }); +}); + +/** + * SC-018 and FR-048: every offer carries the region it stands in for. + * + * Checked against the two shapes an editor's own word rules get wrong, because those are the ones a + * consumer would get wrong if it derived the span itself: a type name is split at its colon, and a + * value is split at its spaces. The region is sliced out of the text and compared to the characters + * it should select, so a failure reads as the wrong word rather than as two numbers. + */ +describe('every offer carries the region it replaces', () => { + it('replaces a whole type name, colon included', () => { + const result = completionsAt(model, typeNameIn(model, 'Material:NoMass,'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.offers.length).toBeGreaterThan(0); + for (const offer of result.offers) { + expect(sliced(model, offer.replaces)).toBe('Material:NoMass'); + expect(offer.kind).toBe('objectType'); + } + // The shape being guarded against is really in the list, so this cannot pass on a schema of + // single-word names. + expect(result.offers.some((offer) => offer.value.includes(':'))).toBe(true); + }); + + it('replaces a whole value, spaces included', () => { + const result = completionsAt(model, valueOn(model, '!- Zone Name'), v26, { document }); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.offers.length).toBeGreaterThan(0); + for (const offer of result.offers) { + expect(sliced(model, offer.replaces)).toBe('Office Zone 1'); + } + }); + + it('replaces a whole choice value, from a cursor part way through it', () => { + const written = valueOn(model, '!- Roughness'); + const result = completionsAt(model, written + 4, v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + for (const offer of result.offers) { + expect(sliced(model, offer.replaces)).toBe('VeryRough'); + } + }); + + it('replaces nothing where a statement has not been written yet', () => { + // An insertion rather than a replacement, and an empty region at the cursor is how that is + // said. A consumer applying an offer here inserts at the caret and disturbs no text. + const text = `${model}\n`; + const result = completionsAt(text, text.length, v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + for (const offer of result.offers) { + expect(offer.replaces).toEqual({ start: text.length, end: text.length }); + } + }); +}); + +// --------------------------------------------------------------------------- +// explainAt +// --------------------------------------------------------------------------- + +describe('explainAt, on a field', () => { + it("reports the schema's own facts for the field, and not a paraphrase of them", () => { + const result = explainAt(model, valueOn(model, '!- Thickness'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + const { explanation } = result; + + expect(explanation.of).toBe('field'); + expect(explanation.typeName).toBe('Material'); + expect(explanation.fieldName).toBe('thickness'); + expect(sliced(model, explanation.region)).toBe('0.1'); + // FR-029: exactly what `describeObjectType` produces, member for member, rather than a second + // reading of the same bundle that could drift from the reference page. + expect(explanation.field).toEqual( + describeObjectType(v26, 'Material').fields.find((field) => field.name === 'thickness') + ); + // Named individually as well, so the comparison above cannot pass by both sides being empty. + expect(explanation.field?.fieldType).toBe('number'); + expect(explanation.field?.units).toBe('m'); + expect(explanation.field?.required).toBe(true); + expect(explanation.field?.exclusiveMinimum).toBe(0); + }); + + it('reports the permitted values of a choice field', () => { + const result = explainAt(model, valueOn(model, '!- Roughness'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.explanation.field?.enumValues).toEqual([ + ...v26.get('Material')!.p['roughness']!.e!, + ]); + }); + + it('reports the default a field carries', () => { + const result = explainAt(model, valueOn(model, '!- Direction of Relative North'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.explanation.fieldName).toBe('direction_of_relative_north'); + expect(result.explanation.field?.default).toBe(0); + expect(result.explanation.field?.units).toBe('deg'); + }); + + it("reports the field's own prose when the caller supplied a pool", async () => { + const pool = await loadProse(); + const result = explainAt(model, valueOn(model, '!- Zone Name'), v26, pool); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.explanation.prose).toBe(result.explanation.field?.note); + expect(result.explanation.prose).toEqual(expect.any(String)); + }); +}); + +describe('explainAt, on a type name', () => { + it("reports the type's prose and the manual location", async () => { + const pool = await loadProse(); + const result = explainAt(model, typeNameIn(model, 'Material,'), v26, pool); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + const { explanation } = result; + + expect(explanation.of).toBe('objectType'); + expect(explanation.typeName).toBe('Material'); + expect(explanation.fieldName).toBeUndefined(); + expect(explanation.field).toBeUndefined(); + expect(sliced(model, explanation.region)).toBe('Material'); + expect(explanation.prose).toBe(describeObjectType(v26, 'Material', pool).memo); + expect(explanation.prose).toEqual(expect.any(String)); + // FR-029 again: the manual location is `docsUrlForObject`'s answer and not a URL assembled + // here, so a documentation move is one change rather than two. + expect(explanation.docs).toEqual(docsUrlForObject('Material', v26.version, v26)); + expect(explanation.docs).toBeDefined(); + }); +}); + +describe('explainAt, with no prose pool', () => { + /** Every string the value carries, however deeply, which is what FR-022 is asserted over. */ + function stringsIn(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(stringsIn); + if (value !== null && typeof value === 'object') { + return Object.values(value).flatMap(stringsIn); + } + return []; + } + + it('reports the structural facts and no prose at all', () => { + const result = explainAt(model, valueOn(model, '!- Thickness'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.explanation.prose).toBeUndefined(); + expect(result.explanation.field?.note).toBeUndefined(); + // The facts are still there. An explanation that reported nothing would satisfy the assertion + // above and would be useless. + expect(result.explanation.field?.units).toBe('m'); + }); + + it('never derives a sentence from the field name', () => { + // FR-022 forbids this by name, which is why it is asserted by name. The only strings an + // explanation may spell the field with are the two that ARE its name; anything else carrying + // it is text made up from the spelling, which is the failure this exists to catch. + const result = explainAt(model, valueOn(model, '!- Thickness'), v26); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + const { explanation } = result; + + const spelled = stringsIn(explanation).filter( + (value) => value !== explanation.fieldName && value !== explanation.field?.name + ); + expect(spelled.length).toBeGreaterThan(0); + for (const value of spelled) { + expect(value.toLowerCase()).not.toContain('thickness'); + } + }); +}); + +describe('explainAt, where there is nothing to explain', () => { + it('reports notApplicable on the whitespace beside a value', () => { + // Reporting the nearest field here is the behaviour that makes a hover feel haunted (FR-049). + const written = valueOn(model, '!- Roughness'); + const result = explainAt(model, written + 'VeryRough,'.length + 2, v26); + + expect(result).toEqual({ status: 'notApplicable' }); + }); + + it('reports notApplicable on a separator', () => { + const written = valueOn(model, '!- Roughness'); + expect(model[written + 'VeryRough'.length]).toBe(','); + + expect(explainAt(model, written + 'VeryRough'.length, v26)).toEqual({ + status: 'notApplicable', + }); + }); + + it('reports notApplicable on the blank line between two statements', () => { + const blank = model.indexOf('\n\nMaterial:NoMass'); + + expect(explainAt(model, blank + 1, v26)).toEqual({ status: 'notApplicable' }); + }); + + it('reports notApplicable inside a comment', () => { + expect(explainAt(model, model.indexOf('!- Thickness') + 3, v26)).toEqual({ + status: 'notApplicable', + }); + }); +}); + +/** + * FR-028: a caller who never supplies the pool loads none of it. + * + * Fenced the way `check-bundle-purity.mjs` fences the schema data it is about: not by measuring how + * much prose came back, but by establishing that the thing which loads it never enters the graph at + * all. The pool is a plain array the caller reads for itself, through `@idfkit/schemas`'s bundle + * source, and this package cannot reach one unless a module here imports something that reads + * bytes. So the static half asserts that none of them does, and the behavioural half asserts that + * nothing is hydrated when no pool arrives, with the control that makes both able to fail. + */ +describe('the prose pool stays the caller to load', () => { + const sourceDir = fileURLToPath(new URL('../src/', import.meta.url)); + const sources = readdirSync(sourceDir) + .filter((entry) => entry.endsWith('.ts')) + .map((entry) => ({ name: entry, text: readFileSync(`${sourceDir}${entry}`, 'utf8') })); + + it('has sources to read, so the checks below are not vacuous', () => { + expect(sources.map((source) => source.name)).toContain('explain.ts'); + }); + + it.each(sources)('imports nothing that could read the pool, in $name', ({ text }) => { + // The two ways bytes reach a module here: Node's own file reading, and either package's `/node` + // entry point, which is where every reader in this repository lives. Neither may appear. + expect(text).not.toMatch(/from '(node:[a-z/]+|@idfkit\/[a-z]+\/node)'/); + }); + + it.each(sources)('names ProsePool as a type alone, in $name', ({ text }) => { + // A value import of the pool's declaring module would pull the loader in behind it. As a type + // it is erased entirely, so the parameter can be named without anything being reachable. + for (const statement of text.match(/import[\s\S]*?from '[^']+';/g) ?? []) { + if (!statement.includes('ProsePool')) continue; + expect(statement).toMatch(/\btype ProsePool\b/); + } + }); + + it('hydrates no prose anywhere when no pool is supplied', () => { + let explained = 0; + for (let offset = 0; offset <= model.length; offset += 1) { + const result = explainAt(model, offset, v26); + if (result.status !== 'ok') continue; + explained += 1; + expect(result.explanation.prose).toBeUndefined(); + expect(result.explanation.field?.note).toBeUndefined(); + } + // Several hundred of this model's offsets are on a type name or a value; the rest are on + // padding, comments and separators. A sweep that explained none of them would be a broken + // sweep rather than a clean one, so the count is asserted rather than assumed. + expect(explained).toBeGreaterThan(100); + + const offers = completionsAt(model, typeNameIn(model, 'Material:NoMass,'), v26); + expect(offers.status).toBe('ok'); + if (offers.status !== 'ok') return; + expect(offers.offers.every((offer) => offer.prose === undefined)).toBe(true); + }); + + it('hydrates prose when a pool is supplied, so the sweep above can fail', async () => { + const pool: ProsePool = await loadProse(); + + const explained = explainAt(model, typeNameIn(model, 'Material,'), v26, pool); + expect(explained.status).toBe('ok'); + if (explained.status !== 'ok') return; + expect(explained.explanation.prose).toEqual(expect.any(String)); + + const offers = completionsAt(model, typeNameIn(model, 'Material:NoMass,'), v26, { + prose: pool, + }); + expect(offers.status).toBe('ok'); + if (offers.status !== 'ok') return; + expect(offers.offers.some((offer) => typeof offer.prose === 'string')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// declarationAt +// --------------------------------------------------------------------------- + +describe('declarationAt', () => { + it('selects the name field of the one statement that declares the name', () => { + const result = declarationAt(model, valueOn(model, '!- Construction Name'), v26, document); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.declarations).toHaveLength(1); + + const [declaration] = result.declarations; + expect(declaration?.typeName).toBe('Construction'); + expect(sliced(model, declaration!.region)).toBe('Exterior Wall'); + // The DECLARING statement's name field, and not the surface's own value or the free-text field + // further down that happens to carry the same string. + expect(declaration?.region.start).toBe( + valueOn(model, '!- Name', model.indexOf('Construction,')) + ); + }); + + it('returns every declaration when two objects declare the name', () => { + // Two types declaring into one list is ordinary in this format: a layer may be a `Material` or + // a `Material:NoMass`, and a model that carries both under one name has two declaration sites. + // Reporting the first would send a reader to whichever one the document happened to hold first. + const text = [ + 'Version, 26.1;', + '', + 'Material,', + ' Shared Layer, !- Name', + ' VeryRough, !- Roughness', + ' 0.1, !- Thickness {m}', + ' 0.5, !- Conductivity {W/m-K}', + ' 100, !- Density {kg/m3}', + ' 900; !- Specific Heat {J/kg-K}', + '', + 'Material:NoMass,', + ' Shared Layer, !- Name', + ' Smooth, !- Roughness', + ' 0.15; !- Thermal Resistance {m2-K/W}', + '', + 'Construction,', + ' Wall Assembly, !- Name', + ' Shared Layer; !- Outside Layer', + '', + ].join('\n'); + const both = parseIdf(text, v26, { strict: false }).document; + + const result = declarationAt(text, valueOn(text, '!- Outside Layer'), v26, both); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.declarations.map((declaration) => declaration.typeName)).toEqual([ + 'Material', + 'Material:NoMass', + ]); + expect(result.declarations.map((declaration) => declaration.region.start)).toEqual([ + valueOn(text, '!- Name'), + valueOn(text, '!- Name', text.indexOf('Material:NoMass,')), + ]); + for (const declaration of result.declarations) { + expect(sliced(text, declaration.region)).toBe('Shared Layer'); + } + }); + + it('returns none, and guesses nothing, when the name is declared nowhere', () => { + // The dangling-reference finding is what tells the reader why there is nothing here. Offering a + // near miss would contradict it. + const result = declarationAt( + model, + valueOn(model, '!- Outside Layer', model.indexOf('Ghost Wall')), + v26, + document + ); + + expect(result).toEqual({ status: 'ok', declarations: [] }); + }); + + it('reports notApplicable for a field that points at nothing', () => { + // `Output:Variable`'s key value is free text: it points into no reference list, and the string + // written there is a `Construction` name in this very model. Searching the document for an + // object that happens to be called whatever is written would find one, and would be a guess + // dressed as an answer. + const result = declarationAt(model, valueOn(model, '!- Key Value'), v26, document); + + expect(result).toEqual({ status: 'notApplicable' }); + }); + + it('finds that same string from a field that does point at it, so the test above bites', () => { + const result = declarationAt(model, valueOn(model, '!- Construction Name'), v26, document); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') return; + expect(result.declarations.map((declaration) => sliced(model, declaration.region))).toEqual([ + 'Exterior Wall', + ]); + }); +}); diff --git a/packages/language/tests/cursor.test.ts b/packages/language/tests/cursor.test.ts new file mode 100644 index 0000000..58417d9 --- /dev/null +++ b/packages/language/tests/cursor.test.ts @@ -0,0 +1,259 @@ +import { beforeAll, describe, expect, it } from 'vitest'; + +import { scanIdf } from '@idfkit/core'; +import type { Schema } from '@idfkit/schemas'; + +import { schema, syntaxFixture, syntaxFixtures } from '../../core/tests/helpers.js'; +import { contextAt } from '../src/cursor.js'; + +let v26: Schema; +beforeAll(async () => { + v26 = await schema('26.1.0'); +}); + +/** + * Offset of the character after `needle`, which is where a cursor sits when somebody has just + * finished typing it. + * + * Spelled out once because every test here is about a position, and `text.indexOf(x) + x.length` + * repeated a dozen times hides which position was meant. + */ +function after(text: string, needle: string, from = 0): number { + const at = text.indexOf(needle, from); + if (at < 0) throw new Error(`fixture does not contain ${JSON.stringify(needle)}`); + return at + needle.length; +} + +/** Offset of the first character of `needle`. */ +function at(text: string, needle: string, from = 0): number { + const found = text.indexOf(needle, from); + if (found < 0) throw new Error(`fixture does not contain ${JSON.stringify(needle)}`); + return found; +} + +describe('contextAt, inside a statement', () => { + it('places a half-typed field with no separator after it in that field', () => { + // The last field of the file, written but never closed: no comma, no semicolon, nothing after + // it but a line feed. This is the state a cursor is in for most of the time it is anywhere. + const text = syntaxFixture('unterminated-final-statement'); + const context = contextAt(text, after(text, '0.0', at(text, '0.0,') + 1), v26); + + expect(context.at).toBe('field'); + expect(context.fieldIndex).toBe(2); + expect(context.typeName).toBe('Zone'); + expect(context.fieldName).toBe('x_origin'); + expect(context.statement.unterminated).toBe(true); + }); + + it('places a value written across two lines in the field that holds both halves', () => { + const text = syntaxFixture('value-across-two-lines'); + const context = contextAt(text, after(text, 'My'), v26); + + expect(context.at).toBe('field'); + expect(context.fieldIndex).toBe(0); + expect(context.fieldName).toBe('name'); + expect(text.slice(context.statement.fields[0]!.start, context.statement.fields[0]!.end)).toBe( + 'My\n Zone' + ); + }); + + it('places an offset on the type name in the type name', () => { + const text = syntaxFixture('line-endings-lf'); + const context = contextAt(text, at(text, 'Zone,') + 2, v26); + + expect(context.at).toBe('typeName'); + expect(context.fieldIndex).toBeUndefined(); + expect(context.typeName).toBe('Zone'); + expect(context.fieldName).toBeUndefined(); + }); + + it('places an offset on the terminator in the last field it closes', () => { + const text = syntaxFixture('line-endings-lf'); + const context = contextAt(text, at(text, '0.0;'), v26); + + expect(context.at).toBe('field'); + expect(context.fieldIndex).toBe(2); + expect(context.fieldName).toBe('x_origin'); + }); +}); + +describe('contextAt, between statements', () => { + it('reports betweenStatements immediately after a terminator', () => { + const text = syntaxFixture('line-endings-lf'); + const context = contextAt(text, after(text, 'Version, 26.1;'), v26); + + expect(context.at).toBe('betweenStatements'); + expect(context.fieldIndex).toBeUndefined(); + expect(context.statement.typeNameText).toBe(''); + }); + + it('reports betweenStatements in trailing whitespace at end of file', () => { + const text = syntaxFixture('line-endings-lf'); + expect(text.endsWith('\n')).toBe(true); + + const context = contextAt(text, text.length, v26); + + expect(context.at).toBe('betweenStatements'); + // The statement a cursor is beginning selects nothing, at the cursor, which is the region an + // offer accepted here would insert into. + expect(context.statement.region).toEqual({ start: text.length, end: text.length }); + }); + + it('reports betweenStatements on a blank line between two statements', () => { + const text = syntaxFixture('line-endings-lf'); + const context = contextAt(text, at(text, '\n\nZone,') + 1, v26); + + expect(context.at).toBe('betweenStatements'); + }); + + it('reports betweenStatements for every offset in empty text', () => { + expect(contextAt('', 0, v26).at).toBe('betweenStatements'); + expect(contextAt(' \n ', 4, v26).at).toBe('betweenStatements'); + }); +}); + +describe('contextAt, in a comment', () => { + it('reports comment on the exclamation mark itself', () => { + const text = syntaxFixture('line-endings-lf'); + const bang = at(text, '!- Name'); + + expect(contextAt(text, bang, v26).at).toBe('comment'); + expect(contextAt(text, bang + 4, v26).at).toBe('comment'); + }); + + it('reports comment at the end of the comment text, before the line feed', () => { + const text = syntaxFixture('line-endings-lf'); + const end = at(text, '\n', at(text, '!- Name')); + + expect(contextAt(text, end, v26).at).toBe('comment'); + }); + + it('still reports the statement the comment interrupts', () => { + const text = syntaxFixture('line-endings-lf'); + const context = contextAt(text, at(text, '!- Direction'), v26); + + expect(context.at).toBe('comment'); + expect(context.typeName).toBe('Zone'); + expect(context.fieldIndex).toBeUndefined(); + }); + + it('reports comment in a file that is nothing but comments', () => { + const text = syntaxFixture('comments-only'); + + expect(contextAt(text, 0, v26).at).toBe('comment'); + expect(contextAt(text, 30, v26).at).toBe('comment'); + }); + + it('is not fooled by a semicolon inside a comment', () => { + // The comment on the Name field carries both a comma and a semicolon. A backward scan that + // took that semicolon for a terminator would resolve the two fields below it into a statement + // that does not exist. + const text = syntaxFixture('comma-inside-trailing-comment'); + const context = contextAt(text, at(text, '0.0;'), v26); + + expect(context.at).toBe('field'); + expect(context.typeName).toBe('Zone'); + expect(context.fieldIndex).toBe(2); + }); +}); + +describe('contextAt, above a malformation', () => { + it('resolves an offset above an unterminated statement further down the file', () => { + const text = syntaxFixture('unterminated-final-statement'); + const context = contextAt(text, at(text, 'Version') + 3, v26); + + expect(context.at).toBe('typeName'); + expect(context.typeName).toBe('Version'); + expect(context.statement.unterminated).toBe(false); + expect(text.slice(context.statement.region.start, context.statement.region.end)).toBe( + 'Version, 26.1;' + ); + }); + + it('resolves an offset above a statement that swallowed the one after it', () => { + const text = syntaxFixture('missing-terminator-swallows-next'); + const above = contextAt(text, at(text, '26.1'), v26); + + expect(above.at).toBe('field'); + expect(above.typeName).toBe('Version'); + + // The Zone that forgot its terminator runs on through the Zone below it, exactly as the layer + // reads it, and a cursor in either of them is in that one statement. + const inside = contextAt(text, at(text, 'Zone Two'), v26); + expect(inside.typeName).toBe('Zone'); + expect(text.slice(inside.statement.region.end - 1, inside.statement.region.end)).toBe(';'); + expect(inside.statement.region.start).toBe(at(text, 'Zone,')); + }); + + it('reports a written type name the schema does not define, and no field name', () => { + const text = syntaxFixture('unknown-object-type'); + const context = contextAt(text, at(text, 'Whatever'), v26); + + expect(context.at).toBe('field'); + expect(context.fieldIndex).toBe(0); + expect(context.statement.typeNameText).toBe('NotAnObject:AtAll'); + // The context is still returned: a finding about the unknown type needs positioning too. + expect(context.typeName).toBeUndefined(); + expect(context.fieldName).toBeUndefined(); + }); + + it('answers without a schema, reporting the written name and the field index', () => { + const text = syntaxFixture('no-version-declared'); + const context = contextAt(text, at(text, 'Zone One')); + + expect(context.at).toBe('field'); + expect(context.fieldIndex).toBe(0); + expect(context.statement.typeNameText).toBe('Zone'); + expect(context.typeName).toBeUndefined(); + expect(context.fieldName).toBeUndefined(); + }); +}); + +describe('contextAt, out of range', () => { + const text = syntaxFixture('line-endings-lf'); + + it('clamps a negative offset to the start rather than throwing', () => { + expect(contextAt(text, -1, v26)).toEqual(contextAt(text, 0, v26)); + expect(contextAt(text, -1e9, v26).at).toBe('typeName'); + }); + + it('clamps an offset past the end to the end rather than throwing', () => { + expect(contextAt(text, text.length + 1000, v26)).toEqual(contextAt(text, text.length, v26)); + }); + + it('answers for an offset that is not a whole number, and for NaN', () => { + expect(contextAt(text, 4.7, v26)).toEqual(contextAt(text, 4, v26)); + expect(contextAt(text, Number.NaN, v26)).toEqual(contextAt(text, 0, v26)); + }); +}); + +/** + * The load-bearing property, and the reason this file exists. + * + * The cursor reads one statement by scanning outwards from an offset; the layer reads every + * statement in one forward pass. Two implementations of the same character rules that differ by + * one character would put a completion on the neighbouring field, and nothing else in the suite + * would notice. So every statement the layer finds is asked for again, one offset at a time, + * through the cursor, and the two must produce the same statement. + * + * The layer is what a test may consult. `contextAt` must not, and does not. + */ +describe('contextAt agrees with the syntax layer', () => { + for (const fixture of syntaxFixtures()) { + it(`resolves the same statement as scanIdf, in ${fixture.name}.idf`, () => { + const layer = scanIdf(fixture.text); + for (const statement of layer.statements) { + const { start, end } = statement.region; + const offsets = new Set([start, start + 1, (start + end) >> 1, end - 1]); + for (const offset of offsets) { + if (offset < start || offset >= end) continue; + const context = contextAt(fixture.text, offset); + expect( + context.statement, + `offset ${offset} of ${fixture.name}.idf resolved to the wrong statement` + ).toEqual(statement); + } + } + }); + } +}); diff --git a/packages/language/tests/degraded.test.ts b/packages/language/tests/degraded.test.ts new file mode 100644 index 0000000..60703ac --- /dev/null +++ b/packages/language/tests/degraded.test.ts @@ -0,0 +1,269 @@ +import { beforeAll, describe, expect, it } from 'vitest'; + +import { parseIdf } from '@idfkit/core'; +import type { IdfDocument, Schema } from '@idfkit/core'; + +import { schema, syntaxFixture } from '../../core/tests/helpers.js'; +// Through the public surface rather than through the modules, because "every answer returns" is a +// claim about what a consumer can reach: a function that degraded well and was never exported would +// satisfy every other file in this suite and none of SC-009. +import { completionsAt, contextAt, declarationAt, explainAt, findingsIn } from '../src/index.js'; + +let v26: Schema; +beforeAll(async () => { + v26 = await schema('26.1.0'); +}); + +/** One file that violates something, named by what it violates. */ +interface Malformed { + readonly name: string; + readonly text: string; +} + +/** + * Text that breaks one of the rules an answer would like to rely on. + * + * Taken from the syntax fixture corpus, which exists for exactly this and holds the bytes as they + * are on disk, so a fixture added there for one case is held to these invariants too. + */ +const malformed: readonly Malformed[] = [ + { name: 'empty text', text: syntaxFixture('empty') }, + { name: 'a file of nothing but comments', text: syntaxFixture('comments-only') }, + { name: 'a single unterminated word', text: syntaxFixture('single-unterminated-word') }, + { name: 'no version statement', text: syntaxFixture('no-version-declared') }, + { name: 'a version no schema is shipped for', text: syntaxFixture('unsupported-version') }, + { name: 'an unknown object type', text: syntaxFixture('unknown-object-type') }, + { name: 'an unterminated final statement', text: syntaxFixture('unterminated-final-statement') }, + { + name: 'a terminator that swallowed the statement below it', + text: syntaxFixture('missing-terminator-swallows-next'), + }, +]; + +/** The five statuses the three discriminated results are allowed to carry. */ +const STATUSES = new Set(['ok', 'unconstrained', 'noSchema', 'unknownType', 'notApplicable']); + +/** + * Offsets worth asking about in a file this small, the impossible ones included. + * + * FR-032: an offset outside `[0, text.length]` is clamped rather than refused, because a cursor + * arrives from an editor that may be a keystroke ahead of the text it was measured against. `NaN` + * and a fraction are here for the same reason: they are what an arithmetic slip upstream produces, + * and answering about the nearest character beats throwing at a consumer who cannot fix it. + */ +function probeOffsets(text: string): readonly number[] { + return [ + -1e9, + -1, + 0, + 1, + Math.floor(text.length / 2), + 4.7, + Number.NaN, + text.length, + text.length + 1, + text.length + 1000, + ]; +} + +/** The document a consumer holds, read from the same text without a strict run refusing it. */ +function documentOf(text: string, against: Schema): IdfDocument { + return parseIdf(text, against, { strict: false }).document; +} + +describe('every answer returns rather than failing', () => { + it.each(malformed)('answers with a schema, over $name', ({ text }) => { + const document = documentOf(text, v26); + + for (const offset of probeOffsets(text)) { + const context = contextAt(text, offset, v26); + expect(context.statement).toBeDefined(); + expect(['typeName', 'field', 'comment', 'betweenStatements']).toContain(context.at); + // Whatever the text says, the region it reports is inside the text it was given. + expect(context.statement.region.start).toBeGreaterThanOrEqual(0); + expect(context.statement.region.end).toBeLessThanOrEqual(text.length); + + for (const result of [ + completionsAt(text, offset, v26, { document }), + explainAt(text, offset, v26), + declarationAt(text, offset, v26, document), + ]) { + expect(STATUSES.has(result.status)).toBe(true); + } + } + }); + + it.each(malformed)('answers with no schema at all, over $name', ({ text }) => { + // FR-031: "I could not consult a schema" is a state of its own, and never an empty `'ok'`. A + // consumer that showed an empty list here would tell the reader the model permits nothing, + // where the truth is that nothing was consulted. + const document = documentOf(text, v26); + + for (const offset of probeOffsets(text)) { + const context = contextAt(text, offset); + expect(context.typeName).toBeUndefined(); + expect(context.fieldName).toBeUndefined(); + + for (const result of [ + completionsAt(text, offset, undefined, { document }), + explainAt(text, offset, undefined), + declarationAt(text, offset, undefined, document), + ]) { + expect(['noSchema', 'notApplicable']).toContain(result.status); + } + } + }); + + it.each(malformed)('positions every finding over $name', ({ text }) => { + // `findingsIn` has no schemaless state to test: its signature requires one, because findings + // are what a parse and a validation produced and neither runs without a schema. What it owes + // here is the rest of SC-009: malformed text produces positioned findings rather than a throw. + const positioned = findingsIn(text, v26); + + expect(Array.isArray(positioned)).toBe(true); + for (const finding of positioned) { + expect(finding.region.start).toBeGreaterThanOrEqual(0); + expect(finding.region.end).toBeLessThanOrEqual(text.length); + expect(finding.region.start).toBeLessThanOrEqual(finding.region.end); + } + }); +}); + +/** + * The statements above every malformation, which the answers must read exactly as if it were not + * there. + * + * Terminated, schema-resolvable, and carrying a reference so that every one of the four answers has + * something real to say about it: type names to complete, fields to explain, and a name whose + * declaration is a region further up. Its density is deliberately unreadable, so that there are + * findings above the malformation too and the positions they carry can be held to the same + * equality as everything else. + */ +const head = [ + 'Version, 26.1;', + '', + 'Material,', + ' Insulation Board, !- Name', + ' VeryRough, !- Roughness', + ' 0.1, !- Thickness {m}', + ' 0.5, !- Conductivity {W/m-K}', + ' NotANumber, !- Density {kg/m3}', + ' 900; !- Specific Heat {J/kg-K}', + '', + 'Construction,', + ' Exterior Wall, !- Name', + ' Insulation Board; !- Outside Layer', + '', + '', +].join('\n'); + +/** One malformation, and the same file with that malformation repaired and nothing else changed. */ +interface Repair { + readonly name: string; + /** What the malformed tail says. */ + readonly broken: string; + /** What the repaired tail says instead. */ + readonly fixed: string; +} + +/** + * The tails, written in pairs so that the repair is visible rather than described. + * + * Each pair differs only below `head`, which is what makes "above the malformation" a position + * rather than a judgement: the two files are byte-identical up to `head.length`, asserted before + * anything is compared. + */ +const repairs: readonly Repair[] = [ + { + name: 'an unterminated final statement', + broken: ['Zone,', ' Office Zone 1,', ' 0.0,', ' 0.0', ''].join('\n'), + fixed: ['Zone,', ' Office Zone 1,', ' 0.0,', ' 0.0;', ''].join('\n'), + }, + { + name: 'a missing terminator that swallows the statement below it', + broken: ['Zone,', ' Office Zone 1,', ' 0.0', '', 'Timestep, 6;', ''].join('\n'), + fixed: ['Zone,', ' Office Zone 1,', ' 0.0;', '', 'Timestep, 6;', ''].join('\n'), + }, + { + name: 'an object of a type the schema does not define', + broken: ['NotAnObject:AtAll,', ' Whatever,', ' 1.0;', ''].join('\n'), + fixed: ['Building,', ' Whatever,', ' 1.0;', ''].join('\n'), + }, + { + name: 'a single unterminated word', + broken: 'Zone', + fixed: 'Zone,\n Office Zone 1;\n', + }, +]; + +describe('answers above a malformation match answers over the repair', () => { + /** + * SC-009 as a property rather than a wish. + * + * "Degrades gracefully" is easy to satisfy by returning nothing everywhere and hard to satisfy + * honestly, so the claim is stated as an equality: for every offset above the malformation, all + * four answers are the ones the same file gives once the malformation is repaired. A reader + * editing the bottom of a file loses nothing at the top, which is the thing a reader actually + * notices, and no assertion here can be satisfied by a service that has stopped answering. + */ + it.each(repairs)('reads the statements above $name unchanged', ({ broken, fixed }) => { + const malformedText = head + broken; + const repairedText = head + fixed; + + expect(malformedText.slice(0, head.length)).toBe(repairedText.slice(0, head.length)); + expect(malformedText).not.toBe(repairedText); + + const malformedDocument = documentOf(malformedText, v26); + const repairedDocument = documentOf(repairedText, v26); + + let completed = 0; + let explained = 0; + let declared = 0; + + for (let offset = 0; offset < head.length; offset += 1) { + expect(contextAt(malformedText, offset, v26)).toEqual(contextAt(repairedText, offset, v26)); + + const completion = completionsAt(malformedText, offset, v26, { + document: malformedDocument, + }); + expect(completion).toEqual( + completionsAt(repairedText, offset, v26, { document: repairedDocument }) + ); + if (completion.status === 'ok') completed += 1; + + const explanation = explainAt(malformedText, offset, v26); + expect(explanation).toEqual(explainAt(repairedText, offset, v26)); + if (explanation.status === 'ok') explained += 1; + + const declaration = declarationAt(malformedText, offset, v26, malformedDocument); + expect(declaration).toEqual(declarationAt(repairedText, offset, v26, repairedDocument)); + if (declaration.status === 'ok' && declaration.declarations.length > 0) declared += 1; + } + + // Equal answers are only worth having if some of them said something. Each of the three counts + // is over the head above, which offers type names, explains a field, and follows a layer name + // to the material that declares it. + expect(completed).toBeGreaterThan(0); + expect(explained).toBeGreaterThan(0); + expect(declared).toBeGreaterThan(0); + }); + + it.each(repairs)('positions the findings above $name in the same places', ({ broken, fixed }) => { + // Keyed by code and region rather than compared as objects, so a failure reads as a finding + // that moved rather than as two large records that differ somewhere. + const above = (text: string): string[] => + findingsIn(text, v26) + .filter((finding) => finding.region.end <= head.length) + .map((finding) => `${finding.code}@${finding.region.start}:${finding.region.end}`); + + const unrepaired = above(head + broken); + + // The head's own unreadable density, found in both files and in the same place in each. An + // empty list on both sides would satisfy the equality and prove nothing. + expect(unrepaired.length).toBeGreaterThan(0); + expect(unrepaired).toEqual(above(head + fixed)); + // The malformation below is genuinely a malformation, so the equality above is about a file + // that really is broken. + expect(findingsIn(head + broken, v26).length).toBeGreaterThan(unrepaired.length); + }); +}); diff --git a/packages/language/tests/findings.test.ts b/packages/language/tests/findings.test.ts new file mode 100644 index 0000000..8ccd96a --- /dev/null +++ b/packages/language/tests/findings.test.ts @@ -0,0 +1,266 @@ +import { beforeAll, describe, expect, it } from 'vitest'; + +import { + lineColumnAt, + parseIdf, + scanIdf, + validateDocument, + type ParseDiagnostic, + type Schema, + type ValidationError, +} from '@idfkit/core'; + +// The syntax fixture corpus and the schema loader live beside the core tests and are read from +// there rather than copied. Three of those fixtures differ from one another only in their line +// endings, so a second reader that normalised anything would leave these tests passing against text +// that no longer carries the case they were written for. +import { schema, syntaxFixture, syntaxFixtures } from '../../core/tests/helpers.js'; +import { findingsIn, position, type PositionedFinding } from '../src/findings.js'; + +let v26: Schema; +beforeAll(async () => { + v26 = await schema('26.1.0'); +}); + +const corpus = syntaxFixtures(); + +/** What the two runs produce on their own, which is what positioning has to account for exactly. */ +function findingCount(text: string, against: Schema): number { + const { document, diagnostics } = parseIdf(text, against, { strict: false }); + return diagnostics.length + validateDocument(document).totalIssues; +} + +/** + * SC-004: every finding either run produces about model text carries a region, and none is dropped. + * + * Counted against the two runs made separately rather than against a number written here, so a + * finding added to either library is held to this without anybody remembering to update a total. + */ +describe('every finding carries a region', () => { + it.each(corpus)('positions all of $name', ({ text }) => { + const positioned = findingsIn(text, v26); + + expect(positioned).toHaveLength(findingCount(text, v26)); + for (const finding of positioned) { + expect(finding.region.start).toBeGreaterThanOrEqual(0); + expect(finding.region.end).toBeLessThanOrEqual(text.length); + expect(finding.region.start).toBeLessThanOrEqual(finding.region.end); + expect(finding.precision === 'field' || finding.precision === 'statement').toBe(true); + } + }); + + it('finds something to position, so the assertion above can fail', () => { + const total = corpus.reduce((sum, { text }) => sum + findingsIn(text, v26).length, 0); + expect(total).toBeGreaterThan(0); + }); + + it('gives every corpus finding a region the layer itself names', () => { + // Three regions are the only ones a placement may produce: a statement, its type name, or one of + // its written fields. The fallback for a finding nothing in the text answers to is an empty + // region at offset zero, which is none of them; it exists so SC-004 cannot be satisfied by + // dropping the hard cases, and a fixture that reaches it is a correlation failure wearing a + // region rather than a positioned finding. + for (const { name, text } of corpus) { + const named = new Set(); + for (const statement of scanIdf(text).statements) { + for (const region of [statement.region, statement.typeName, ...statement.fields]) { + named.add(`${region.start}:${region.end}`); + } + } + + for (const finding of findingsIn(text, v26)) { + const key = `${finding.region.start}:${finding.region.end}`; + expect(named.has(key), `${name}.idf placed "${finding.message}" at ${key}`).toBe(true); + } + } + }); +}); + +/** + * SC-005 and FR-013: the region selects the offending value and nothing beside it. + * + * Sliced rather than compared against offsets, because a pair of offsets that is wrong by one is + * unreadable in a failure message and a slice that is wrong by one is obvious. + */ +describe('a field region selects the value exactly', () => { + /** The finding a caller would act on, found by its code and its field rather than by position. */ + function findingFor(text: string, code: string, field: string) { + const found = findingsIn(text, v26).find( + (finding) => finding.code === code && 'field' in finding && finding.field === field + ); + if (found === undefined) throw new Error(`no ${code} finding on ${field}`); + return found; + } + + it('selects a fixed field written beside its comment', () => { + const text = [ + 'Version, 26.1;', + '', + 'Material,', + ' IN46, !- Name', + ' VeryRough, !- Roughness', + ' NotANumber, !- Thickness {m}', + ' 2.3, !- Conductivity {W/m-K}', + ' 1000, !- Density', + ' 900; !- Specific Heat', + '', + ].join('\n'); + + const finding = findingFor(text, 'E003', 'thickness'); + + expect(text.slice(finding.region.start, finding.region.end)).toBe('NotANumber'); + expect(finding.precision).toBe('field'); + }); + + it('selects a field written alone on its line', () => { + const text = ['Version, 26.1;', '', 'Zone,', ' Zone One,', ' NotANumber,', ' 0.0;', ''].join( + '\n' + ); + + const finding = findingFor(text, 'E003', 'direction_of_relative_north'); + + expect(text.slice(finding.region.start, finding.region.end)).toBe('NotANumber'); + expect(finding.precision).toBe('field'); + }); + + it('selects a field separated from its statement by comments', () => { + // The comments carry a comma and a semicolon, neither of which is a delimiter here, so a region + // computed by counting separators without stepping over comments lands on the wrong field. + const text = [ + 'Version, 26.1;', + '', + 'Zone,', + ' Zone One,', + ' !- the separator above and the value below are three lines apart,', + ' !- and these two comments sit between them;', + ' NotANumber,', + ' 0.0;', + '', + ].join('\n'); + + const finding = findingFor(text, 'E003', 'direction_of_relative_north'); + + expect(text.slice(finding.region.start, finding.region.end)).toBe('NotANumber'); + expect(finding.precision).toBe('field'); + }); + + /** + * The case the extensible arithmetic exists for, and the one that fails quietly when it is wrong. + * + * `fixedCount + repeat * groupWidth + offsetWithinGroup` on a surface is `11 + 8 * 3 + 0`, so the + * ninth vertex's x coordinate is field 35 of the statement. Getting it wrong by one group puts the + * underline on a neighbouring vertex, which looks plausible in a screenshot and is useless in an + * editor, so the first vertex is asserted beside the ninth: a placement that ignored the repeat + * would satisfy one of the two and not both. + * + * The findings are built here rather than taken from a run, because no code path in `@idfkit/core` + * reports a value inside an extensible group today: `validateDocument` skips the extensible key, + * and `parseIdf` collects its unreadable-value findings from the fixed fields alone. What is under + * test is the arithmetic that positions such a finding, which is reached the moment either + * producer starts making one. + */ + describe('a field late in an extensible group', () => { + const text = syntaxFixture('surface-bad-ninth-vertex'); + + /** A `ValidationError` carrying the repeat its field belongs to, which the type cannot say. */ + function vertexFinding(index: number): ValidationError & { readonly index: number } { + return { + severity: 'error', + objType: 'BuildingSurface:Detailed', + objName: 'South Wall', + field: 'vertex_x_coordinate', + message: 'Expected number, got string', + code: 'E003', + index, + }; + } + + it('selects the ninth vertex, not the first', () => { + const [ninth] = position([vertexFinding(8)], scanIdf(text), v26); + + expect(text.slice(ninth?.region.start, ninth?.region.end)).toBe('not-a-number'); + expect(ninth?.precision).toBe('field'); + }); + + it('selects the first vertex when the finding is about the first', () => { + const [first] = position([vertexFinding(0)], scanIdf(text), v26); + + expect(text.slice(first?.region.start, first?.region.end)).toBe('0.0'); + expect(first?.precision).toBe('field'); + }); + + it('falls back to the statement when the finding cannot say which repeat', () => { + const { index: _repeat, ...withoutRepeat } = vertexFinding(8); + const [placed] = position([withoutRepeat], scanIdf(text), v26); + + expect(placed?.precision).toBe('statement'); + expect(text.slice(placed?.region.start, placed?.region.end)).toMatch( + /^BuildingSurface:Detailed,/ + ); + }); + }); +}); + +/** + * FR-012: a finding about a statement selects the statement, and a fallback says it was taken. + * + * The line and column are the ones the finding already reports, which matters beyond tidiness: the + * conformance corpus compares findings across the two libraries on `(code, line, typeName)`, so a + * region that derived a different line would move a number a gate is watching. + */ +describe('a statement finding selects the statement', () => { + /** Reading findings alone, which are the ones that name a place rather than an object. */ + function readingFindings(text: string): readonly PositionedFinding[] { + return findingsIn(text, v26).filter( + (finding): finding is PositionedFinding => 'line' in finding + ); + } + + it('selects the type name of the duplicate, at the line and column already reported', () => { + const text = syntaxFixture('duplicate-object-name'); + const layer = scanIdf(text); + + const duplicate = readingFindings(text).find((finding) => finding.code === 'ParseError'); + + expect(duplicate).toBeDefined(); + expect(text.slice(duplicate?.region.start, duplicate?.region.end)).toBe('Zone'); + expect(duplicate?.precision).toBe('statement'); + // The second Zone and not the first: the finding is about the statement the parse refused, + // while the first is the one the document kept. Correlating this one by type and name would + // have underlined the wrong object, which is why a reading finding is never correlated. + expect(lineColumnAt(layer, duplicate!.region.start)).toEqual({ + line: duplicate?.line, + column: duplicate?.column, + }); + expect(lineColumnAt(layer, duplicate!.region.start)).toEqual({ line: 7, column: 1 }); + }); + + it('selects the type name of an unknown type, at the line and column already reported', () => { + const text = syntaxFixture('unknown-object-type'); + const layer = scanIdf(text); + + const unknown = readingFindings(text).find((finding) => finding.code === 'UnknownObjectType'); + + expect(unknown).toBeDefined(); + expect(text.slice(unknown?.region.start, unknown?.region.end)).toBe('NotAnObject:AtAll'); + expect(unknown?.precision).toBe('statement'); + expect(lineColumnAt(layer, unknown!.region.start)).toEqual({ + line: unknown?.line, + column: unknown?.column, + }); + }); + + it('falls back to the whole statement for a field that was never written, and says so', () => { + // Required fields the statement stops short of. There is no text to select, so the statement + // stands in and `precision` records that the fallback was taken. + const text = ['Version, 26.1;', '', 'Material,', ' IN46;', ''].join('\n'); + + const missing = findingsIn(text, v26).filter((finding) => finding.code === 'E001'); + + expect(missing.length).toBeGreaterThan(0); + for (const finding of missing) { + expect(finding.precision).toBe('statement'); + expect(text.slice(finding.region.start, finding.region.end)).toBe('Material,\n IN46;'); + } + }); +}); diff --git a/packages/language/tsconfig.json b/packages/language/tsconfig.json new file mode 100644 index 0000000..08d217c --- /dev/null +++ b/packages/language/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../core" }] +} diff --git a/packages/language/typedoc.json b/packages/language/typedoc.json new file mode 100644 index 0000000..ee51c20 --- /dev/null +++ b/packages/language/typedoc.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "extends": ["../../typedoc.base.json"], + "entryPoints": ["src/index.ts"], + "tsconfig": "tsconfig.json" +} diff --git a/scripts/check-absent-component.mjs b/scripts/check-absent-component.mjs index 7c58f57..d547d9c 100644 --- a/scripts/check-absent-component.mjs +++ b/scripts/check-absent-component.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node /** - * The absent-component gate (task T099a, FR-074, SC-031). + * The absent-component gate (task T099a, FR-074, SC-031; task T031, FR-046). * * THE CRITERION * @@ -10,50 +10,71 @@ * that has to be true: "`idfkit/weather` stays in the export map and resolves * once the peer is installed." * - * So three fixtures, not two. The two the task names establish the failure and + * So three claims, not two. The two the task names establish the failure and * the non-failure; the third is the control that stops both of them from being * satisfied by a subpath that is simply broken. A gate with only the first two * passes if `idfkit/weather` never works at all, which is not the design. * - * weather-absent installs `idfkit` alone, imports `idfkit/weather`, and - * requires the failure to NAME `npm install @idfkit/weather` - * core-only installs `idfkit` alone, imports `idfkit` and - * `idfkit/node` and never `idfkit/weather`, and requires - * `tsc --noEmit` to report nothing and the program to run. - * It reaches no scoped name at all, which is the other - * half of what the facade is for (FR-036) - * weather-present installs `idfkit` and the peer, imports `idfkit/weather`, - * and requires it to work + * FR-046 makes the same three claims about `idfkit/language`, in the same + * words, because `@idfkit/language` is an optional peer for the same reason: + * "Reaching the service through the shared name MUST work once its component is + * installed, and MUST fail with a message naming the component to install when + * it is not. A project that never reaches for it MUST build and type-check + * cleanly with the component absent." * - * WHY THE FIRST ONE IS NOT AUTOMATIC + * Hence the component table below, rather than a second copy of this file. Two + * components, three claims each, five fixtures: * - * The obvious implementation of the subpath, `export * from '@idfkit/weather'`, + * weather-absent installs `idfkit` alone, imports `idfkit/weather`, and + * requires the failure to NAME `npm install @idfkit/weather` + * language-absent the same, for `idfkit/language` + * core-only installs `idfkit` alone, imports `idfkit` and + * `idfkit/node` and NEITHER opt-in subpath, and requires + * `tsc --noEmit` to report nothing and the program to run. + * It reaches no scoped name at all, which is the other + * half of what the facade is for (FR-036) + * weather-present installs `idfkit` and the peer, imports `idfkit/weather`, + * and requires it to work + * language-present the same, for `idfkit/language` + * + * Five fixtures and not six, because the middle claim is one claim. It is about + * a project that imports NEITHER subpath, and `core-only` is already that + * project: both peers are absent from it, and it proves both absences by asking + * Node to resolve each and requiring each to fail. A second fixture installing + * the same manifest and compiling the same file would measure the same thing + * twice and cost another `npm install` to do it. + * + * WHY THE FIRST CLAIM IS NOT AUTOMATIC + * + * The obvious implementation of a subpath, `export * from '@idfkit/weather'`, * cannot produce a named failure. Static re-exports are resolved and linked * before any module in the graph is evaluated, so no code in the shim ever * runs and Node reports a bare `ERR_MODULE_NOT_FOUND` naming a file path inside * `node_modules/idfkit`. A reader seeing that has been handed the internals of - * a package they did not install and no instruction. `weather.js` therefore - * uses a caught dynamic import, and this gate is what keeps it that way: - * FAILING IS NOT ENOUGH, and the gate rejects a bare resolution error as - * explicitly as it rejects a success. + * a package they did not install and no instruction. `weather.js` and + * `language.js` therefore use a caught dynamic import, and this gate is what + * keeps them that way: FAILING IS NOT ENOUGH, and the gate rejects a bare + * resolution error as explicitly as it rejects a success. * - * WHY THE SECOND ONE IS NOT AUTOMATIC EITHER + * WHY THE SECOND CLAIM IS NOT AUTOMATIC EITHER * - * `weather.d.ts` is `export * from '@idfkit/weather'`, a declaration file - * referring to a package that is deliberately not installed. Whether that - * poisons an unrelated project depends on whether TypeScript reads the file, - * which depends on the module resolution mode and on nothing being configured - * to pull the whole package's types in. Under `nodenext` it reads only the - * subpath that is imported, so a project importing `idfkit` alone never sees - * it. That is SC-031, and it is a property of a TypeScript version and a + * `weather.d.ts` and `language.d.ts` are each `export * from` a package that is + * deliberately not installed. Whether that poisons an unrelated project depends + * on whether TypeScript reads the file, which depends on the module resolution + * mode and on nothing being configured to pull the whole package's types in. + * Under `nodenext` it reads only the subpath that is imported, so a project + * importing `idfkit` alone never sees either. That is SC-031 and the last + * sentence of FR-046, and it is a property of a TypeScript version and a * tsconfig rather than of anything in this repository, which is exactly why it - * is checked by running `tsc` rather than by reasoning about it. + * is checked by running `tsc` rather than by reasoning about it. Two + * unresolvable declaration files are also strictly more exposure than one: a + * resolution mode that read them eagerly would now poison the project twice. * * `tsc` comes from this repository's own toolchain, run against the fixture's * tsconfig. TypeScript resolves modules from the FILE it is compiling, so the * fixture's `node_modules` is what it reads; nothing of this workspace leaks in. * - * Exit codes: 0 all three hold, 1 at least one does not, 2 could not run. + * Exit codes: 0 all of it holds, 1 at least one claim does not, 2 could not run. */ import { execFileSync } from 'node:child_process'; @@ -65,6 +86,7 @@ import { CannotRun, FACADE, Finding, + LANGUAGE, REPO, WEATHER, fixtureRoot, @@ -77,13 +99,15 @@ import { writeJson, } from './lib/clean-install.mjs'; -/** The install the failure must name. Same constant the facade gate pins. */ -const INSTALL_COMMAND = `npm install ${WEATHER}`; -/** Importing the subpath with no peer. Prints the error rather than dying on it. */ -const ABSENT = ` +/** + * Importing an opt-in subpath with no peer. Prints the error rather than dying + * on it, because the message is the thing being checked. + */ +function absentProgram(subpath) { + return ` try { - await import('${FACADE}/weather'); + await import('${FACADE}/${subpath}'); console.log('IMPORTED'); } catch (error) { console.log('THREW ' + error?.constructor?.name + ' ' + (error?.code ?? '-')); @@ -91,24 +115,80 @@ try { console.log('CAUSE ' + JSON.stringify(String(error?.cause?.code ?? ''))); } `; +} -/** Importing the subpath with the peer installed. */ -const PRESENT = ` -const weather = await import('${FACADE}/weather'); -const direct = await import('${WEATHER}'); -const missing = Object.keys(direct).filter((name) => weather[name] === undefined); +/** + * Importing an opt-in subpath with the peer installed. + * + * Two halves, and both matter. The generic half compares the subpath's runtime + * names against the peer's own, which is what catches a hand-written re-export + * list that has fallen behind. The `probe` half calls something, because a + * subpath every one of whose names is `undefined` would satisfy the first half + * perfectly. + */ +function presentProgram(peer, subpath, probe) { + return ` +const viaFacade = await import('${FACADE}/${subpath}'); +const direct = await import('${peer}'); +const missing = Object.keys(direct).filter((name) => viaFacade[name] === undefined); if (missing.length > 0) { - console.error('idfkit/weather is missing ' + missing.join(', ')); + console.error('${FACADE}/${subpath} is missing ' + missing.join(', ')); process.exit(1); } -if (typeof weather.haversineKm !== 'function') { +${probe} +`; +} + +const WEATHER_PROBE = ` +if (typeof viaFacade.haversineKm !== 'function') { console.error('haversineKm is not a function through the subpath'); process.exit(1); } -const km = weather.haversineKm(45.5, -73.6, 45.5, -73.5); +const km = viaFacade.haversineKm(45.5, -73.6, 45.5, -73.5); console.log('OK ' + Object.keys(direct).length + ' names, haversineKm -> ' + km.toFixed(3)); `; +// contextAt with no schema, which the contract says is a supported call: without +// one the context still reports where the offset is. The returned value is +// checked for not being a promise, because the whole design claim of +// @idfkit/language is that its answers are synchronous (FR-024), and an +// asynchronous module graph behind the facade is exactly where that could be +// lost without anyone noticing. +const LANGUAGE_PROBE = ` +if (typeof viaFacade.contextAt !== 'function') { + console.error('contextAt is not a function through the subpath'); + process.exit(1); +} +const context = viaFacade.contextAt('Version,26.1;\\n', 3); +if (context === null || typeof context !== 'object' || typeof context.then === 'function') { + console.error('contextAt did not return a synchronous context: ' + JSON.stringify(context)); + process.exit(1); +} +console.log('OK ' + Object.keys(direct).length + ' names, contextAt -> at ' + context.at); +`; + +/** + * The two opt-in components, and what each of the three claims means for each. + * + * `install` is the exact command the failure must name, and it is the same + * string `check-facade.mjs` pins: the two gates would otherwise agree on the + * requirement and disagree on the text. + */ +const COMPONENTS = [ + { + peer: WEATHER, + subpath: 'weather', + install: `npm install ${WEATHER}`, + probe: WEATHER_PROBE, + }, + { + peer: LANGUAGE, + subpath: 'language', + install: `npm install ${LANGUAGE}`, + probe: LANGUAGE_PROBE, + }, +]; + /** * A project that uses the library and never mentions weather. * @@ -199,68 +279,93 @@ async function main() { const { tarballs } = packWorkspaces(scratch.tarballDir); const findings = []; - // ---- 1. weather-absent: the failure names the install ------------------ - const absent = installSharedNameOrFail(scratch, tarballs, { label: 'weather-absent' }); - if (resolveFrom(absent.dir, WEATHER).resolved !== null) { - throw new CannotRun( - `${WEATHER} resolves from the weather-absent fixture, so it is not absent and nothing ` + - 'this fixture reports is about the case FR-074 describes.' - ); - } - const absentRun = runInFixture(absent.dir, 'app.mjs', ABSENT); - const threw = absentRun.stdout.includes('THREW'); - const message = JSON.parse((absentRun.stdout.match(/^MESSAGE (.*)$/m) ?? ['', '""'])[1]); - const cause = JSON.parse((absentRun.stdout.match(/^CAUSE (.*)$/m) ?? ['', '""'])[1]); - const code = (absentRun.stdout.match(/^THREW \S+ (\S+)$/m) ?? [])[1]; + // ---- 1. one absent fixture per component: the failure names the install + const absences = []; + for (const component of COMPONENTS) { + const label = `${component.subpath}-absent`; + const fixture = installSharedNameOrFail(scratch, tarballs, { label }); - if (absentRun.code !== 0) { - findings.push( - new Finding( - 'the weather-absent fixture could not even report the failure', - `Exit ${absentRun.code}: ${(absentRun.stderr || absentRun.stdout).trim().split('\n').slice(-3).join(' ')}` - ) - ); - } else if (!threw) { - findings.push( - new Finding( - `importing ${FACADE}/weather succeeded with ${WEATHER} not installed`, - 'Either the peer is being auto-installed after all, which is FR-043, or the facade has ' + - 'grown an implementation of its own, which it must not have (FR-037).' - ) - ); - } else { - if (!message.includes(INSTALL_COMMAND)) { + // Positive proof of absence. An absence you did not try to resolve is an + // absence you did not check, and this one produced a false green once. + if (resolveFrom(fixture.dir, component.peer).resolved !== null) { + throw new CannotRun( + `${component.peer} resolves from the ${label} fixture, so it is not absent and ` + + 'nothing this fixture reports is about the case FR-074 and FR-046 describe.' + ); + } + + const ran = runInFixture(fixture.dir, 'app.mjs', absentProgram(component.subpath)); + const threw = ran.stdout.includes('THREW'); + const message = JSON.parse((ran.stdout.match(/^MESSAGE (.*)$/m) ?? ['', '""'])[1]); + const cause = JSON.parse((ran.stdout.match(/^CAUSE (.*)$/m) ?? ['', '""'])[1]); + const code = (ran.stdout.match(/^THREW \S+ (\S+)$/m) ?? [])[1]; + absences.push({ component, ran, threw, message, cause, code }); + + const subpath = `${FACADE}/${component.subpath}`; + if (ran.code !== 0) { + findings.push( + new Finding( + `the ${label} fixture could not even report the failure`, + `Exit ${ran.code}: ${(ran.stderr || ran.stdout).trim().split('\n').slice(-3).join(' ')}` + ) + ); + continue; + } + if (!threw) { + findings.push( + new Finding( + `importing ${subpath} succeeded with ${component.peer} not installed`, + 'Either the peer is being auto-installed after all, which is FR-043, or the facade ' + + 'has grown an implementation of its own, which it must not have (FR-037).' + ) + ); + continue; + } + if (!message.includes(component.install)) { findings.push( new Finding( - `importing ${FACADE}/weather fails without naming "${INSTALL_COMMAND}"`, - `The message was: ${JSON.stringify(message.slice(0, 200))}. FR-074 requires the ` + - 'failure to name the component to install. A reader who gets a bare module error ' + - 'has been handed the internals of a package they never installed.' + `importing ${subpath} fails without naming "${component.install}"`, + `The message was: ${JSON.stringify(message.slice(0, 200))}. FR-074 and FR-046 ` + + 'require the failure to name the component to install. A reader who gets a bare ' + + 'module error has been handed the internals of a package they never installed.' ) ); } if (code === 'ERR_MODULE_NOT_FOUND') { findings.push( new Finding( - `importing ${FACADE}/weather raises a bare ERR_MODULE_NOT_FOUND`, - 'That is what a static `export * from "@idfkit/weather"` produces: static re-exports ' + - 'are linked before any code in weather.js runs, so the guard never executes. The ' + - 'shim has to use a caught dynamic import (FR-074).' + `importing ${subpath} raises a bare ERR_MODULE_NOT_FOUND`, + `That is what a static \`export * from "${component.peer}"\` produces: static ` + + 're-exports are linked before any code in the shim runs, so the guard never ' + + 'executes. The shim has to use a caught dynamic import (FR-074, FR-046).' ) ); } - if (!message.includes(WEATHER)) { + if (!message.includes(component.peer)) { findings.push( new Finding( - `the failure does not name ${WEATHER}`, + `the failure does not name ${component.peer}`, `The message was: ${JSON.stringify(message.slice(0, 200))}.` ) ); } } - // ---- 2. core-only: builds and type-checks clean ------------------------- + // ---- 2. core-only: builds and type-checks clean, with BOTH peers absent - + // + // One fixture, two components. The claim is about a project that imports + // neither opt-in subpath, and this is that project: it installs the shared + // name alone, so `weather.d.ts` and `language.d.ts` both sit in its + // node_modules pointing at packages that are not there. const coreOnly = installSharedNameOrFail(scratch, tarballs, { label: 'core-only' }); + for (const component of COMPONENTS) { + if (resolveFrom(coreOnly.dir, component.peer).resolved !== null) { + throw new CannotRun( + `${component.peer} resolves from the core-only fixture, so this fixture is not the ` + + 'project the criterion is about and its clean type-check would prove nothing.' + ); + } + } writeFileSync(join(coreOnly.dir, 'app.ts'), CORE_ONLY_TS); writeJson(join(coreOnly.dir, 'tsconfig.json'), CORE_ONLY_TSCONFIG); const types = typecheck(coreOnly.dir); @@ -268,12 +373,12 @@ async function main() { const diagnostics = types.output.trim().split('\n').filter(Boolean); findings.push( new Finding( - `a project that imports only ${FACADE} does not type-check with the peer absent`, + `a project that imports only ${FACADE} does not type-check with the peers absent`, `tsc reported ${diagnostics.length} diagnostic(s): ` + - `${diagnostics.slice(0, 5).join(' | ')}. SC-031: the facade's own weather.d.ts is ` + - "`export * from '@idfkit/weather'`, and TypeScript must only read it when something " + - 'imports the subpath. A project that never does must not pay for the peer being ' + - 'absent.' + `${diagnostics.slice(0, 5).join(' | ')}. SC-031 and FR-046: the facade's own ` + + 'weather.d.ts and language.d.ts are each `export * from` an absent package, and ' + + 'TypeScript must only read one when something imports its subpath. A project that ' + + 'never does must not pay for either peer being absent.' ) ); } @@ -283,57 +388,89 @@ async function main() { if (coreOnlyRun.code !== 0) { findings.push( new Finding( - `a project that imports only ${FACADE} does not run with the peer absent`, + `a project that imports only ${FACADE} does not run with the peers absent`, `Exit ${coreOnlyRun.code}: ${(coreOnlyRun.stderr || coreOnlyRun.stdout).trim().split('\n').slice(-3).join(' ')}` ) ); } - // ---- 3. weather-present: the control ----------------------------------- - const present = installSharedNameOrFail(scratch, tarballs, { - label: 'weather-present', - also: [WEATHER], - }); - const presentRun = runInFixture(present.dir, 'app.mjs', PRESENT); - if (presentRun.code !== 0) { - findings.push( - new Finding( - `${FACADE}/weather does not work once ${WEATHER} is installed`, - `Exit ${presentRun.code}: ${(presentRun.stderr || presentRun.stdout).trim().split('\n').slice(-3).join(' ')}. ` + - 'Clause 4 of contracts/distribution.md: the subpath stays in the export map and ' + - 'resolves once the peer is installed. Without this control the two fixtures above ' + - 'would both pass on a subpath that is simply broken.' - ) + // ---- 3. one present fixture per component: the control ------------------ + const presences = []; + for (const component of COMPONENTS) { + const label = `${component.subpath}-present`; + const fixture = installSharedNameOrFail(scratch, tarballs, { + label, + also: [component.peer], + }); + const ran = runInFixture( + fixture.dir, + 'app.mjs', + presentProgram(component.peer, component.subpath, component.probe) ); + presences.push({ component, ran }); + if (ran.code !== 0) { + findings.push( + new Finding( + `${FACADE}/${component.subpath} does not work once ${component.peer} is installed`, + `Exit ${ran.code}: ${(ran.stderr || ran.stdout).trim().split('\n').slice(-3).join(' ')}. ` + + 'Clause 4 of contracts/distribution.md, and FR-046: the subpath stays in the ' + + 'export map and resolves once the peer is installed. Without this control the ' + + 'fixtures above would both pass on a subpath that is simply broken.' + ) + ); + } } - console.log('idfkit-js absent-component gate (FR-074, SC-031)'); + console.log('idfkit-js absent-component gate (FR-074, FR-046, SC-031)'); console.log(` fixtures ${scratch.root}/fixtures`); console.log(''); - console.log(' 1. weather-absent npm install idfkit, then import idfkit/weather'); - console.log(` threw ${threw ? `yes (code ${code ?? '-'}, cause ${cause || '-'})` : 'NO, it succeeded'}`); - console.log(` names install ${message.includes(INSTALL_COMMAND) ? `yes: "${INSTALL_COMMAND}"` : 'NO'}`); - console.log(` first line ${message.split('\n')[0].slice(0, 96)}`); - console.log(''); - console.log(' 2. core-only npm install idfkit; imports idfkit and idfkit/node only'); + let step = 0; + for (const { component, threw, message, cause, code } of absences) { + step += 1; + const subpath = `${FACADE}/${component.subpath}`; + console.log( + ` ${step}. ${component.subpath}-absent`.padEnd(22) + + `npm install ${FACADE}, then import ${subpath}` + ); + console.log( + ` threw ${threw ? `yes (code ${code ?? '-'}, cause ${cause || '-'})` : 'NO, it succeeded'}` + ); + console.log( + ` names install ${message.includes(component.install) ? `yes: "${component.install}"` : 'NO'}` + ); + console.log(` first line ${message.split('\n')[0].slice(0, 96)}`); + console.log(''); + } + step += 1; console.log( - ` tsc --noEmit ${types.code === 0 ? 'clean' : `${types.output.trim().split('\n').length} diagnostic(s)`} (module nodenext, strict, skipLibCheck off)` + ` ${step}. core-only`.padEnd(22) + + `npm install ${FACADE}; imports ${FACADE} and ${FACADE}/node only` ); console.log( - ` runs ${coreOnlyRun.code === 0 ? `yes: ${coreOnlyRun.stdout.trim().slice(3, 76)}` : 'NO'}` + ` tsc --noEmit ${types.code === 0 ? 'clean' : `${types.output.trim().split('\n').length} diagnostic(s)`} (module nodenext, strict, skipLibCheck off)` ); - console.log(''); - console.log(' 3. weather-present npm install idfkit @idfkit/weather (the control)'); console.log( - ` subpath works ${presentRun.code === 0 ? `yes: ${presentRun.stdout.trim().slice(3, 76)}` : 'NO'}` + ` runs ${coreOnlyRun.code === 0 ? `yes: ${coreOnlyRun.stdout.trim().slice(3, 76)}` : 'NO'}` ); console.log(''); + for (const { component, ran } of presences) { + step += 1; + console.log( + ` ${step}. ${component.subpath}-present`.padEnd(22) + + `npm install ${FACADE} ${component.peer} (the control)` + ); + console.log( + ` subpath works ${ran.code === 0 ? `yes: ${ran.stdout.trim().slice(3, 76)}` : 'NO'}` + ); + console.log(''); + } return verdict( findings, - `importing ${FACADE}/weather without the peer names the install, a project that never ` + - 'imports it type-checks and runs clean, and the subpath works once the peer is added.', - 'the absent opt-in component does not behave as FR-074 and SC-031 require.' + `importing an absent opt-in subpath of ${FACADE} names the install, a project that ` + + 'imports neither type-checks and runs clean, and each subpath works once its peer is ' + + 'added.', + 'an absent opt-in component does not behave as FR-074, FR-046 and SC-031 require.' ); } finally { scratch.dispose(); diff --git a/scripts/check-facade.mjs b/scripts/check-facade.mjs index ad94d32..06aebda 100644 --- a/scripts/check-facade.mjs +++ b/scripts/check-facade.mjs @@ -4,7 +4,7 @@ * * WHAT THE FACADE IS * - * `packages/idfkit/` contains no implementation. It is a manifest and eight + * `packages/idfkit/` contains no implementation. It is a manifest and ten * one-line-ish re-export files, so that `npm install idfkit` gives a working * library without the reader ever learning the scoped names, while the scoped * packages stay published and stay the real implementations (FR-036, FR-037). @@ -21,15 +21,17 @@ * install idfkit` succeeds, `import 'idfkit/results'` fails at run time, * and the failure lands on the reader rather than on CI. The export map is * therefore pinned to exactly the four entries in - * `contracts/distribution.md`, and every one of them must resolve to a - * package that is really there. + * `contracts/distribution.md` plus the `./language` entry added by + * `005-idf-language-service/contracts/language-service.md`, and every one + * of them must resolve to a package that is really there. * - * 2. WEATHER STOPS BEING OPT-IN (FR-043, SC-016). Moving `@idfkit/weather` - * from `peerDependencies` into `dependencies` is a one-word edit that no - * test would notice, and it puts a 1.6 MB station index on disk for every - * reader who never asked for weather. `optionalDependencies` is not the - * mechanism either despite the name: npm installs those by default and - * merely tolerates failure. + * 2. AN OPT-IN COMPONENT STOPS BEING OPT-IN (FR-043, SC-016, SC-015). Moving + * `@idfkit/weather` or `@idfkit/language` from `peerDependencies` into + * `dependencies` is a one-word edit that no test would notice, and it puts + * a 1.6 MB station index, or a language service nobody without an editor + * can use, on disk for every reader who never asked for either. + * `optionalDependencies` is not the mechanism either despite the name: npm + * installs those by default and merely tolerates failure. * * 3. THE ENGINE ARRIVES (FR-070). `@idfkit/engine-assets` is 51 MB of * WebAssembly and datasets, and it versions on the EnergyPlus release it @@ -37,20 +39,20 @@ * helpful `./engine` subpath, breaks the install-size and bundle-purity * criteria at a stroke and pins every facade user to one engine version. * - * 4. THE WEATHER SHIM DRIFTS (FR-074). `weather.js` cannot use `export *`, - * because the whole point of it is to catch the missing-peer failure and - * name the install, and a static re-export is linked before any code in it - * runs. So it writes the names out. A name added to `@idfkit/weather` then - * exists under `@idfkit/weather` and not under `idfkit/weather`, with the - * types insisting otherwise, and nothing says so. This gate reads both - * surfaces and fails on any difference. + * 4. AN OPT-IN SHIM DRIFTS (FR-074, FR-046). `weather.js` and `language.js` + * cannot use `export *`, because the whole point of them is to catch the + * missing-peer failure and name the install, and a static re-export is + * linked before any code in them runs. So they write the names out. A name + * added to `@idfkit/weather` then exists under `@idfkit/weather` and not + * under `idfkit/weather`, with the types insisting otherwise, and nothing + * says so. This gate reads both surfaces and fails on any difference. * * WHAT IT READS * * packages/idfkit/package.json the export map, the dependency shape * packages/idfkit/*.js, *.d.ts the specifier each subpath re-exports * node_modules, walking up whether each specifier resolves - * @idfkit/weather its real runtime exports, for the drift check + * each optional peer its real runtime exports, for the drift check * * Resolution is done by hand rather than through `import.meta.resolve` so a * failure can say which package was missing and where it looked, and so the @@ -73,14 +75,16 @@ const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const FACADE = join(REPO, 'packages', 'idfkit'); /** - * The export map, verbatim from `contracts/distribution.md`. + * The export map: `contracts/distribution.md` verbatim, plus `./language` from + * `005-idf-language-service/contracts/language-service.md`. * - * Subpath -> the specifier it must re-export. Both halves are pinned: a fifth - * entry is a dead subpath waiting to happen, and a fourth entry pointing + * Subpath -> the specifier it must re-export. Both halves are pinned: a sixth + * entry is a dead subpath waiting to happen, and one of these five pointing * somewhere else is a silent rename of the public surface. */ const CONTRACTED = new Map([ ['.', '@idfkit/core'], + ['./language', '@idfkit/language'], ['./node', '@idfkit/core/node'], ['./schemas', '@idfkit/schemas'], ['./weather', '@idfkit/weather'], @@ -89,15 +93,37 @@ const CONTRACTED = new Map([ /** Plain `dependencies`, always installed, never optional (FR-042, T094). */ const REQUIRED_DEPENDENCIES = ['@idfkit/core', '@idfkit/schemas']; -/** The one opt-in component, and the only legal `peerDependencies` entry. */ -const OPTIONAL_PEER = '@idfkit/weather'; +/** + * The opt-in components, and the only legal `peerDependencies` entries. + * + * Each is a package the shared name reaches through a guarded shim rather than + * a static re-export, so each carries the two things this gate checks about a + * shim: the install it must name, and the file whose written-out names are held + * against the package's real surface. + */ +const OPTIONAL_PEERS = [ + { + name: '@idfkit/weather', + file: 'weather.js', + install: 'npm install @idfkit/weather', + why: + 'This is what keeps the 1.6 MB station index off disk under the shared name ' + + '(FR-043, SC-016).', + }, + { + name: '@idfkit/language', + file: 'language.js', + install: 'npm install @idfkit/language', + why: + 'This is what keeps the language service off disk for the readers who never open an ' + + 'editor, which is what leaves the install-size budget satisfied with no amendment ' + + '(FR-046, SC-015).', + }, +]; /** Not a subpath, not a dependency, not an optional peer (FR-070). */ const BANNED = ['@idfkit/engine', '@idfkit/engine-assets']; -/** The install `idfkit/weather` must name when the peer is absent (FR-074). */ -const INSTALL_COMMAND = 'npm install @idfkit/weather'; - /** Install-time scripting, rejected outright (FR-042, SC-015). */ const INSTALL_SCRIPTS = ['preinstall', 'install', 'postinstall', 'prepare']; @@ -209,8 +235,8 @@ function checkExportMap(manifest, findings) { findings.push( new Finding( `the export map has ${extra.length} entry beyond the contract: ${extra.join(', ')}`, - 'contracts/distribution.md pins the facade to exactly ' + - `${contracted.join(', ')}. A fifth subpath is how a name reserved in the register ` + + 'contracts/distribution.md, plus the ./language entry, pins the facade to exactly ' + + `${contracted.join(', ')}. A sixth subpath is how a name reserved in the register ` + 'leaks into the published surface as a subpath that resolves to nothing (FR-077).' ) ); @@ -219,7 +245,7 @@ function checkExportMap(manifest, findings) { findings.push( new Finding( `the export map is missing ${missing.join(', ')}`, - 'All four subpaths are mandatory. A flat facade with one eager entry point drags the ' + + 'All five subpaths are mandatory. A flat facade with one eager entry point drags the ' + 'schema data and the station index into every browser bundle (FR-038).' ) ); @@ -261,8 +287,9 @@ function checkSubpathTargets(manifest, findings) { ); } - const specifiers = new Set(reexportedSpecifiers(read(path))); - for (const match of read(path).matchAll(DYNAMIC_IMPORT)) specifiers.add(match[1]); + const source = read(path); + const specifiers = new Set(reexportedSpecifiers(source)); + for (const match of source.matchAll(DYNAMIC_IMPORT)) specifiers.add(match[1]); if (specifiers.size === 0) { findings.push( new Finding( @@ -363,38 +390,35 @@ function checkDependencyShape(manifest, findings) { } const peerNames = Object.keys(peers); - if (!peerNames.includes(OPTIONAL_PEER)) { - findings.push( - new Finding( - `${OPTIONAL_PEER} is not a "peerDependencies" entry`, - 'This is what keeps the 1.6 MB station index off disk under the shared name ' + - '(FR-043, SC-016).' - ) - ); - } else if (meta[OPTIONAL_PEER]?.optional !== true) { - findings.push( - new Finding( - `${OPTIONAL_PEER} is a peer dependency but is not marked optional`, - 'Without `"peerDependenciesMeta": { "@idfkit/weather": { "optional": true } }` npm 7+ ' + - 'auto-installs the peer, and the index is back on disk for everyone (FR-043).' - ) - ); - } - if (dependencies[OPTIONAL_PEER] !== undefined || optional[OPTIONAL_PEER] !== undefined) { - findings.push( - new Finding( - `${OPTIONAL_PEER} is also declared as a dependency`, - 'Then it installs, and the optional peer declaration means nothing.' - ) - ); + for (const peer of OPTIONAL_PEERS) { + if (!peerNames.includes(peer.name)) { + findings.push(new Finding(`${peer.name} is not a "peerDependencies" entry`, peer.why)); + } else if (meta[peer.name]?.optional !== true) { + findings.push( + new Finding( + `${peer.name} is a peer dependency but is not marked optional`, + `Without \`"peerDependenciesMeta": { "${peer.name}": { "optional": true } }\` npm 7+ ` + + 'auto-installs the peer, and it is back on disk for everyone (FR-043, FR-046).' + ) + ); + } + if (dependencies[peer.name] !== undefined || optional[peer.name] !== undefined) { + findings.push( + new Finding( + `${peer.name} is also declared as a dependency`, + 'Then it installs, and the optional peer declaration means nothing.' + ) + ); + } } - for (const extra of peerNames.filter((name) => name !== OPTIONAL_PEER)) { + const expectedPeers = OPTIONAL_PEERS.map((peer) => peer.name); + for (const extra of peerNames.filter((name) => !expectedPeers.includes(name))) { findings.push( new Finding( `${extra} is an unexpected peer dependency`, - 'Weather is the one opt-in component of the shared name. Anything else here is either ' + - 'a dependency the reader should not have to know about, or a component that needs ' + - 'its own decision.' + `Weather and the language service are the opt-in components of the shared name. ` + + 'Anything else here is either a dependency the reader should not have to know about, ' + + 'or a component that needs its own decision.' ) ); } @@ -471,34 +495,44 @@ function checkNoEngine(manifest, findings) { } /** - * The weather shim: it names the install, and its written-out names are the + * One opt-in shim: it names its install, and its written-out names are the * peer's real ones. + * + * The same three failures apply to every shim, which is why this is a loop over + * OPTIONAL_PEERS rather than one function per component. A second copy of this + * check written for the language subpath would be the place the two drift apart. + * + * Only runtime values are compared. Both `.d.ts` twins are the plain + * `export * from`, so the type-only names of a peer are re-exported whole and + * have nothing to write out here. */ -async function checkWeatherShim(findings) { - const path = join(FACADE, 'weather.js'); +async function checkShim(peer, findings) { + const path = join(FACADE, peer.file); if (!existsSync(path)) { - findings.push(new Finding('weather.js is missing', 'The ./weather subpath has no target.')); + findings.push( + new Finding(`${peer.file} is missing`, `The subpath re-exporting ${peer.name} has no target.`) + ); return; } const text = read(path); - if (!text.includes(INSTALL_COMMAND)) { + if (!text.includes(peer.install)) { findings.push( new Finding( - `weather.js does not name "${INSTALL_COMMAND}"`, - 'FR-074: importing an absent opt-in component must fail with a message naming the ' + - 'component to install, rather than a bare unresolved-module error.' + `${peer.file} does not name "${peer.install}"`, + 'FR-074 and FR-046: importing an absent opt-in component must fail with a message ' + + 'naming the component to install, rather than a bare unresolved-module error.' ) ); } + REEXPORT.lastIndex = 0; if (REEXPORT.test(text)) { - REEXPORT.lastIndex = 0; findings.push( new Finding( - 'weather.js uses a static re-export', - 'A static `export * from "@idfkit/weather"` is linked before any code in this file ' + + `${peer.file} uses a static re-export`, + `A static \`export * from "${peer.name}"\` is linked before any code in this file ` + 'runs, so the guard never executes and the reader gets ERR_MODULE_NOT_FOUND ' + - 'instead of the install command (FR-074).' + 'instead of the install command (FR-074, FR-046).' ) ); } @@ -506,14 +540,14 @@ async function checkWeatherShim(findings) { const written = new Set([...text.matchAll(NAMED_CONST)].map((match) => match[1])); if (written.size === 0) { - findings.push(new Finding('weather.js re-exports no names', 'The subpath would be empty.')); + findings.push(new Finding(`${peer.file} re-exports no names`, 'The subpath would be empty.')); return; } - const pkg = findPackage(OPTIONAL_PEER, FACADE); + const pkg = findPackage(peer.name, FACADE); if (pkg.manifest === null) { throw new CannotRun( - `${OPTIONAL_PEER} is not installed, so the shim's names cannot be checked against it. ` + + `${peer.name} is not installed, so the shim's names cannot be checked against it. ` + 'It is an optional peer for consumers and a workspace package here; run `npm install`.' ); } @@ -522,30 +556,31 @@ async function checkWeatherShim(findings) { real = new Set(Object.keys(await import(pathToFileURL(join(pkg.dir, 'dist/index.js')).href))); } catch (error) { throw new CannotRun( - `cannot load ${OPTIONAL_PEER} to read its exports: ${error.message}. Run \`npx tsc --build\`.` + `cannot load ${peer.name} to read its exports: ${error.message}. Run \`npx tsc --build\`.` ); } const dropped = [...real].filter((name) => !written.has(name)).sort(); const invented = [...written].filter((name) => !real.has(name)).sort(); + const types = peer.file.replace(/\.js$/, '.d.ts'); if (dropped.length > 0) { findings.push( new Finding( - `weather.js is missing ${dropped.length} of ${OPTIONAL_PEER}'s exports: ${dropped.join(', ')}`, - 'weather.d.ts re-exports the peer whole, so these type-check under idfkit/weather and ' + - 'are undefined at run time. Add them to weather.js.' + `${peer.file} is missing ${dropped.length} of ${peer.name}'s exports: ${dropped.join(', ')}`, + `${types} re-exports the peer whole, so these type-check under the subpath and are ` + + `undefined at run time. Add them to ${peer.file}.` ) ); } if (invented.length > 0) { findings.push( new Finding( - `weather.js exports ${invented.join(', ')}, which ${OPTIONAL_PEER} does not`, + `${peer.file} exports ${invented.join(', ')}, which ${peer.name} does not`, 'Each is undefined at run time.' ) ); } - return { written: written.size, real: real.size }; + return { peer, written: written.size, real: real.size }; } async function main() { @@ -559,7 +594,11 @@ async function main() { const resolved = checkSubpathTargets(manifest, findings); checkDependencyShape(manifest, findings); const tree = checkNoEngine(manifest, findings); - const shim = await checkWeatherShim(findings); + const shims = []; + for (const peer of OPTIONAL_PEERS) { + const shim = await checkShim(peer, findings); + if (shim !== undefined) shims.push(shim); + } console.log('idfkit-js facade gate'); console.log(` package ${manifest.name}@${manifest.version}`); @@ -575,13 +614,18 @@ async function main() { ); } console.log(` dependencies ${Object.keys(manifest.dependencies ?? {}).join(', ') || 'none'}`); - console.log( - ` optional peer ${Object.keys(manifest.peerDependencies ?? {}).join(', ') || 'none'}` + - ` (optional: ${manifest.peerDependenciesMeta?.[OPTIONAL_PEER]?.optional === true})` - ); + const peerNames = Object.keys(manifest.peerDependencies ?? {}); + console.log(peerNames.length === 0 ? ' optional peers none' : ' optional peers'); + for (const name of peerNames) { + console.log( + ` ${name.padEnd(20)} optional: ${manifest.peerDependenciesMeta?.[name]?.optional === true}` + ); + } console.log(` dep tree ${tree.size} packages, none of ${BANNED.join(', ')}`); - if (shim !== undefined) { - console.log(` weather shim ${shim.written} names re-exported, ${shim.real} in the peer`); + for (const shim of shims) { + console.log( + ` ${shim.peer.file.padEnd(12)} ${shim.written} names re-exported, ${shim.real} in the peer` + ); } console.log(''); diff --git a/scripts/check-install-size.mjs b/scripts/check-install-size.mjs index 49550e1..5f5f33d 100644 --- a/scripts/check-install-size.mjs +++ b/scripts/check-install-size.mjs @@ -11,8 +11,7 @@ * WHICH 1.75 MB, AND WHICH "ON DISK" * * Both halves of that sentence need pinning down, because the two readings of - * "on disk" differ by enough to have flipped the verdict under the previous - * budget, and will again once the package grows. + * "on disk" now disagree about the verdict rather than merely about the number. * * 1.75 MB is 1.75 MiB, 1,835,008 bytes. Every other size in the contract * is quoted the way npm quotes them, and npm's are binary. @@ -22,13 +21,13 @@ * tool built on the registry means. * * The alternative reading, allocated blocks, is the one `du` gives. Measured on - * 2026-09-03, across 141 files: + * 2026-09-04, across 164 files: * - * 1.33 MB apparent 76 percent of the budget - * 1.66 MB by du -sk 95 percent of the budget + * 1.71 MB apparent 97.9 percent of the budget + * 2.08 MB by du -sk 119 percent of the budget * - * The 346 KB between them is not weight in the package. It is the filesystem - * rounding 141 mostly-small files up to its allocation unit, 4 KB on the ext4 + * The 378 KB between them is not weight in the package. It is the filesystem + * rounding 164 mostly-small files up to its allocation unit, 4 KB on the ext4 * of a GitHub runner and on the APFS of a laptop. That number would move on a * tmpfs, on ZFS with compression, on a filesystem with tail packing, and on any * runner image that changes its storage driver. A criterion whose verdict @@ -36,15 +35,17 @@ * one the package can be engineered against either: the only way to improve an * allocation figure is to ship fewer, larger files, which is a worse package. * - * So the gate FAILS ON THE APPARENT FIGURE ONLY, and PRINTS BOTH. The - * pessimistic number is not hidden, because a gate that quietly picks the - * flattering measure is the thing this whole contract is written against. + * So the gate FAILS ON THE APPARENT FIGURE ONLY, and PRINTS BOTH. What `du` + * reports is printed for honesty and is never measured against. The pessimistic + * number is not hidden, because a gate that quietly picks the flattering + * measure is the thing this whole contract is written against. * - * Both readings sit under the budget today. They did not under the previous one, - * where the same install measured 88 percent apparent against 110 percent - * allocated, and they will not again once the schema prose lands. The choice of - * measure is therefore settled on the principle above rather than on which side - * of the line the two figures happen to fall this month. + * That distinction used to cost nothing here and now decides the outcome. At + * the previous measurement both readings sat inside the budget; the schema + * prose has since landed, and the allocated figure is now over it while the + * apparent one passes with 38 KB to spare. Which is why the choice of measure + * was settled on the principle above, at a moment when it changed no verdict, + * rather than now, when it decides one. * * WHY 1.75 MB, AND WHAT IT WAS BEFORE * @@ -69,20 +70,27 @@ * originally set with, and it keeps what SC-012 is for: a 4.3x reduction from * 7.9 MB, against 5.0x under the old figure. * - * HEADROOM, AND WHAT THE INCREASE IS FOR + * HEADROOM, AND WHAT THE INCREASE WAS FOR * - * 1.33 of 1.75 MB is 76 percent, about 434 KB of slack. That is not permission - * to spend it: roughly 190 KB is already promised to the schema prose, which - * will take the install to 86 percent and the slack to about 248 KB. The gate - * therefore does not just print PASS: it prints the percentage, the remaining - * headroom, and the per-package breakdown, so the number that matters is - * visible in the log of every run rather than only in the run that finally - * fails. + * The increase has been spent, on the thing it was raised for. 1.71 of 1.75 MB + * is 97.9 percent, about 38 KB of slack, with the prose in `@idfkit/schemas` + * and the syntax layer in `@idfkit/core`. The language service is not in this + * measurement at all: it is an optional peer, so it puts nothing on disk here, + * and the per-package breakdown is what would say otherwise, since a package + * the contract does not list is a finding whether or not the total passes. + * + * 38 KB is thin, and it is meant to be read that way. The gate therefore does + * not just print PASS: it prints the percentage, the remaining headroom, and + * the per-package breakdown, so the number that matters is visible in the log + * of every run rather than only in the run that finally fails. The next feature + * that wants to add weight to either installed package should expect to argue + * for it, and either amend SC-012 deliberately or move the weight to an opt-in + * component, which is the choice this budget exists to force. * * WHAT IS COUNTED * * Everything under the fixture's `node_modules`, including npm's own - * `.package-lock.json`. That file is roughly 1 KB and is genuinely on the + * `.package-lock.json`. That file is roughly 2 KB and is genuinely on the * reader's disk after the install, so excluding it would be a small lie in the * gate's favour. * diff --git a/scripts/lib/clean-install.mjs b/scripts/lib/clean-install.mjs index a2afa45..f5efa47 100644 --- a/scripts/lib/clean-install.mjs +++ b/scripts/lib/clean-install.mjs @@ -44,7 +44,7 @@ * * NO CACHING, DELIBERATELY * - * Packing all six workspace packages costs a couple of seconds and every gate + * Packing every workspace package costs a couple of seconds and every gate * pays it. A cache keyed on mtimes would be faster and would occasionally * measure a tree that no longer exists; a verification tool that reports on * stale evidence is worse than a slow one. @@ -456,6 +456,7 @@ export const FACADE = 'idfkit'; export const CORE = '@idfkit/core'; export const SCHEMAS = '@idfkit/schemas'; export const WEATHER = '@idfkit/weather'; +export const LANGUAGE = '@idfkit/language'; export const TYPE_PACKAGES = ['@idfkit/types-v26-1', '@idfkit/types-v9-4']; export const ENGINE = ['@idfkit/engine', '@idfkit/engine-assets']; @@ -479,7 +480,7 @@ export function installSharedName(scratch, tarballs, { label, also = [], flags = // Every scoped name the facade could ask for is redirected to a local // tarball, including the ones that must not appear. See fixtureManifest(). const overrides = {}; - for (const name of [CORE, SCHEMAS, WEATHER, ...TYPE_PACKAGES]) { + for (const name of [CORE, SCHEMAS, WEATHER, LANGUAGE, ...TYPE_PACKAGES]) { if (tarballs.has(name)) overrides[name] = file(name); } diff --git a/tsconfig.json b/tsconfig.json index e2b6602..6c47d1e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,9 @@ { "path": "./packages/core" }, + { + "path": "./packages/language" + }, { "path": "./packages/weather" }, diff --git a/tsconfig.test.json b/tsconfig.test.json index 83501f1..0e1ad32 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -16,6 +16,7 @@ "@idfkit/core/node": ["./packages/core/src/node.ts"], "@idfkit/weather": ["./packages/weather/src/index.ts"], "@idfkit/weather/node": ["./packages/weather/src/node.ts"], + "@idfkit/language": ["./packages/language/src/index.ts"], "@idfkit/types-v26-1": ["./packages/types-v26-1/index.d.ts"], "@idfkit/types-v9-4": ["./packages/types-v9-4/index.d.ts"] }, diff --git a/typedoc.json b/typedoc.json index 8f46a5f..e98d0c9 100644 --- a/typedoc.json +++ b/typedoc.json @@ -2,7 +2,7 @@ "$schema": "https://typedoc.org/schema.json", "extends": ["./typedoc.base.json"], "entryPointStrategy": "packages", - "entryPoints": ["packages/schemas", "packages/core", "packages/weather"], + "entryPoints": ["packages/schemas", "packages/core", "packages/language", "packages/weather"], "name": "idfkit-js", "basePath": ".", "readme": "none" diff --git a/vitest.config.ts b/vitest.config.ts index 79d93e1..feb53ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ '@idfkit/schemas': here('./packages/schemas/src/index.ts'), '@idfkit/core/node': here('./packages/core/src/node.ts'), '@idfkit/core': here('./packages/core/src/index.ts'), + '@idfkit/language': here('./packages/language/src/index.ts'), '@idfkit/weather/node': here('./packages/weather/src/node.ts'), '@idfkit/weather': here('./packages/weather/src/index.ts'), },