diff --git a/.github/ep-testfiles.sparse b/.github/ep-testfiles.sparse new file mode 100644 index 0000000..fa5c05d --- /dev/null +++ b/.github/ep-testfiles.sparse @@ -0,0 +1,17 @@ +# Which EnergyPlus example files the multi-version sweep reads. +# +# This lives in a file, and the sweep's cache key is a hash OF this file, so that changing what is +# swept necessarily changes the key. Keying on the release tag alone was wrong: the tag is +# immutable but the SELECTION is not, so an exclusion added here was served a pre-exclusion cache +# and never took effect. +# +# One pattern per line, passed to `git sparse-checkout set --no-cone`. A leading `!` excludes. + +testfiles/*.idf + +# Excluded: carries an un-migrated `report variable dictionary;` at the four tags it exists in +# (8.9.0 through 9.2.0), gone from EnergyPlus by 26.1. A defect in that file rather than in either +# library, and the two disagree about it: one reports an unknown object type, the other's pattern +# discards it for having no comma. See idfkit#193, closed as not-an-issue. Sweeping it would hold +# four versions at a count that says nothing about this repository's own regressions. +!testfiles/_1a-Long0.0.idf diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c8ce0ae..30a3c96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -257,8 +257,56 @@ jobs: run: npm run docs:build conformance: - name: Round-trip EnergyPlus example files + # Read every EnergyPlus example file, for every version @idfkit/schemas bundles. + # + # This used to install EnergyPlus 26.1, roughly a gigabyte, to get at one release's example + # files, and read them with one schema. It needed neither: the library parses against its own + # bundled schemas, so the only thing wanted from a release is the files, and a blobless, + # shallow, sparse clone of `testfiles/*.idf` is about 210 MB. Cached on the release tag, which + # is immutable, so a hit is permanent and seventeen downloads happen once rather than per pull + # request. + # + # Sweeping every version is what catches the two defects a single release cannot show. A parser + # change safe on the newest schema and wrong on an older one, because field lists and + # extensible groups moved between releases. And a file whose content belongs to one version + # while its Version object declares another, which EnergyPlus ships several of. + # + # The counts are held per version and are compared against the Python library's, which sweeps + # the same files. A number that moves on one side and not the other is a divergence. + name: "E+ ${{ matrix.version }}" runs-on: ubuntu-latest + strategy: + # Every version reports for itself. One release failing must not cancel the sixteen that + # would have told you whether the fault is that release or the change under test. + fail-fast: false + matrix: + # `findings` is that version's MEASURED count, not a round number, so a regression of one + # is caught. `errors` is zero: unlike the Python library, this one detects a Version object + # wherever it sits, so no example file fails to read at all. + # + # These numbers are IDENTICAL to the Python library's, version for version, and that is + # the point: the two sweep the same files and any number that moves on one side alone is a + # divergence. Three everywhere is the two parametric-preprocessor files; 22.2.0 adds nine + # objects absent from the schema its files declare, 25.2.0 and 24.1.0 one stale-stamp file + # each. + include: + - { version: "8.9.0", findings: 3, errors: 0 } + - { version: "9.0.1", findings: 3, errors: 0 } + - { version: "9.1.0", findings: 3, errors: 0 } + - { version: "9.2.0", findings: 3, errors: 0 } + - { version: "9.3.0", findings: 3, errors: 0 } + - { version: "9.4.0", findings: 3, errors: 0 } + - { version: "9.5.0", findings: 3, errors: 0 } + - { version: "9.6.0", findings: 3, errors: 0 } + - { version: "22.1.0", findings: 3, errors: 0 } + - { version: "22.2.0", findings: 13, errors: 0 } + - { version: "23.1.0", findings: 3, errors: 0 } + - { version: "23.2.0", findings: 3, errors: 0 } + - { version: "24.1.0", findings: 4, errors: 0 } + - { version: "24.2.0", findings: 3, errors: 0 } + - { version: "25.1.0", findings: 3, errors: 0 } + - { version: "25.2.0", findings: 8, errors: 0 } + - { version: "26.1.0", findings: 3, errors: 0 } steps: - uses: actions/checkout@v4 @@ -269,47 +317,47 @@ jobs: - run: npm ci - - name: Install EnergyPlus - # The example set is the real conformance suite: 760 files covering - # extensible shapes, blank names, and autosized fields that hand-written - # fixtures never reach. Unit tests skip these when absent, so this job - # exists to make sure they actually run somewhere. - env: - GH_TOKEN: ${{ github.token }} + # The sweep reads `dist/`, which is what an npm consumer receives, rather than the sources a + # bundler would transform. `typecheck` is `tsc --build`, so this is the build. + - run: npm run typecheck + + - name: Cache the example files + id: cache-examples + uses: actions/cache@v4 + with: + path: ep/testfiles + # The hash of the sparse file is in the key, so changing what is swept changes the key. + # Keying on the tag alone served a stale selection when the exclusion below was added. + key: ep-testfiles-v${{ matrix.version }}-${{ hashFiles('.github/ep-testfiles.sparse') }} + + - name: Fetch the example files for this release + if: steps.cache-examples.outputs.cache-hit != 'true' run: | set -euo pipefail - TAG=v26.1.0 - - # The asset name cannot be built from the version: it embeds a build - # hash (EnergyPlus-26.1.0-6f2e40d102-...) and names whichever Ubuntu - # that release targeted, which moves between releases. Ask the API. - ASSETS="$(gh api "repos/NREL/EnergyPlus/releases/tags/${TAG}" --jq '.assets[].name')" - - # Prefer the build for the runner's own Ubuntu; a mismatched glibc - # fails at exec time, well after this step has reported success. - UBUNTU="$(. /etc/os-release && echo "$VERSION_ID")" - NAME="$(printf '%s\n' "$ASSETS" | grep -E "Linux-Ubuntu${UBUNTU}-x86_64\.tar\.gz$" || true)" - if [ -z "$NAME" ]; then - NAME="$(printf '%s\n' "$ASSETS" \ - | grep -E 'Linux-Ubuntu[0-9.]+-x86_64\.tar\.gz$' | sort -V | tail -1 || true)" - echo "::warning::No ${TAG} build for Ubuntu ${UBUNTU}; falling back to ${NAME:-none}" - fi - if [ -z "$NAME" ]; then - echo "::error::No Linux x86_64 tarball on the ${TAG} release. Assets were:" - printf '%s\n' "$ASSETS" - exit 1 - fi - - echo "Installing $NAME" - curl -fsSL -o ep.tar.gz \ - "https://github.com/NREL/EnergyPlus/releases/download/${TAG}/${NAME}" - mkdir -p "$HOME/EnergyPlus" - tar -xzf ep.tar.gz -C "$HOME/EnergyPlus" --strip-components=1 - echo "ENERGYPLUS_DIR=$HOME/EnergyPlus" >> "$GITHUB_ENV" - - - run: npm test + # NREL/EnergyPlus redirects here since the rename. The canonical name is used so that the + # redirect being dropped some day is a loud failure rather than a silent one. + REPO=https://github.com/NatLabRockies/EnergyPlus + + # --filter=blob:none fetches no file contents until checkout and --depth 1 no history; + # the sparse pattern then pulls only the IDF files. + git clone --filter=blob:none --no-checkout --depth 1 \ + --branch "v${{ matrix.version }}" "$REPO" ep + cd ep + # The selection lives in .github/ep-testfiles.sparse, which the cache key hashes, so the + # two cannot disagree about what should be present. + mapfile -t PATTERNS < <(grep -vE '^[[:space:]]*(#|$)' "$GITHUB_WORKSPACE/.github/ep-testfiles.sparse") + git sparse-checkout set --no-cone "${PATTERNS[@]}" + git checkout - name: Fail if the example files were not found run: | - test -d "$ENERGYPLUS_DIR/ExampleFiles" \ - || (echo "::error::ExampleFiles missing; conformance job proved nothing" && exit 1) + # An empty sweep passes every threshold while proving nothing. + count=$(find ep/testfiles -maxdepth 1 -name '*.idf' | wc -l) + echo "found $count example files for ${{ matrix.version }}" + test "$count" -gt 0 \ + || (echo "::error::no example files at v${{ matrix.version }}; the sweep proved nothing" && exit 1) + + - name: Read every file + run: | + node scripts/sweep-example-files.mjs ep/testfiles \ + --max-findings ${{ matrix.findings }} --max-errors ${{ matrix.errors }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b9a94..a97c6fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,69 @@ The packages in this repository, `@idfkit/core`, `@idfkit/schemas`, and ## [Unreleased] +### Added + +- `describeObjectType` reports the schema's explanatory prose. It takes an + optional third argument, the prose pool, and fills `memo` on a type + description and `note` on a field description from it. + + The pool ships in the default install at `@idfkit/schemas`'s `data/` + directory, as `docs.json.gz`, and is read the same way the manifests and the + type store are. It is 4,878 distinct strings standing in for roughly 119,000 + occurrences across the seventeen bundled schemas. + + The signature stays synchronous and the pool is passed in rather than reached + for. Reading a file is not synchronous, and making the function async to + fetch something most callers do not want would be a breaking change serving + the minority. **A caller who passes nothing gets exactly what they got + before**: `undefined` prose, everywhere. + + The prose never reaches the model-reading path. It is a separate file under + `data/`, which the bundle-purity check already fences: an esbuild metafile + for a minimal read-and-write page contains zero inputs under any `data/` + directory, and that check now covers the pool with no change. + +- `writeIdf` takes `compressed`, putting each object on one line with no + comments and no blank separators. The counterpart of the Python library's + `output_type="compressed"`, and it means the same thing. + + `comments: false` is not this: it skips the padding and the comment and still + puts every field on its own line. + +- `IdfParseError` carries `diagnostics`, every finding that stopped the parse, + rather than one finding flattened into two fields. `.line` and `.typeName` + still resolve to the first finding's values, so no existing caller breaks. + +- `ParseDiagnostic` gains a `code` and an `objectName`, and declares a `column` + and a `filepath` so both libraries carry the same kinds of location. `code` is + one of eight values shared with the Python library. Match on it rather than + on `message`: the corpus compares findings on `(code, line, typeName)` and + never on wording. + + `column` and `filepath` are declared but not yet filled: the lexer counts + lines and not columns, and `parseIdf` takes text rather than a path, so + neither value exists at the point a finding is built. They are optional, so a + reader must treat them as absent until the lexer tracks a column and the + file-reading edge attaches the path it read from. + +### Fixed + +- `enumValues` reports the values it was omitting. The empty string is included + for the enums that declare one, and the sentinels `Autosize` and + `Autocalculate` are read from the collapsed `anyOf` string branch, which + validation has always read and this path never did. + + Validation is unaffected: the blank is still filtered out of the list + validation checks against and is restored only in the description. + +- Three object types reported their fields in alphabetical order rather than + declaration order: `ZoneProperty:UserViewFactors:BySurfaceName`, + `ZoneTerminalUnitList`, and `SolarCollector:UnglazedTranspired:Multisystem`. + These are the three whose positional field list holds only the name, so the + description fell back to the key order of the property map, which the + content-addressing serializer had sorted. The bundle now records their + declaration order explicitly. + ### Changed - The install-size budget for the shared name rose from 1.5 MB to 1.75 MB diff --git a/docs-snippets/explanation/two-writers-one-model/controls.ts b/docs-snippets/explanation/two-writers-one-model/controls.ts new file mode 100644 index 0000000..9150407 --- /dev/null +++ b/docs-snippets/explanation/two-writers-one-model/controls.ts @@ -0,0 +1,20 @@ +// Preamble, not shown on the page: the values this example assumes it already +// has, each with the type the page's earlier steps would have given it. +import { writeIdf, type IdfDocument } from '@idfkit/core'; +declare const model: IdfDocument; + +// --8<-- [start:controls] +// Every control, at a value that is not the default. +const text = writeIdf(model, { + indent: ' ', + commentColumn: 45, + comments: true, +}); +// --8<-- [end:controls] + +// --8<-- [start:compressed] +const compact = writeIdf(model, { compressed: true }); +// --8<-- [end:compressed] + +void text; +void compact; diff --git a/packages/core/package.json b/packages/core/package.json index df3d36c..fab80fc 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -42,8 +42,8 @@ "node": ">=20" }, "idfkit": { - "conformance": "conformance-2026.7", - "governance": "governance-2026.9" + "conformance": "conformance-2026.8", + "governance": "governance-2026.10" }, "dependencies": { "@idfkit/schemas": "0.0.0" diff --git a/packages/core/src/conformance.ts b/packages/core/src/conformance.ts index 43119ba..0964e58 100644 --- a/packages/core/src/conformance.ts +++ b/packages/core/src/conformance.ts @@ -17,4 +17,4 @@ * This is not a version number and it is not compared to one. Two installed libraries agree on the * formats when they declare the same level, whatever their own versions say (FR-025). */ -export const CONFORMANCE_LEVEL = 'conformance-2026.7'; +export const CONFORMANCE_LEVEL = 'conformance-2026.8'; diff --git a/packages/core/src/introspect/describe.ts b/packages/core/src/introspect/describe.ts index d3680c7..43f97a9 100644 --- a/packages/core/src/introspect/describe.ts +++ b/packages/core/src/introspect/describe.ts @@ -105,7 +105,23 @@ export interface ObjectDescription { * Python raises `UnknownObjectTypeError`, which has no registered TypeScript * counterpart and so must not become a new exported class. */ -export function describeObjectType(schema: Schema, objType: string): ObjectDescription { +/** + * The schema's explanatory prose, loaded on demand. + * + * A plain array of strings, indexed by `SlimType.m` and `SlimField.n`. It is + * passed in rather than reached for, and that is deliberate: reading it is + * asynchronous, `describeObjectType` is synchronous, and making the function + * async to fetch a file most callers do not want would be a breaking change + * serving the minority. Loading it is the caller's step, and its cost is + * visible at the call site instead of hidden inside a description. + */ +export type ProsePool = readonly string[]; + +export function describeObjectType( + schema: Schema, + objType: string, + prose?: ProsePool +): ObjectDescription { const type = schema.require(objType); // Exact-cased input resolves to itself, so this matches Python's verbatim // echo of the argument for every name Python accepts. @@ -119,13 +135,17 @@ export function describeObjectType(schema: Schema, objType: string): ObjectDescr const fieldNames = orderedFieldNames(type); const fields: FieldDescription[] = fieldNames.map((name) => - describeField(name, properties[name], required.has(name)) + describeField(name, properties[name], required.has(name), prose) ); return { objType: canonical, - // Not carried by the slim bundle. See ObjectDescription.memo. - memo: undefined, + // Resolved from the pool when one is supplied, and `undefined` otherwise — + // which is exactly what this returned before the pool existed, so a caller + // who passes nothing sees no change at all (FR-014). A type with no memo in + // the source schema also reports `undefined`, in both languages: absence is + // reported as absence rather than filled with a placeholder. + memo: lookupProse(type.m, prose), fields, requiredFields: [...requiredFields], hasName: type.anon !== 1, @@ -162,7 +182,11 @@ function orderedFieldNames(type: SlimType): string[] { // not schema order. `AvailabilityManagerAssignmentList` is the type that shows // it — its two extensible fields are declared object-type first and sort // name first. - const ordered = names.length > 0 ? names : Object.keys(type.p); + // `fo` is the declaration order, recorded by the bundle for the three types + // whose `f` holds only the name. Without it the fallback below returns the + // key order of `p`, which `canonical()` sorted for content-addressing, and + // the reader gets alphabetical order where Python gives declaration order. + const ordered = names.length > 0 ? names : (type.fo ?? Object.keys(type.p)).slice(); if (type.x !== undefined) { for (const name of type.x.fields) { @@ -187,7 +211,8 @@ function mergedProperties(type: SlimType): Record { function describeField( name: string, field: SlimField | undefined, - required: boolean + required: boolean, + prose?: ProsePool ): FieldDescription { if (field === undefined) { // In the positional order but absent from the schema. Python reaches the @@ -225,17 +250,62 @@ function describeField( required, default: field.d, units: field.u, - enumValues: field.e === undefined ? undefined : [...field.e], + enumValues: acceptedValues(field), minimum: collapsedAnyOf ? undefined : field.min, maximum: collapsedAnyOf ? undefined : field.max, exclusiveMinimum: collapsedAnyOf ? undefined : field.xmin, exclusiveMaximum: collapsedAnyOf ? undefined : field.xmax, - note: undefined, + note: lookupProse(field.n, prose), isReference: field.ol !== undefined, objectList: field.ol === undefined ? undefined : [...field.ol], }; } +/** + * Resolve a prose index against the pool. + * + * Returns `undefined` for a record with no prose, and for every record when no + * pool was supplied. Out-of-range is `undefined` too rather than a throw: a + * caller who hands in a pool from a different bundle build gets no prose, which + * is the same thing they had before, instead of a description that cannot be + * produced at all. + * + * @internal + */ +function lookupProse(index: number | undefined, prose: ProsePool | undefined): string | undefined { + if (index === undefined || prose === undefined) return undefined; + return prose[index]; +} + +/** + * The values a field accepts, as Python reports them. + * + * Two sources, and until feature 002 this read neither completely: + * + * - `e`, the choice list, with the empty string filtered out by the bundle + * because `e` is what validation checks against. Python keeps the blank, so + * `eb` records that it was there and it goes back on the front. It is always + * the front: measured across all 17 schemas, all 21,962 blank-bearing enums + * carry it at position 0. + * - `se`, the collapsed `anyOf` string branch, which holds the sentinels. + * `Autosize` on 10,565 fields and `Autocalculate` on 1,781. Validation has + * always read it (`validate/validate.ts:543`); this path never did, so + * `WindowMaterial:Glazing:EquivalentLayer.diffuse_diffuse_solar_transmittance` + * reported nothing where Python reported `['', 'Autocalculate']`. + * + * A field carries one or the other, never both: `se` exists only when the field + * was an `anyOf`, and `e` is then hoisted from the numeric branch. The 68 + * fields that carry both are the numeric-enum ones, and Python reports the + * string branch for those, which is what taking `se` first does. + * + * @internal + */ +function acceptedValues(field: SlimField): (string | number)[] | undefined { + if (field.se !== undefined) return [...field.se]; + if (field.e === undefined) return undefined; + return field.eb === 1 ? ['', ...field.e] : [...field.e]; +} + /** * Map the slim storage class back to the epJSON JSON-Schema type string Python * reports. diff --git a/packages/core/src/node.ts b/packages/core/src/node.ts index 5063281..3959513 100644 --- a/packages/core/src/node.ts +++ b/packages/core/src/node.ts @@ -13,7 +13,14 @@ import { localBundle } from '@idfkit/schemas/node'; import type { IdfDocument } from './document.js'; import { getEpJsonVersion, parseEpJson } from './parse/epjson.js'; -import { getIdfVersion, parseIdf, type ParseOptions, type ParseResult } from './parse/idf.js'; +import { + getIdfVersion, + IdfParseError, + parseIdf, + type ParseDiagnostic, + type ParseOptions, + type ParseResult, +} from './parse/idf.js'; import type { AnyTypeMap, UntypedMap } from './typemap.js'; import { resolveVersion } from './versions.js'; import { writeEpJson, type WriteEpJsonOptions } from './write/epjson.js'; @@ -79,7 +86,28 @@ export async function loadIdfWithDiagnostics( // names) that are not valid utf-8 and would otherwise decode to U+FFFD. const text = await readFile(path, 'latin1'); const schema = await schemaFor(getIdfVersion(text), options); - return parseIdf(text, schema, options); + + // `parseIdf` takes text and cannot know where the text came from, so the path is attached here, + // at the one place that does. Python's parser opens the file itself and fills `filepath` from + // its own constructor argument; this is the same field reaching a caller by the only route this + // library has (FR-033). + // + // Stamped on both paths: the findings that stop the parse arrive on the error, and the ones that + // do not arrive in the result. Neither is useful without saying which file it was. + try { + const result = parseIdf(text, schema, options); + return { ...result, diagnostics: result.diagnostics.map((d) => withPath(d, path)) }; + } catch (error) { + if (error instanceof IdfParseError) { + throw new IdfParseError(error.diagnostics.map((d) => withPath(d, path))); + } + throw error; + } +} + +/** Attach the source path to a finding, leaving one that already names a file alone. */ +function withPath(diagnostic: ParseDiagnostic, path: string): ParseDiagnostic { + return diagnostic.filepath === undefined ? { ...diagnostic, filepath: path } : diagnostic; } /** Read and parse an epJSON file. */ diff --git a/packages/core/src/parse/idf.ts b/packages/core/src/parse/idf.ts index bfcd4d9..a5aa62c 100644 --- a/packages/core/src/parse/idf.ts +++ b/packages/core/src/parse/idf.ts @@ -6,8 +6,14 @@ import type { AnyTypeMap, UntypedMap } from '../typemap.js'; import { lex, type LexDiagnostic, type RawObject } from './lexer.js'; export interface ParseDiagnostic extends LexDiagnostic { - /** Object type the problem occurred in, when known. */ - typeName?: string; + /** + * Object name the problem occurred in, when known. + * + * Python spells this `obj_name`, and `typeName` against its `obj_type` is the same idiomatic + * casing difference the naming register already records. Not a gap, and not renamed: spending a + * rename to make the register harder to read is the wrong trade. + */ + objectName?: string; } export interface ParseOptions { @@ -49,6 +55,20 @@ export function parseIdf( options.onDiagnostic?.(diagnostic); }; + /** + * Record a finding that does NOT stop the parse, whatever `strict` says. + * + * `report` throws under `strict`, which is right for a finding that leaves nothing to return: an + * object whose type is unknown cannot be built. A value of the wrong KIND is different. The + * object is built, the document is complete, and a caller reading strictly gets exactly what they + * got before this finding existed (FR-014). Routing it through `report` made a strict parse fail + * on files that used to load, which is the opposite of additive. + */ + const note = (diagnostic: ParseDiagnostic): void => { + diagnostics.push(diagnostic); + options.onDiagnostic?.(diagnostic); + }; + const raw = lex(text, { onDiagnostic: report }); const document = new IdfDocument(schema); @@ -58,20 +78,50 @@ export function parseIdf( report({ message: `Unknown object type "${object.typeName}" in EnergyPlus ${schema.version}`, line: object.line, + column: object.column, typeName: object.typeName, + // Best effort, and the same rule Python's `_extract_object_name` uses: the first positional + // value is the name for every named type, and is a real field for the anonymous ones. An + // unknown type has no definition to tell the two apart, so the raw first value is what there + // is. Declaring `objectName` and never filling it would make the field a claim rather than a + // value. + objectName: object.values[0], + code: 'UnknownObjectType', }); continue; } const definition = schema.require(canonical); try { - const { name, values } = interpret(definition, object); + const invalid: { field: string; index: number; value: string }[] = []; + const { name, values } = interpret(definition, object, invalid); document.addRaw(canonical, definition.anon === 1 ? null : name, values); + + // Reported after the object is built, never instead of building it: a value of the wrong + // kind does not stop the parse, so a caller reading strictly gets the document they always + // got. This adds a finding where there was silence, and nothing else. + // + // Positioned at the offending FIELD rather than at the object, because that is what makes it + // useful. The shape this catches is a missing semicolon swallowing the object below, and the + // object's own first line is nowhere near the damage. + for (const { field, index, value } of invalid) { + if (stringIsLegal(definition, field, value)) continue; + note({ + message: `Field "${field}" expects a number, got "${value}"`, + line: fieldLine(text, object, index), + typeName: canonical, + objectName: definition.anon === 1 ? undefined : object.values[0], + code: 'InvalidField', + }); + } } catch (error) { report({ message: error instanceof Error ? error.message : String(error), line: object.line, + column: object.column, typeName: canonical, + objectName: definition.anon === 1 ? undefined : object.values[0], + code: 'ParseError', }); } } @@ -80,7 +130,11 @@ export function parseIdf( } /** Map positional IDF values onto named schema fields. */ -function interpret(definition: SlimType, object: RawObject): { name: string; values: FieldValues } { +function interpret( + definition: SlimType, + object: RawObject, + invalid?: { field: string; index: number; value: string }[] +): { name: string; values: FieldValues } { const order = definition.f; const named = definition.anon !== 1 && order[0] === 'name'; const values: FieldValues = {}; @@ -98,11 +152,17 @@ function interpret(definition: SlimType, object: RawObject): { name: string; val const fixed = named ? order.slice(1) : order; for (const field of fixed) { + const index = cursor; const raw = object.values[cursor++]; if (raw === undefined) break; if (raw === '') continue; const coerced = coerce(definition, field, raw); if (coerced !== undefined) values[field] = coerced; + // A numeric field that kept its string is a coercion that fell through. Whether that is + // actually wrong needs the schema and is asked on the reporting path, which runs almost never. + if (invalid !== undefined && coerced === raw && isNumericField(definition, field)) { + invalid.push({ field, index, value: raw }); + } } // Everything past the fixed fields belongs to the extensible section, read in @@ -141,6 +201,107 @@ function interpret(definition: SlimType, object: RawObject): { name: string; val return { name, values }; } +/** + * The two sizing sentinels, accepted in ANY numeric field. + * + * Not read from the field's own `se` branch, deliberately. The schema EnergyPlus ships is NARROWER + * than the engine it ships with, and its own example files prove it. Both of these declare a string + * branch naming exactly one sentinel, and the shipped files use the other one: + * + * `PlantLoop.plant_loop_volume` allows `Autocalculate`; 28 files write `Autosize`. + * `AirTerminal:SingleDuct:VAV:Reheat.maximum_flow_fraction_during_reheat` allows `Autosize`; + * 673 files write `Autocalculate`. + * + * The schema is well formed: across all 17 bundled versions no field declares a non-numeric string + * default on a numeric type with no string branch. It is simply stricter about WHICH sentinel + * belongs where than EnergyPlus is. Reading each field's branch literally produced 3,775 findings + * across the 760 example files of one release, every one against a model EnergyPlus reads happily. + * + * A parse finding says the value is not of the kind the field takes. Whether the exact sentinel is + * the one THIS field documents is a schema question, and `validateObject` already answers it. + */ +const SIZING_SENTINELS = new Set(['autosize', 'autocalculate']); + +/** @internal */ +function isNumericField(definition: SlimType, field: string): boolean { + const kind = definition.p[field]?.t; + return kind === 'n' || kind === 'i'; +} + +/** + * Whether a string the numeric coercion rejected is legal for this field anyway. + * + * Consulted only when coercion has already failed, so the parse path pays nothing for it. + * + * @internal + */ +function stringIsLegal(definition: SlimType, field: string, value: string): boolean { + if (SIZING_SENTINELS.has(value.toLowerCase())) return true; + + const slim = definition.p[field]; + if (slim === undefined) return true; + // `auto` marks a collapsed `anyOf`. Without an `se` the string branch declared no enum, so any + // string satisfies it; with one, only its members do. + if (slim.se === undefined) return slim.auto === 1; + + const folded = value.toLowerCase(); + return slim.se.some((allowed) => allowed.toLowerCase() === folded); +} + +/** + * The 1-based line a field sits on, found by rescanning from the object's own offset. + * + * The lexer records one offset per object rather than a line per field, because a finding about a + * field is rare and an array per object is not. This walks forward counting separators, stepping + * over `!` comments so a comma inside one is not mistaken for one, which is the same rule the + * lexer itself follows. + * + * `index` counts into `RawObject.values`, which the lexer has already shifted the type name off. + * The scan starts at the type name, so it steps over one more separator than the index: the comma + * that ends `Building,` is what puts values[0] on the line after it. + * + * @internal + */ +function fieldLine(text: string, object: RawObject, index: number): number { + if (object.offset === undefined) return object.line; + + const separators = index + 1; + let seen = 0; + let position = object.offset; + while (position < text.length && seen < separators) { + const char = text[position]; + if (char === '!') { + const newline = text.indexOf('\n', position); + if (newline < 0) break; + position = newline + 1; + continue; + } + if (char === ';') break; + if (char === ',') seen += 1; + position += 1; + } + if (seen < separators) return object.line; + + // Step over whitespace AND any comment between the separator and the value. A field's comma is + // routinely followed by `!- Field Name` on the same line, and stopping at the `!` would report + // the line the PREVIOUS value sits on, one too early. + while (position < text.length) { + const char = text[position] ?? ''; + if (char === '!') { + const newline = text.indexOf('\n', position); + if (newline < 0) break; + position = newline + 1; + continue; + } + if (!/\s/.test(char)) break; + position += 1; + } + + let line = object.line; + for (let i = object.offset; i < position; i += 1) if (text[i] === '\n') line += 1; + return line; +} + function coerce(definition: SlimType, field: string, raw: string): StoredValue | undefined { return coerceValue(definition.p[field]?.t, raw); } @@ -178,12 +339,32 @@ function coerceValue(kind: string | undefined, raw: string): StoredValue | undef export class IdfParseError extends Error { readonly line: number; readonly typeName: string | undefined; + /** + * Every finding that stopped the parse. + * + * This error carried `line` and `typeName` from a single diagnostic, flattened into fields, and + * a caller who wanted the rest had nowhere to look. Python's `IDFParseError` has always carried + * the collection; this is the half of the difference that was real. + * + * `line` and `typeName` still resolve to the first finding's values, so no existing caller + * breaks (FR-013, FR-014). They are a convenience over `diagnostics[0]` rather than a second + * source of truth. + */ + readonly diagnostics: readonly ParseDiagnostic[]; - constructor(diagnostic: ParseDiagnostic) { - super(`${diagnostic.message} (line ${diagnostic.line})`); + constructor(diagnostic: ParseDiagnostic | readonly ParseDiagnostic[]) { + const diagnostics = Array.isArray(diagnostic) + ? (diagnostic as readonly ParseDiagnostic[]) + : [diagnostic as ParseDiagnostic]; + const first = diagnostics[0]; + if (first === undefined) { + throw new TypeError('IdfParseError needs at least one diagnostic'); + } + super(`${first.message} (line ${first.line})`); this.name = 'IdfParseError'; - this.line = diagnostic.line; - this.typeName = diagnostic.typeName; + this.line = first.line; + this.typeName = first.typeName; + this.diagnostics = Object.freeze([...diagnostics]); } } diff --git a/packages/core/src/parse/lexer.ts b/packages/core/src/parse/lexer.ts index 34168d9..cb2a79e 100644 --- a/packages/core/src/parse/lexer.ts +++ b/packages/core/src/parse/lexer.ts @@ -6,13 +6,58 @@ export interface RawObject { values: string[]; /** 1-based line where the object starts, for diagnostics. */ line: number; + /** 1-based column where the object starts, for diagnostics. */ + column?: number; + /** + * Absolute offset of the object's first character in the source text. + * + * One number per object, so that a finding about a FIELD can be positioned without the lexer + * recording a line for every field of every object. The rescan that uses it runs only when a + * finding is being built. + */ + offset?: number; } export interface LexDiagnostic { message: string; line: number; + /** + * Machine-readable kind, from the vocabulary both libraries share. + * + * Derived from Python's exception hierarchy by dropping the `Error` suffix, so neither language + * invented it and the mapping stays mechanical. The conformance corpus compares a finding on + * `(code, line, typeName)` and never on `message`: wording is a presentation choice each library + * should stay free to improve, and pinning it would turn every improvement into a failure. + */ + code?: ParseDiagnosticCode; + /** 1-based column, when the lexer knew one. */ + column?: number; + /** Path the text came from, when it came from a file rather than a string. */ + filepath?: string; + /** + * Object type the problem occurred in, when known. + * + * Declared here rather than only on `ParseDiagnostic` because the lexer knows it too: an + * unterminated object has read its type name before it runs out of input, and a finding that + * drops it says only that something went wrong somewhere. + */ + typeName?: string; } +/** + * The shared diagnostic vocabulary. The table lives in + * `idfkit-conformance/runners/compare.md`; a code outside it is a difference, not a near match. + */ +export type ParseDiagnosticCode = + | 'UnknownObjectType' + | 'InvalidField' + | 'Range' + | 'DuplicateObject' + | 'ParseError' + | 'VersionMismatch' + | 'UnsupportedVersion' + | 'SchemaNotFound'; + export interface LexOptions { /** Report a problem instead of throwing. */ onDiagnostic?: (diagnostic: LexDiagnostic) => void; @@ -51,6 +96,21 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { let fieldStart = 0; let objectLine = 1; let objectStarted = false; + /** Offset of the first character of the current line, for turning an offset into a column. */ + let lineStart = 0; + /** + * Column the current object's type name begins at, or 0 while none has been seen. + * + * The first NON-BLANK character, not the start of the field text: Python's regex matches the + * type name itself, so an object indented three spaces reports column 4 there and has to report + * column 4 here too, or the corpus compares two different notions of position. + */ + let objectColumn = 0; + /** Absolute offset of the current object's first non-blank character. */ + let objectOffset = -1; + + /** 1-based column of an offset on the line it falls in. Matches Python's `_line_and_column`. */ + const columnAt = (offset: number): number => offset - lineStart + 1; const endField = (end: number): string => { chunks.push(text.slice(fieldStart, end)); @@ -62,6 +122,15 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { while (index < length) { const char = text[index]; + // The first non-blank character of an object fixes where the object starts. Recorded here + // rather than at the delimiter, because by then the leading whitespace has been consumed and + // the offset that remains points at the padding rather than at the name. + if (objectColumn === 0 && char !== undefined && !/\s/.test(char)) { + objectLine = line; + objectColumn = columnAt(index); + objectOffset = index; + } + if (char === '!') { // Preserve any field text seen before the comment, then resume after the // newline. This is what lets `Zone1, !- Name` work: the comment is not @@ -76,9 +145,12 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { index = newline + 1; fieldStart = index; line += 1; + lineStart = index; if (!objectStarted && chunks.join('').trim() === '') { chunks = []; objectLine = line; + objectColumn = 0; + objectOffset = -1; } continue; } @@ -98,18 +170,32 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { const typeName = values.shift() ?? ''; if (typeName === '') { - report?.({ message: 'Object with no type name', line: objectLine }); + report?.({ + message: 'Object with no type name', + line: objectLine, + column: objectColumn || undefined, + code: 'ParseError', + }); } else { - objects.push({ typeName, values, line: objectLine }); + objects.push({ + typeName, + values, + line: objectLine, + column: objectColumn || undefined, + offset: objectOffset >= 0 ? objectOffset : undefined, + }); } values = []; objectStarted = false; objectLine = line; + objectColumn = 0; + objectOffset = -1; continue; } if (char === '\n') { line += 1; + lineStart = index + 1; if ( !objectStarted && chunks.join('').trim() === '' && @@ -119,6 +205,8 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { chunks = []; fieldStart = index + 1; objectLine = line; + objectColumn = 0; + objectOffset = -1; } } @@ -130,6 +218,12 @@ export function lex(text: string, options: LexOptions = {}): RawObject[] { report?.({ message: `Unterminated object near "${trailing.slice(0, 40) || values[0]}" (missing ";")`, line: objectLine, + code: 'ParseError', + column: objectColumn || undefined, + // `values` has not been shifted, because the shift happens on `;` and there was none, so the + // type name is still at the front. Reporting it is what lets the corpus compare this finding + // against Python's on `(code, line, typeName)`. + typeName: values[0], }); } diff --git a/packages/core/src/write/idf.ts b/packages/core/src/write/idf.ts index 170b14d..e47160a 100644 --- a/packages/core/src/write/idf.ts +++ b/packages/core/src/write/idf.ts @@ -27,6 +27,36 @@ export interface WriteIdfOptions { * @defaultValue true */ versionFirst?: boolean; + /** + * How object types are ordered. + * + * `'source'` keeps the order the types first appeared in the document, which is what this writer + * has always done and remains the default: no default moves (FR-017). `'sorted'` orders them by + * type name, which is the other language's default and what its `!-Option SortedOrder` header + * declares. + * + * An enumeration rather than a boolean, because three behaviours exist across the two languages + * and two formats and a flag cannot say which of the three is wanted. + * + * Orthogonal to `versionFirst`, which pins `Version` ahead of whichever order this selects. + * + * @defaultValue 'source' + */ + ordering?: 'sorted' | 'source'; + /** + * Put each object on a single line, with no comments and no blank separators. + * + * The counterpart of the other language's `output_type="compressed"`, and it means the same + * thing: type name and values joined by commas, one object per line, no generator header, no + * blank line between objects. The corpus checks that the two agree structurally rather than + * textually, because the two writers differ on defaults that compressed output does not remove. + * + * `comments: false` is not this. That skips the padding and the comment and still puts every + * field on its own line, which is a different, coarser output that both languages already had. + * + * @defaultValue false + */ + compressed?: boolean; } /** @@ -44,14 +74,22 @@ export function writeIdf( document: IdfDocument, options: WriteIdfOptions = {} ): string { - const comments = options.comments ?? true; + const compressed = options.compressed ?? false; + // Compressed output has no comments by definition. Asking for both is not an error, because the + // narrower request is unambiguous: comments cannot survive a single-line object. + const comments = compressed ? false : (options.comments ?? true); const commentColumn = options.commentColumn ?? 30; const indent = options.indent ?? ' '; const versionFirst = options.versionFirst ?? true; + const ordering = options.ordering ?? 'source'; const parts: string[] = []; const types = document.types(); + // Sorted first, then Version pinned, so the two controls compose the way the other language's + // output does: its `SortedOrder` header describes the type ordering and Version sits above it. + if (ordering === 'sorted') types.sort(); + if (versionFirst && types.includes('Version')) { types.splice(types.indexOf('Version'), 1); types.unshift('Version'); @@ -61,9 +99,11 @@ export function writeIdf( const collection = document.all(typeName); if (collection.size === 0) continue; for (const obj of collection) { - parts.push(writeObject(obj, { comments, commentColumn, indent })); + parts.push(writeObject(obj, { comments, commentColumn, indent, compressed })); } - parts.push(''); + // The blank separator after each type is one of the two things compressed removes. The other + // is the per-field line break, in `writeObject`. + if (!compressed) parts.push(''); } return parts.join('\n'); @@ -73,6 +113,8 @@ export interface ObjectWriteOptions { comments: boolean; commentColumn: number; indent: string; + /** Put the whole object on one line. See `WriteIdfOptions.compressed`. */ + compressed?: boolean; } /** Serialize one object. */ @@ -109,11 +151,21 @@ export function writeObject(obj: IdfObject, options: ObjectWriteOptions): string } } - const lines: string[] = [`${obj.typeName},`]; + // Compressed output carries no trailing newline of its own: `writeIdf` joins the parts with one, + // and adding a second here is what puts a blank line between objects. Removing that blank line + // is half of what compressed means. if (cells.length === 0) { - return `${obj.typeName};\n`; + return options.compressed ? `${obj.typeName};` : `${obj.typeName};\n`; + } + + if (options.compressed) { + // Type name and every value on one line, comma-separated, terminated once. The same shape the + // other language produces, whose writer joins the values and skips the header and separators. + return `${obj.typeName},${cells.map((cell) => cell.value).join(',')};`; } + const lines: string[] = [`${obj.typeName},`]; + cells.forEach((cell, index) => { const terminator = index === cells.length - 1 ? ';' : ','; const body = `${options.indent}${cell.value}${terminator}`; diff --git a/packages/core/tests/helpers.ts b/packages/core/tests/helpers.ts index 9f14ddd..20c3799 100644 --- a/packages/core/tests/helpers.ts +++ b/packages/core/tests/helpers.ts @@ -1,7 +1,7 @@ import { existsSync } from 'node:fs'; import type { Schema } from '@idfkit/schemas'; -import { localBundle } from '@idfkit/schemas/node'; +import { localBundle, nodeSource } from '@idfkit/schemas/node'; const bundle = localBundle(); const cache = new Map>(); @@ -32,3 +32,20 @@ export function exampleFilesDir(): string | undefined { return candidates.find((dir) => existsSync(dir)); } + +/** + * The schema's explanatory prose, loaded once for the whole test run. + * + * Read through the ordinary bundle source, because that is exactly how a caller + * reaches it: there is no new export for the pool, and deliberately so. A Node + * caller uses `readBundleFileSync('docs.json')`, a browser caller + * `httpSource(base).read('docs.json')`, and the file sits in `data/` where the + * bundle-purity gate already fences it off the parse path. + */ +let prosePromise: Promise | undefined; +export function prose(): Promise { + prosePromise ??= nodeSource() + .read('docs.json') + .then((value) => value as readonly string[]); + return prosePromise; +} diff --git a/packages/core/tests/introspect.test.ts b/packages/core/tests/introspect.test.ts index 978e1c6..844a2ba 100644 --- a/packages/core/tests/introspect.test.ts +++ b/packages/core/tests/introspect.test.ts @@ -9,7 +9,7 @@ import type { Schema } from '@idfkit/schemas'; import { describeObjectType } from '../src/introspect/describe.js'; import type { FieldDescription, ObjectDescription } from '../src/introspect/describe.js'; -import { schema } from './helpers.js'; +import { prose as loadProse, schema } from './helpers.js'; let v26: Schema; beforeAll(async () => { @@ -271,7 +271,9 @@ const PYTHON_SHAPES: Record = { const PYTHON_FIELD_NAME_DIGEST = '9a62d871cc2c194a1ae1b273db7c9730195f4161202696c8058a9772f6955925'; /** - * The only two types whose field ORDER cannot be reproduced from the bundle. + * The two types whose field ORDER could not be reproduced from the bundle, until + * feature 002 recorded declaration order in `SlimType.fo`. Kept, with Python's + * order as the expectation, so the closure cannot silently come undone. * * Both take Python's fallback path — `legacy_idd.fields` is just `["name"]`, so * Python orders their fields by the schema's property declaration order. The @@ -280,11 +282,11 @@ const PYTHON_FIELD_NAME_DIGEST = '9a62d871cc2c194a1ae1b273db7c9730195f4161202696 * gone. For 57 of the 59 fallback types in 26.1.0 that is invisible, because * they have at most one fixed property; these two have two. * - * Deliberately not patched with an "array key sorts last" rule: it would happen - * to fix both here, and it is wrong for `SolarCollector:UnglazedTranspired:Multisystem` - * in 8.9.0 through 9.2.0, where the array really is declared first. Closing this - * needs the bundle to record declaration order, which is a `@idfkit/schemas` - * change. + * It was deliberately not patched with an "array key sorts last" rule: that + * would have happened to fix both here and is wrong for + * `SolarCollector:UnglazedTranspired:Multisystem` in 8.9.0 through 9.2.0, where + * the array really is declared first. It was closed the way that comment said + * it had to be, by recording declaration order in the bundle. * * These two are the whole list for 9.4.0 through 26.1.0. 8.9.0 through 9.2.0 add * six more, because there the array's `items.properties` order also differs from @@ -350,8 +352,8 @@ function field(description: ObjectDescription, name: string): FieldDescription { } /** Every description in the version, computed once and reused. */ -function describeAll(s: Schema): ObjectDescription[] { - return s.typeNames.map((name) => describeObjectType(s, name)); +function describeAll(s: Schema, prosePool?: readonly string[]): ObjectDescription[] { + return s.typeNames.map((name) => describeObjectType(s, name, prosePool)); } describe('describeObjectType', () => { @@ -451,13 +453,15 @@ describe('agreement with Python', () => { expect(digest).toBe('02aef807379219fa13dcf3c7df3c6592126dfca01684e1c868b3b5dae5fdeadd'); }); - it('diverges from Python on exactly the two documented orderings', () => { + it('agrees with Python on the two orderings that used to diverge', () => { + // Closed by feature 002, exactly as the comment on KNOWN_ORDER_DIVERGENCES + // said it would have to be: the bundle now records declaration order in + // `SlimType.fo` for the three types whose `legacy_idd.fields` holds only + // the name, and `orderedFieldNames` reads it instead of falling back to the + // alphabetized key list. for (const [typeName, expected] of Object.entries(KNOWN_ORDER_DIVERGENCES)) { const names = describeObjectType(v26, typeName).fields.map((f) => f.name); - expect(names).toEqual(expected.typescript); - expect(names).not.toEqual(expected.python); - // The names are all there; only their order differs. - expect([...names].sort()).toEqual([...expected.python].sort()); + expect(names).toEqual(expected.python); } }); @@ -670,36 +674,40 @@ describe('ordinary field constraints', () => { }); }); -describe('metadata the slim schema does not carry', () => { - it('reports memo and note as absent rather than inventing them', () => { - // Python fills memo for 845 of 858 types and note for 6212 of 12712 fields, - // both from keys the slim bundle drops on purpose (see @idfkit/schemas' - // types.ts header). The members stay in the type — the naming register - // requires the same field set on both sides — but nothing here fabricates a - // value from the type or field name. +/** + * Feature 002 closed all three of these. Each `it` below asserted the gap until + * 2026-09-04 and now asserts the agreement, which is the same test pointed the + * other way rather than a new one. + */ +describe('metadata the slim schema carries since feature 002', () => { + it('reports memo and note as absent when no prose pool is supplied', () => { + // The pool is opt-in and the signature stayed synchronous, so a caller who + // passes nothing must see exactly what they saw before (FR-014). This is + // the guarantee that makes the change additive, and it is permanent. const zone = describeObjectType(v26, 'Zone'); expect(zone.memo).toBeUndefined(); expect(zone.fields.every((f) => f.note === undefined)).toBe(true); }); - it('drops the empty-string choice that Python keeps', () => { - // The bundle filters "" out of every enum. Python keeps it: 1378 of its - // 2293 enum-bearing fields in 26.1.0 include "". Not recoverable here. - const compact = field(describeObjectType(v26, 'Zone'), 'part_of_total_floor_area'); + it('keeps the empty-string choice Python keeps', () => { + // The bundle still filters "" out of `e`, because `e` is what validation + // checks against and admitting the blank there would change what validate() + // accepts. `eb` records that it was there and the description path restores + // it, which is why this agrees with Python without validation moving. + const partOfArea = field(describeObjectType(v26, 'Zone'), 'part_of_total_floor_area'); - expect(compact.enumValues).toEqual(['No', 'Yes']); + expect(partOfArea.enumValues).toEqual(['', 'No', 'Yes']); }); - it('has no enum for an autosizable field, where Python reports the branch enum', () => { - // Python's `enum_values` falls through to the first anyOf branch carrying an - // enum, which for these fields is ["", "Autocalculate"] or ["", "Autosize"]. - // `enumValues` reports the NUMERIC branch's enum, which these fields do not - // have. The string branch's literals are in the bundle — `se`, which is what - // validation reads — but surfacing them here would change a difference the - // parity ledger records, which is a decision for that ledger and not for - // this port. - expect(field(describeObjectType(v26, 'Zone'), 'ceiling_height').enumValues).toBeUndefined(); + it('reports the branch enum for an autosizable field, as Python does', () => { + // `se` holds the collapsed anyOf string branch and validation has always + // read it. The description path now reads it too, so the sentinels are + // visible on both sides: Autosize on 10,565 fields, Autocalculate on 1,781. + expect(field(describeObjectType(v26, 'Zone'), 'ceiling_height').enumValues).toEqual([ + '', + 'Autocalculate', + ]); }); }); @@ -744,3 +752,166 @@ describe('purity', () => { expect(first.fields[0]).not.toBe(second.fields[0]); }); }); + +/** + * Feature 002, US2: the three closures, asserted from the reader's side. + * + * These began as pins of the pre-closure behaviour, written before anything was + * touched so that a regression would be a red test rather than a discovery. + * They now assert what the closure produces. The two that still assert absence + * are not leftovers: they are the FR-014 guarantee that a caller who supplies + * no prose pool sees exactly what they saw before, which is permanent. + */ +describe('feature 002, describing a type agrees across both languages', () => { + it('reports no type prose when no pool is supplied', () => { + const zone = describeObjectType(v26, 'Zone'); + + expect(zone.memo).toBeUndefined(); + }); + + it('reports no field prose when no pool is supplied', () => { + const zone = describeObjectType(v26, 'Zone'); + const ceilingHeight = zone.fields.find((f) => f.name === 'ceiling_height'); + + expect(ceilingHeight).toBeDefined(); + expect(ceilingHeight?.note).toBeUndefined(); + }); + + it('reports the type prose Python reports, when a pool is supplied', async () => { + const prose = await loadProse(); + const zone = describeObjectType(v26, 'Zone', prose); + + // Length and prefix rather than the whole paragraph, which is 485 + // characters. The exhaustive check is the digest below; this one is here so + // that a failure is readable. + expect(zone.memo).toMatch(/^Defines a thermal zone of the building\./); + expect(zone.memo).toHaveLength(485); + }); + + it('reports the field prose Python reports, when a pool is supplied', async () => { + const prose = await loadProse(); + const zone = describeObjectType(v26, 'Zone', prose); + const ceilingHeight = zone.fields.find((f) => f.name === 'ceiling_height'); + + expect(ceilingHeight?.note).toMatch(/^If this field is 0\.0, negative or autocalculate/); + expect(ceilingHeight?.note).toHaveLength(352); + }); + + /** + * SC-005, the whole of it: every type in the version, every field, memo and + * note, against a digest taken from Python. + * + * Regenerate from the idfkit checkout with: + * + * ```py + * import hashlib + * from idfkit import get_schema, LATEST_VERSION + * from idfkit.introspection import describe_object_type + * s = get_schema(LATEST_VERSION) + * parts = [] + * for name in sorted(s.object_types): + * d = describe_object_type(s, name) + * parts.append(f"{name}|{d.memo or ''}|" + ",".join((f.note or '') for f in d.fields)) + * print(hashlib.sha256("\n".join(parts).encode()).hexdigest()) + * ``` + * + * This is the assertion that makes the prose closure real. One sentence + * matching proves the pool is wired; 858 types matching proves it is the + * right pool, indexed correctly, for every record in the version. + */ + it('matches Python on every memo and note in the version', async () => { + const prose = await loadProse(); + const lines = [...v26.typeNames].sort().map((name) => { + const d = describeObjectType(v26, name, prose); + const notes = d.fields.map((f) => f.note ?? '').join(','); + return `${name}|${d.memo ?? ''}|${notes}`; + }); + + const digest = createHash('sha256').update(lines.join('\n')).digest('hex'); + expect(digest).toBe('dd4edd387ac4a365192cc6ba0b831e923d9f125ade8be2b9265d80c7d656800a'); + }); + + /** + * Acceptance scenario 2: where the schema carries no prose, both languages + * report its absence rather than filling it with a placeholder. + * + * 13 of the 858 types in 26.1.0 have no memo. The pool cannot express "no + * prose" as a string, so a type with none carries no index at all, and the + * lookup returns undefined for the same reason it does with no pool. + */ + it('reports absent prose as absent, even with a pool supplied', async () => { + const prose = await loadProse(); + const all = describeAll(v26, prose); + + const withoutMemo = all.filter((d) => d.memo === undefined); + expect(withoutMemo.length).toBeGreaterThan(0); + // Absent, never an empty string and never a fabricated stand-in. + expect(withoutMemo.every((d) => d.memo === undefined)).toBe(true); + + const notes = all.flatMap((d) => d.fields.map((f) => f.note)); + expect(notes.some((n) => n === undefined)).toBe(true); + expect(notes.every((n) => n === undefined || n.length > 0)).toBe(true); + }); + + it('keeps the blank that an enum declares', () => { + // Not `do_zone_sizing_calculation`, which BOTH sides drop: SimulationControl + // is anonymous, so the positional `[1:]` slice eats its first real field. + // That is a separate, already-recorded divergence and not this one. + const control = describeObjectType(v26, 'SimulationControl'); + const doSystemSizing = control.fields.find((f) => f.name === 'do_system_sizing_calculation'); + + expect(doSystemSizing?.enumValues).toEqual(['', 'No', 'Yes']); + }); + + it('reports the sentinels held in the collapsed anyOf string branch', () => { + const layer = describeObjectType(v26, 'WindowMaterial:Glazing:EquivalentLayer'); + const transmittance = layer.fields.find( + (f) => f.name === 'diffuse_diffuse_solar_transmittance' + ); + + expect(transmittance?.enumValues).toEqual(['', 'Autocalculate']); + }); + + /** + * The three types whose positional field list holds only the name, in every + * one of the 17 bundled versions. Two of them diverged from Python; the third, + * SolarCollector, agreed by alphabetical luck and is fixed anyway, because a + * schema edit renaming either field would otherwise break it in silence. + */ + it('puts the name first for ZoneProperty:UserViewFactors:BySurfaceName', () => { + expect( + describeObjectType(v26, 'ZoneProperty:UserViewFactors:BySurfaceName').fields.map( + (f) => f.name + ) + ).toEqual([ + 'zone_or_zonelist_or_space_or_spacelist_name', + 'view_factors', + 'from_surface', + 'to_surface', + 'view_factor', + ]); + }); + + it('puts the name first for ZoneTerminalUnitList', () => { + expect(describeObjectType(v26, 'ZoneTerminalUnitList').fields.map((f) => f.name)).toEqual([ + 'zone_terminal_unit_list_name', + 'terminal_units', + 'zone_terminal_unit_name', + ]); + }); + + it('leaves SolarCollector:UnglazedTranspired:Multisystem where it already was', () => { + expect( + describeObjectType(v26, 'SolarCollector:UnglazedTranspired:Multisystem').fields.map( + (f) => f.name + ) + ).toEqual([ + 'solar_collector_name', + 'systems', + 'outdoor_air_system_collector_inlet_node', + 'outdoor_air_system_collector_outlet_node', + 'outdoor_air_system_mixed_air_node', + 'outdoor_air_system_zone_node', + ]); + }); +}); diff --git a/packages/core/tests/lexer.test.ts b/packages/core/tests/lexer.test.ts index 0238568..90694c1 100644 --- a/packages/core/tests/lexer.test.ts +++ b/packages/core/tests/lexer.test.ts @@ -4,7 +4,9 @@ import { lex, type LexDiagnostic } from '@idfkit/core'; describe('lex', () => { it('reads a single-line object', () => { - expect(lex('Version, 26.1;')).toEqual([{ typeName: 'Version', values: ['26.1'], line: 1 }]); + expect(lex('Version, 26.1;')).toEqual([ + { typeName: 'Version', values: ['26.1'], line: 1, column: 1, offset: 0 }, + ]); }); it('reads a multi-line object with field comments', () => { @@ -15,7 +17,9 @@ describe('lex', () => { ' 1.5; !- X Origin', ].join('\n'); - expect(lex(text)).toEqual([{ typeName: 'Zone', values: ['Zone One', '0', '1.5'], line: 1 }]); + expect(lex(text)).toEqual([ + { typeName: 'Zone', values: ['Zone One', '0', '1.5'], line: 1, column: 1, offset: 0 }, + ]); }); it('ignores full-line comments between objects', () => { @@ -64,6 +68,43 @@ describe('lex', () => { const first = lex('Zone,\n Z1,\n 4.0'); const second = lex('Version, 26.1;'); expect(first).toEqual([]); - expect(second).toEqual([{ typeName: 'Version', values: ['26.1'], line: 1 }]); + expect(second).toEqual([ + { typeName: 'Version', values: ['26.1'], line: 1, column: 1, offset: 0 }, + ]); + }); +}); + +/** + * FR-033: a finding carries the same kinds of location in both languages. + * + * The column is the first NON-BLANK character of the object, not the start of the field text. + * Python's `_OBJECT_PATTERN` matches the type name itself and reports `match.start(1)`, so an + * indented object reports the indent width plus one there; reporting the padding here instead + * would have the corpus comparing two different notions of position. + */ +describe('object position (FR-033)', () => { + it('reports the column of the type name, not of the indentation', () => { + const objects = lex('Version,\n 26.1;\n\n Zone,\n Z;\n'); + + expect(objects.map((o) => [o.line, o.column])).toEqual([ + [1, 1], + [4, 4], + ]); + }); + + it('counts the column from the start of its own line', () => { + const objects = lex('Version, 26.1;\n\n\n Building, B;\n'); + + expect(objects[1]?.line).toBe(4); + expect(objects[1]?.column).toBe(6); + }); + + it('gives an unterminated object a position too', () => { + const seen: { line: number; column?: number }[] = []; + lex('Version, 26.1;\n\n Zone,\n Unfinished', { + onDiagnostic: (d) => seen.push({ line: d.line, column: d.column }), + }); + + expect(seen).toEqual([{ line: 3, column: 3 }]); }); }); diff --git a/packages/core/tests/parse.test.ts b/packages/core/tests/parse.test.ts index 67deb25..cbcac8f 100644 --- a/packages/core/tests/parse.test.ts +++ b/packages/core/tests/parse.test.ts @@ -155,3 +155,160 @@ describe('parseEpJson', () => { expect(() => parseEpJson('{not json', v26)).toThrow(/Invalid JSON/); }); }); + +/** + * Feature 002, US3: what a fatal parse carries, and what it must keep carrying. + * + * The record described this gap as one-sided, "Python raises and TypeScript returns". Neither + * library ever did that: `parseIdf` defaults to `strict: true` and throws, `parse_idf` defaults to + * `strict_parsing=True` and raises. What actually differed is that this error carried one finding + * flattened into two fields while Python's carried the whole collection. + */ +describe('feature 002, a fatal parse carries its findings', () => { + it('throws by default, which it always did', async () => { + const s = await schema('26.1.0'); + + expect(() => parseIdf('Version, 26.1;\nNotARealType, x;\n', s)).toThrow(IdfParseError); + }); + + it('carries the findings as a collection', async () => { + const s = await schema('26.1.0'); + + let error: IdfParseError | undefined; + try { + parseIdf('Version, 26.1;\nNotARealType, x;\n', s); + } catch (caught) { + error = caught as IdfParseError; + } + + expect(error).toBeInstanceOf(IdfParseError); + expect(error?.diagnostics).toHaveLength(1); + expect(error?.diagnostics[0]?.code).toBe('UnknownObjectType'); + expect(error?.diagnostics[0]?.typeName).toBe('NotARealType'); + }); + + it('keeps the flattened accessors resolving to the first finding', async () => { + const s = await schema('26.1.0'); + + let error: IdfParseError | undefined; + try { + parseIdf('Version, 26.1;\nNotARealType, x;\n', s); + } catch (caught) { + error = caught as IdfParseError; + } + + // FR-014: `.line` and `.typeName` are what existing callers read, and they still return what + // they returned before. They are a convenience over `diagnostics[0]`, not a second truth. + expect(error?.line).toBe(error?.diagnostics[0]?.line); + expect(error?.typeName).toBe(error?.diagnostics[0]?.typeName); + expect(error?.message).toContain('NotARealType'); + }); + + it('gives every returned finding a code from the shared vocabulary', async () => { + const s = await schema('26.1.0'); + + const result = parseIdf('Version, 26.1;\nNotARealType, x;\nAlsoNotReal, y;\n', s, { + strict: false, + }); + + // One finding per skip, not one per distinct type name, and each carries a code the corpus + // can compare. Message text is deliberately not asserted: it is a presentation choice. + expect(result.diagnostics).toHaveLength(2); + expect(result.diagnostics.map((d) => d.code)).toEqual([ + 'UnknownObjectType', + 'UnknownObjectType', + ]); + expect(result.diagnostics.map((d) => d.typeName)).toEqual(['NotARealType', 'AlsoNotReal']); + expect(result.document.has('Version')).toBe(true); + }); +}); + +/** + * idfkit-js#36: a value of the wrong kind is reported rather than stored in silence. + * + * The shape this catches is a missing semicolon swallowing the object below, which slides that + * object's type name into a numeric field. The field count still fits, so nothing overflows and no + * parser notices by counting. + */ +describe('InvalidField diagnostics', () => { + const SWALLOWED = + 'Version,\n 26.1;\n\nBuilding,\n Conformance,\n 0,\n ,\n ,\n ,\n ,\nTimestep,\n 4;\n'; + + it('reports the wrong kind of value, at the field rather than the object', async () => { + const s = await schema('26.1.0'); + + const result = parseIdf(SWALLOWED, s, { strict: false }); + const invalid = result.diagnostics.filter((d) => d.code === 'InvalidField'); + + // Line 11 is the swallowed `Timestep,`. Line 4 is where Building starts, which would be true + // and useless: the damage is seven lines further down. + expect(invalid).toHaveLength(1); + expect(invalid[0]?.line).toBe(11); + expect(invalid[0]?.typeName).toBe('Building'); + }); + + it('does not stop a strict parse', async () => { + // FR-014. A value of the wrong KIND still leaves a complete document, unlike an unknown type, + // so this finding is recorded rather than thrown even under `strict`. + const s = await schema('26.1.0'); + + expect(() => parseIdf(SWALLOWED, s)).not.toThrow(); + expect(parseIdf(SWALLOWED, s).diagnostics.some((d) => d.code === 'InvalidField')).toBe(true); + }); + + it('accepts a sizing sentinel in any numeric field', async () => { + // The check that keeps this diagnostic worth reading. Reading each field's own string branch + // literally produced 3,775 findings across the 760 EnergyPlus example files of one release, + // every one against a model EnergyPlus accepts: the schema is narrower than the engine. + const s = await schema('26.1.0'); + const text = + 'Version,\n 26.1;\n\nPlantLoop,\n Loop,\n Water,\n ,\n ,\n ,\n ,\n ,\n ,\n Autosize;\n'; + + const result = parseIdf(text, s, { strict: false }); + + expect(result.diagnostics.filter((d) => d.code === 'InvalidField')).toEqual([]); + }); + + it('accepts a sentinel whatever its case', async () => { + const s = await schema('26.1.0'); + const text = + 'Version,\n 26.1;\n\nPeople,\n P,\n Z,\n Sched,\n People,\n 1,\n ,\n ,\n AUTOCALCULATE;\n'; + + const result = parseIdf(text, s, { strict: false }); + + expect(result.diagnostics.filter((d) => d.code === 'InvalidField')).toEqual([]); + }); +}); + +/** + * Found by sweeping the EnergyPlus example files in both languages and comparing the output: the + * two agreed on the file and the field and disagreed on the line by one. + * + * A field's comma is routinely followed by `!- Field Name` on the same line. Stopping the scan at + * the `!` reports the line the PREVIOUS value sits on. The conformance case for this diagnostic did + * not catch it, because its input has no comments. + */ +describe('field position with comments between the fields', () => { + it('reports the line the value is on, not the line the comment is on', async () => { + const s = await schema('26.1.0'); + const text = [ + 'Version,', + ' 26.1;', + '', + 'Material,', + ' IN46, !- Name', + ' VeryRough, !- Roughness', + ' NotANumber, !- Thickness {m}', + ' 2.3; !- Conductivity {W/m-K}', + '', + ].join('\n'); + + const invalid = parseIdf(text, s, { strict: false }).diagnostics.filter( + (d) => d.code === 'InvalidField' + ); + + // `NotANumber` is on line 7. Line 6 is the comment-bearing line above it. + expect(invalid).toHaveLength(1); + expect(invalid[0]?.line).toBe(7); + }); +}); diff --git a/packages/core/tests/write.test.ts b/packages/core/tests/write.test.ts index 20380d8..42aaf16 100644 --- a/packages/core/tests/write.test.ts +++ b/packages/core/tests/write.test.ts @@ -110,3 +110,212 @@ describe('writeEpJson', () => { expect(JSON.parse(writeEpJson(doc))).toEqual({}); }); }); + +/** + * FR-017: no writer default moves, in either language. + * + * The requirement most easily broken by accident. Feature 002 adds a compressed mode here and + * three controls on the other side, and the only thing that makes a slipped default loud rather + * than discovered later is a test that fails. + * + * Six defaults, pinned as the values they are TODAY. Every one is also pinned on the other side, + * in `idfkit/tests/test_writers.py`, at the values THAT writer uses. The two disagree on five of + * the six, both are published, and neither is more correct. That disagreement is the point: it is + * documented on a page rather than resolved, because resolving it would change output somebody + * depends on. + */ +describe('writer defaults are pinned (FR-017)', () => { + const model = (s: Schema): IdfDocument => { + const { document } = parseIdf( + 'Version,\n 26.1;\n\nBuilding,\n Pinned,\n 30.0;\n\nTimestep,\n 4;\n', + s + ); + return document; + }; + + it('indents four spaces', () => { + const text = writeIdf(model(v26)); + + const fieldLines = text.split('\n').filter((l) => l.startsWith(' ') && l.includes('!-')); + expect(fieldLines.length).toBeGreaterThan(0); + // Four, where the other language writes two. + expect(fieldLines.every((l) => l.startsWith(' ') && !l.startsWith(' '))).toBe(true); + }); + + it('puts the comment at column 30', () => { + const text = writeIdf(model(v26)); + + let checked = 0; + for (const line of text.split('\n')) { + const marker = line.indexOf('!-'); + if (marker <= 0) continue; + // Only lines the padding actually reached: a value longer than the column pushes the comment + // right, and that overflow behaviour is itself one of the seven differences. + if (line.slice(0, marker).trimEnd().length < 30) { + expect(marker).toBe(30); + checked += 1; + } + } + expect(checked).toBeGreaterThan(0); + }); + + it('writes objects in insertion order, with Version first', () => { + const text = writeIdf(model(v26)); + + const types = text + .split('\n') + .filter((l) => l.length > 0 && !/^\s/.test(l) && l.endsWith(',')) + .map((l) => l.slice(0, -1)); + // Insertion order, not sorted: the other language sorts by type name. + expect(types).toEqual(['Version', 'Building', 'Timestep']); + }); + + it('consults the schema when rendering a float', () => { + const text = writeIdf(model(v26)); + + // north_axis came in as 30.0 and stays 30.0, because the schema calls the field a number. The + // other language renders every float with %g and writes 30. + expect(text).toContain('30.0'); + }); + + it('lowercases minor words in a field comment', () => { + const text = writeIdf(model(v26)); + + // "Number of Timesteps per Hour", where the other language title-cases every word and writes + // "Number Of Timesteps Per Hour". + expect(text).toMatch(/!- Number of Timesteps per Hour/); + }); + + it('writes no generator header', () => { + const text = writeIdf(model(v26)); + + // The other language opens every file with "!-Generator idfkit v..." and "!-Option + // SortedOrder". This writes neither, which is the first difference a reader diffing two + // outputs would meet. + expect(text.startsWith('!-')).toBe(false); + expect(text).not.toContain('!-Generator'); + expect(text).not.toContain('!-Option'); + }); +}); + +/** + * FR-016 and SC-007: the fifth and last control to close. + * + * Four of the five controls existed on one writer and were added to the other. This is the one + * that went the other way: the other language has had `output_type="compressed"` since it was + * written, and this writer had no compact path at all to extend. + */ +describe('compressed output (FR-016)', () => { + const model = (s: Schema): IdfDocument => { + const { document } = parseIdf('Version,\n 26.1;\n\nBuilding,\n Ctl,\n 30.0;\n', s); + return document; + }; + + it('puts each object on one line, with no blank line between them', () => { + const text = writeIdf(model(v26), { compressed: true }); + + expect(text).toBe('Version,26.1;\nBuilding,Ctl,30.0;'); + }); + + it('means the same thing as the other language’s compressed', () => { + // Python writes 'Version,26.1;\nBuilding,Ctl,30;' for the same input. The two agree on + // structure and differ only on float rendering, which is one of the six pinned defaults and is + // not something compressed removes. That is why the corpus compares this structurally, by + // re-reading both outputs, rather than textually. + const text = writeIdf(model(v26), { compressed: true }); + const lines = text.split('\n'); + + expect(lines).toHaveLength(2); + expect(lines.every((l) => l.endsWith(';'))).toBe(true); + expect(lines.every((l) => !l.includes('!-'))).toBe(true); + expect(text).not.toContain('!-Generator'); + }); + + it('is not the same as turning comments off', () => { + // `comments: false` skips the padding and the comment and still puts every field on its own + // line. Both languages already had that; it is a different, coarser output. + const withoutComments = writeIdf(model(v26), { comments: false }); + const compressed = writeIdf(model(v26), { compressed: true }); + + expect(withoutComments.split('\n').length).toBeGreaterThan(compressed.split('\n').length); + expect(withoutComments).toContain('\n 26.1;'); + }); + + it('re-reads to the same document it came from (FR-019)', () => { + const original = model(v26); + const reread = parseIdf(writeIdf(original, { compressed: true }), v26).document; + + // Structure survives the control, which is what FR-019 asks. Compared over parsed values + // rather than text, because the text is deliberately different. + expect(reread.types().sort()).toEqual(original.types().sort()); + expect(reread.all('Building').size).toBe(original.all('Building').size); + }); + + it('does not add a lossless mode', () => { + // `lossless-round-trip` is a separate Tier 2 entry on the parity record, absent here and + // tracked as not-yet-ported. Compressed is not a step toward it and must not be read as one: + // it discards MORE formatting, not less. + const options: Record = { compressed: true }; + expect(Object.keys(options)).not.toContain('preserveFormatting'); + expect(writeIdf(model(v26), { compressed: true })).not.toContain(' 26.1'); + }); +}); + +/** + * FR-016 and SC-007, the last two controls. + * + * Feature 002 closed five controls and left two spelled on one side only: this writer had + * `versionFirst` and no `ordering`, the other had `ordering` and no way to unpin Version. SC-007 + * asks for zero one-sided controls and US4's second acceptance scenario names object ordering + * explicitly, so both were added rather than argued away. + */ +describe('ordering (FR-016, SC-007)', () => { + const model = (s: Schema): IdfDocument => { + const { document } = parseIdf('Version,\n 26.1;\n\nTimestep,\n 4;\n\nBuilding,\n Ctl;\n', s); + return document; + }; + + const typeNames = (text: string): string[] => + text + .split('\n') + .filter((l) => l.length > 0 && !/^\s/.test(l) && l.endsWith(',')) + .map((l) => l.slice(0, -1)); + + it('defaults to source order, which is what this writer always did', () => { + // FR-017: the default does not move. Timestep before Building is the document's own order, + // not the alphabetical one. + expect(typeNames(writeIdf(model(v26)))).toEqual(['Version', 'Timestep', 'Building']); + }); + + it('sorts by type name when asked', () => { + expect(typeNames(writeIdf(model(v26), { ordering: 'sorted' }))).toEqual([ + 'Version', + 'Building', + 'Timestep', + ]); + }); + + it('changes the output, so a corpus case using it is not a no-op', () => { + // The reason this control had to exist rather than be argued away: `writer-option-ordering` + // would otherwise pass on this side even if the runner dropped the option entirely. + expect(writeIdf(model(v26), { ordering: 'sorted' })).not.toBe(writeIdf(model(v26))); + }); + + it('composes with versionFirst rather than overriding it', () => { + // Sorted decides the type order; versionFirst pins Version above it. Turning the pin off + // leaves Version in whichever position the ordering gives it. + expect(typeNames(writeIdf(model(v26), { ordering: 'sorted', versionFirst: false }))).toEqual([ + 'Building', + 'Timestep', + 'Version', + ]); + }); + + it('re-reads to the same document under either ordering (FR-019)', () => { + const original = model(v26); + for (const ordering of ['sorted', 'source'] as const) { + const reread = parseIdf(writeIdf(original, { ordering }), v26).document; + expect(reread.types().sort()).toEqual(original.types().sort()); + } + }); +}); diff --git a/packages/schemas/data/docs.json.gz b/packages/schemas/data/docs.json.gz new file mode 100644 index 0000000..bb9d256 Binary files /dev/null and b/packages/schemas/data/docs.json.gz differ diff --git a/packages/schemas/data/manifest-22-1-0.json.gz b/packages/schemas/data/manifest-22-1-0.json.gz index de5df52..d6630d6 100644 Binary files a/packages/schemas/data/manifest-22-1-0.json.gz and b/packages/schemas/data/manifest-22-1-0.json.gz differ diff --git a/packages/schemas/data/manifest-22-2-0.json.gz b/packages/schemas/data/manifest-22-2-0.json.gz index 798ca3e..a6d37c2 100644 Binary files a/packages/schemas/data/manifest-22-2-0.json.gz and b/packages/schemas/data/manifest-22-2-0.json.gz differ diff --git a/packages/schemas/data/manifest-23-1-0.json.gz b/packages/schemas/data/manifest-23-1-0.json.gz index 23dc834..de2923d 100644 Binary files a/packages/schemas/data/manifest-23-1-0.json.gz and b/packages/schemas/data/manifest-23-1-0.json.gz differ diff --git a/packages/schemas/data/manifest-23-2-0.json.gz b/packages/schemas/data/manifest-23-2-0.json.gz index 2b0d531..03fa3bd 100644 Binary files a/packages/schemas/data/manifest-23-2-0.json.gz and b/packages/schemas/data/manifest-23-2-0.json.gz differ diff --git a/packages/schemas/data/manifest-24-1-0.json.gz b/packages/schemas/data/manifest-24-1-0.json.gz index 689e402..6a36f9a 100644 Binary files a/packages/schemas/data/manifest-24-1-0.json.gz and b/packages/schemas/data/manifest-24-1-0.json.gz differ diff --git a/packages/schemas/data/manifest-24-2-0.json.gz b/packages/schemas/data/manifest-24-2-0.json.gz index 8ac0834..4646967 100644 Binary files a/packages/schemas/data/manifest-24-2-0.json.gz and b/packages/schemas/data/manifest-24-2-0.json.gz differ diff --git a/packages/schemas/data/manifest-25-1-0.json.gz b/packages/schemas/data/manifest-25-1-0.json.gz index 9103eec..146722f 100644 Binary files a/packages/schemas/data/manifest-25-1-0.json.gz and b/packages/schemas/data/manifest-25-1-0.json.gz differ diff --git a/packages/schemas/data/manifest-25-2-0.json.gz b/packages/schemas/data/manifest-25-2-0.json.gz index 3449883..0d3aafd 100644 Binary files a/packages/schemas/data/manifest-25-2-0.json.gz and b/packages/schemas/data/manifest-25-2-0.json.gz differ diff --git a/packages/schemas/data/manifest-26-1-0.json.gz b/packages/schemas/data/manifest-26-1-0.json.gz index decd562..cf85569 100644 Binary files a/packages/schemas/data/manifest-26-1-0.json.gz and b/packages/schemas/data/manifest-26-1-0.json.gz differ diff --git a/packages/schemas/data/manifest-8-9-0.json.gz b/packages/schemas/data/manifest-8-9-0.json.gz index 249b757..90bd473 100644 Binary files a/packages/schemas/data/manifest-8-9-0.json.gz and b/packages/schemas/data/manifest-8-9-0.json.gz differ diff --git a/packages/schemas/data/manifest-9-0-1.json.gz b/packages/schemas/data/manifest-9-0-1.json.gz index 03acd97..e350498 100644 Binary files a/packages/schemas/data/manifest-9-0-1.json.gz and b/packages/schemas/data/manifest-9-0-1.json.gz differ diff --git a/packages/schemas/data/manifest-9-1-0.json.gz b/packages/schemas/data/manifest-9-1-0.json.gz index 4c25d6b..fff38a0 100644 Binary files a/packages/schemas/data/manifest-9-1-0.json.gz and b/packages/schemas/data/manifest-9-1-0.json.gz differ diff --git a/packages/schemas/data/manifest-9-2-0.json.gz b/packages/schemas/data/manifest-9-2-0.json.gz index 007f92f..cfcacb6 100644 Binary files a/packages/schemas/data/manifest-9-2-0.json.gz and b/packages/schemas/data/manifest-9-2-0.json.gz differ diff --git a/packages/schemas/data/manifest-9-3-0.json.gz b/packages/schemas/data/manifest-9-3-0.json.gz index c21be06..7fd4d50 100644 Binary files a/packages/schemas/data/manifest-9-3-0.json.gz and b/packages/schemas/data/manifest-9-3-0.json.gz differ diff --git a/packages/schemas/data/manifest-9-4-0.json.gz b/packages/schemas/data/manifest-9-4-0.json.gz index ae56214..eb698cb 100644 Binary files a/packages/schemas/data/manifest-9-4-0.json.gz and b/packages/schemas/data/manifest-9-4-0.json.gz differ diff --git a/packages/schemas/data/manifest-9-5-0.json.gz b/packages/schemas/data/manifest-9-5-0.json.gz index 94dd5cd..af6d903 100644 Binary files a/packages/schemas/data/manifest-9-5-0.json.gz and b/packages/schemas/data/manifest-9-5-0.json.gz differ diff --git a/packages/schemas/data/manifest-9-6-0.json.gz b/packages/schemas/data/manifest-9-6-0.json.gz index afd17fc..b75f116 100644 Binary files a/packages/schemas/data/manifest-9-6-0.json.gz and b/packages/schemas/data/manifest-9-6-0.json.gz differ diff --git a/packages/schemas/data/types.json.gz b/packages/schemas/data/types.json.gz index f1c6439..da855cd 100644 Binary files a/packages/schemas/data/types.json.gz and b/packages/schemas/data/types.json.gz differ diff --git a/packages/schemas/scripts/build.mjs b/packages/schemas/scripts/build.mjs index 5897bd4..a8256c0 100644 --- a/packages/schemas/scripts/build.mjs +++ b/packages/schemas/scripts/build.mjs @@ -46,7 +46,7 @@ function versionKey(v) { * Collapse a raw epJSON object-type definition into the slim form. * Everything dropped here is documentation metadata, not parsing metadata. */ -function slimType(raw) { +function slimType(raw, prose) { const legacy = raw.legacy_idd ?? {}; const out = {}; @@ -56,7 +56,32 @@ function slimType(raw) { const props = body.properties ?? {}; const slimProps = {}; for (const [fieldName, def] of Object.entries(props)) { - slimProps[fieldName] = slimField(def); + slimProps[fieldName] = slimField(def, prose); + } + + // The type's explanatory sentence, as an index into the shared pool. One + // integer per record, not the string: 935 distinct memos are shared across + // roughly 14,600 type definitions in the 17 versions, so storing the text + // inline would pay for the same sentence dozens of times. + const memo = prose.ref(raw.memo); + if (memo !== undefined) out.m = memo; + + // `f` holds only the name for exactly three types, in every bundled version: + // ZoneProperty:UserViewFactors:BySurfaceName (spelled bySurfaceName in 8.9.0 + // through 9.3.0), ZoneTerminalUnitList and SolarCollector:UnglazedTranspired: + // Multisystem. `orderedFieldNames` then falls back to the key order of `p`, + // which `canonical()` has sorted for content-addressing, so the reader gets + // alphabetical order where Python gives declaration order. + // + // `fo` records the declaration order the sort is about to destroy. Emitted + // only where the fallback would otherwise be reached and the answer would + // differ, which is why it costs three entries rather than 858. + // + // Two of the three actually diverge. SolarCollector's two fixed fields sort + // into declaration order by luck; it gets an `fo` anyway, because a schema + // edit that renames either field would otherwise break it in silence. + if (out.f.length <= 1 && Object.keys(props).length > 1) { + out.fo = Object.keys(props); } out.p = slimProps; if (body.required?.length) out.r = body.required; @@ -78,7 +103,7 @@ function slimType(raw) { const items = props[legacy.extension]?.items?.properties ?? {}; const inner = {}; for (const fieldName of legacy.extensibles) { - inner[fieldName] = slimField(items[fieldName] ?? {}); + inner[fieldName] = slimField(items[fieldName] ?? {}, prose); } out.x = { key: legacy.extension, fields: legacy.extensibles, p: inner }; } @@ -87,7 +112,7 @@ function slimType(raw) { return out; } -function slimField(def) { +function slimField(def, prose) { const out = {}; // `anyOf` is always "a number, or a string", in that branch order, in all 17 @@ -135,7 +160,21 @@ function slimField(def) { if (effective.object_list?.length) out.ol = effective.object_list; if (effective.reference?.length) out.ref = effective.reference; - if (effective.enum?.length) out.e = effective.enum.filter((v) => v !== ''); + // The blank is a legal value for 21,962 enum-bearing fields across the 17 + // versions, and Python reports it. It is still filtered out of `e` rather + // than kept, because `e` is what validation checks against and admitting '' + // there would change what validate() accepts, which this feature does not do + // (FR-014). `eb` carries it alongside instead, and the description path puts + // it back. + // + // A flag is enough because the blank is at index 0 every single time: + // measured across all 17 schemas, all 21,962 of them, the distribution of + // its index is {0: 21962}. If that ever stops being true this must become an + // index, and the bundle test asserts the position so the change is loud. + if (effective.enum?.length) { + out.e = effective.enum.filter((v) => v !== ''); + if (effective.enum.includes('')) out.eb = 1; + } if (effective.default !== undefined) out.d = effective.default; if (effective.minimum !== undefined) out.min = effective.minimum; if (effective.maximum !== undefined) out.max = effective.maximum; @@ -144,10 +183,74 @@ function slimField(def) { if (effective.units) out.u = effective.units; if (effective.retaincase) out.rc = 1; + // The field's explanatory sentence, as an index into the shared pool. + const note = prose.ref(def.note); + if (note !== undefined) out.n = note; + if (out.e && out.e.length === 0) delete out.e; return out; } +/** + * The prose pool: every distinct `memo` and `note` string in every bundled + * schema, stored once, addressed by index. + * + * Two passes are needed and the reason is the content addressing. A blob's key + * is the hash of its canonical text, so the index a record carries has to be + * decided before any record is hashed, and it has to be the same index on + * every rebuild or the whole bundle churns. So pass one collects the strings + * and sorts them, which fixes the numbering, and pass two builds the records. + * + * Sorted rather than first-encountered for two reasons: it does not depend on + * the order the schema directories happen to be read in, and it puts similar + * sentences next to each other, which is worth a few percent to gzip. + */ +function createProsePool() { + const strings = new Set(); + let index = null; + + return { + /** Pass one: remember a string. */ + collect(text) { + if (typeof text === 'string' && text.length > 0) strings.add(text); + }, + /** Freeze the numbering. Everything after this point resolves against it. */ + freeze() { + const sorted = [...strings].sort(); + index = new Map(sorted.map((text, position) => [text, position])); + return sorted; + }, + /** Pass two: the index for a string, or undefined when there is no prose. */ + ref(text) { + if (index === null) return undefined; + if (typeof text !== 'string' || text.length === 0) return undefined; + return index.get(text); + }, + }; +} + +/** + * Walk one raw schema and hand every prose string to the pool. Mirrors the + * shape `slimType` and `slimField` read, so a string reachable by one is + * reachable by the other. + */ +function collectProse(schema, prose) { + for (const rawType of Object.values(schema.properties ?? {})) { + prose.collect(rawType.memo); + + const body = Object.values(rawType.patternProperties ?? {})[0] ?? {}; + const props = body.properties ?? {}; + for (const def of Object.values(props)) prose.collect(def.note); + + // The extensible half lives on the array's `items`, and carries notes too. + const legacy = rawType.legacy_idd ?? {}; + if (legacy.extension && legacy.extensibles?.length) { + const items = props[legacy.extension]?.items?.properties ?? {}; + for (const def of Object.values(items)) prose.collect(def?.note); + } + } +} + function pick(obj, keys) { const out = {}; for (const k of keys) if (obj[k] !== undefined) out[k] = obj[k]; @@ -183,21 +286,37 @@ function main() { rmSync(OUT, { recursive: true, force: true }); mkdirSync(OUT, { recursive: true }); + // Sorted so the build does not depend on the order the filesystem lists the + // directories in. The pool is sorted too, so this is belt and braces, but a + // content-addressed bundle that is rebuilt and diffed in CI cannot afford to + // be one readdir away from churning. + dirs.sort(); + + const readSchema = (dir) => + JSON.parse( + gunzipSync(readFileSync(join(SOURCE, dir, 'Energy+.schema.epJSON.gz'))).toString('utf8') + ); + + // Pass one: every distinct memo and note across every version, numbered. + const prose = createProsePool(); + for (const dir of dirs) collectProse(readSchema(dir), prose); + const proseStrings = prose.freeze(); + /** hash -> canonical JSON text of one object-type definition. */ const blobs = new Map(); const manifests = {}; const versions = []; + // Pass two: build the records, now that every prose index is decided. for (const dir of dirs) { const version = dirToVersion(dir); versions.push(version); - const gz = readFileSync(join(SOURCE, dir, 'Energy+.schema.epJSON.gz')); - const schema = JSON.parse(gunzipSync(gz).toString('utf8')); + const schema = readSchema(dir); const manifest = {}; for (const [typeName, rawType] of Object.entries(schema.properties ?? {})) { - const text = canonical(slimType(rawType)); + const text = canonical(slimType(rawType, prose)); const h = hash(text); if (!blobs.has(h)) blobs.set(h, text); manifest[typeName] = h; @@ -219,6 +338,16 @@ function main() { manifestFiles[version] = fileName; } + // The prose, in its own file under `data/`. + // + // A separate file, not a key in types.json, because the two are read at + // different times by different callers. Parsing a model needs the records and + // never needs a sentence of English; describing a type for a reader needs + // both. Keeping them apart is what lets the parse path stay unaware the pool + // exists, which `check-bundle-purity.mjs` asserts by building a minimal + // read-and-write graph and failing on any input under a `data/` directory. + writeBundleFile('docs.json', proseStrings); + writeBundleFile('index.json', { versions, manifests: manifestFiles }); const totalDefs = Object.values(manifests).reduce((n, m) => n + Object.keys(m).length, 0); @@ -229,6 +358,10 @@ function main() { console.log(`versions ${versions.length} (${versions[0]} .. ${versions.at(-1)})`); console.log(`type defs ${totalDefs}`); console.log(`unique defs ${blobs.size} (${((100 * blobs.size) / totalDefs).toFixed(1)}%)`); + const docsBytes = readFileSync(join(OUT, 'docs.json.gz')).length; + console.log( + `prose strings ${proseStrings.length} distinct (${(docsBytes / 1024).toFixed(1)} KB gzipped)` + ); console.log(`bundle on disk ${(gzTotal / 1024).toFixed(1)} KB gzipped`); } diff --git a/packages/schemas/src/types.ts b/packages/schemas/src/types.ts index bcd79ef..4258e5c 100644 --- a/packages/schemas/src/types.ts +++ b/packages/schemas/src/types.ts @@ -80,6 +80,28 @@ export interface SlimField { u?: string; /** Value is case-sensitive and must not be normalized. */ rc?: 1; + /** + * The enum also accepts the empty string. + * + * `e` is filtered of `''` because it is what validation checks against, and + * admitting the blank there would change what `validate()` accepts. Python's + * `describe_object_type` reports it, so the description path puts it back + * from this flag and validation never sees it. + * + * A flag rather than an index because the blank is first every time: across + * all 17 bundled schemas, all 21,962 blank-bearing enums carry it at position + * 0. `bundle.test.ts` asserts that, so a schema that breaks it fails loudly + * instead of silently reordering a choice list. + */ + eb?: 1; + /** + * The field's explanatory sentence, as an index into the prose pool. + * + * An index, not the text: 3,837 distinct field notes are shared across about + * 119,000 occurrences. Resolve it against `docs.json.gz`, which is loaded on + * demand and is never on the parse path. + */ + n?: number; } export interface SlimExtensible { @@ -110,6 +132,23 @@ export interface SlimType { x?: SlimExtensible; /** IDD group, e.g. `Thermal Zones and Surfaces`. */ g?: string; + /** + * The type's explanatory sentence, as an index into the prose pool. + * + * See `SlimField.n`. Present for 845 of 858 types in 26.1.0; a type with no + * memo in the source schema has none here, and both languages report nothing + * rather than a placeholder. + */ + m?: number; + /** + * Field names in declaration order, for the types where `f` cannot give it. + * + * `f` holds only the name for exactly three types in every bundled version, + * so the description path would otherwise fall back to the key order of `p` — + * which the content-addressing serializer has sorted alphabetically. This + * records the order that sort destroys, and is emitted only for those three. + */ + fo?: string[]; } /** A manifest maps object type name to a blob hash in the shared store. */ diff --git a/packages/schemas/tests/bundle.test.ts b/packages/schemas/tests/bundle.test.ts index 175dc0a..a549dbb 100644 --- a/packages/schemas/tests/bundle.test.ts +++ b/packages/schemas/tests/bundle.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { httpSource } from '@idfkit/schemas'; -import { localBundle } from '@idfkit/schemas/node'; +import { localBundle, nodeSource } from '@idfkit/schemas/node'; const bundle = localBundle(); @@ -215,3 +215,119 @@ describe('Schema', () => { expect(v26.has('Space')).toBe(true); }); }); + +/** + * Feature 002, US2: the prose pool, and the three field orders it ships beside. + * + * These assert the shape of the bundle rather than the description built from + * it, because a pool that is present but wrongly indexed produces prose that + * looks plausible and belongs to another field. + */ +describe('the prose pool', () => { + it('ships as its own file under data/', async () => { + const pool = (await nodeSource().read('docs.json')) as string[]; + + expect(Array.isArray(pool)).toBe(true); + expect(pool.length).toBeGreaterThan(4000); + }); + + it('holds every string once, and no empties', async () => { + const pool = (await nodeSource().read('docs.json')) as string[]; + + // Deduplication is the whole reason the pool is affordable: 4,878 distinct + // strings stand in for roughly 119,000 occurrences across the 17 versions. + // A duplicate means the interning broke and the file is paying twice. + expect(new Set(pool).size).toBe(pool.length); + expect(pool.every((s) => typeof s === 'string' && s.length > 0)).toBe(true); + }); + + it('is sorted, so the numbering does not depend on read order', async () => { + const pool = (await nodeSource().read('docs.json')) as string[]; + + // The index a record carries is decided before any record is hashed, and a + // content-addressed bundle cannot afford for that numbering to move. Sorted + // is what makes it reproducible on a different machine. + expect([...pool].sort()).toEqual(pool); + }); + + it('is reachable without going near the parse path', async () => { + // The pool is read through the ordinary bundle source, the same way the + // manifests and the type store are. There is no new export and no new + // entry point, so nothing on the read-and-write graph can reach it by + // accident. `check-bundle-purity.mjs` is the gate that proves the negative; + // this asserts the positive half, that a caller who wants it can have it. + const pool = await nodeSource().read('docs.json'); + + expect(pool).toBeDefined(); + }); + + it('is referenced by index from the type records, never inlined', async () => { + const store = (await nodeSource().read('types.json')) as Record; + const pool = (await nodeSource().read('docs.json')) as string[]; + + const records = Object.values(store) as { + m?: number; + p?: Record; + }[]; + + const withMemo = records.filter((r) => r.m !== undefined); + expect(withMemo.length).toBeGreaterThan(0); + // Every reference resolves. An index past the end would render as undefined + // prose, which reads exactly like a type that has none. + expect(withMemo.every((r) => r.m! >= 0 && r.m! < pool.length)).toBe(true); + + const noteRefs = records.flatMap((r) => Object.values(r.p ?? {}).map((f) => f.n)); + const present = noteRefs.filter((n): n is number => n !== undefined); + expect(present.length).toBeGreaterThan(0); + expect(present.every((n) => n >= 0 && n < pool.length)).toBe(true); + }); +}); + +describe('field order and accepted values in the bundle', () => { + /** + * T028: exactly three types need `fo`, and no fourth may silently join them. + * + * `fo` is emitted only where `f` holds at most the name and the type has more + * than one property, which is the condition under which the description path + * would otherwise fall back to the alphabetized key list. If a future schema + * adds a fourth such type, this fails and somebody decides deliberately, + * rather than a reader quietly getting the wrong field order. + */ + it('records declaration order for exactly three types, in every version', async () => { + const index = (await nodeSource().read('index.json')) as { versions: string[] }; + const store = (await nodeSource().read('types.json')) as Record; + + for (const version of index.versions) { + const manifest = (await nodeSource().read( + `manifest-${version.replace(/\./g, '-')}.json` + )) as Record; + + const withOrder = Object.entries(manifest) + .filter(([, hash]) => store[hash]?.fo !== undefined) + .map(([typeName]) => typeName) + .sort(); + + // `bySurfaceName` in 8.9.0 through 9.3.0, `BySurfaceName` after. + expect(withOrder.map((n) => n.toLowerCase())).toEqual([ + 'solarcollector:unglazedtranspired:multisystem', + 'zoneproperty:userviewfactors:bysurfacename', + 'zoneterminalunitlist', + ]); + } + }); + + it('flags the blank enum rather than admitting it into the validated list', async () => { + const store = (await nodeSource().read('types.json')) as Record< + string, + { p?: Record } + >; + + const fields = Object.values(store).flatMap((t) => Object.values(t.p ?? {})); + const flagged = fields.filter((f) => f.eb === 1); + + expect(flagged.length).toBeGreaterThan(0); + // The flag says the blank was there; `e` must still not contain it, because + // `e` is what validate() checks against and this feature does not move that. + expect(flagged.every((f) => !(f.e ?? []).includes(''))).toBe(true); + }); +}); diff --git a/scripts/sweep-example-files.mjs b/scripts/sweep-example-files.mjs new file mode 100644 index 0000000..1323bca --- /dev/null +++ b/scripts/sweep-example-files.mjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node +/** + * Parse every EnergyPlus example file for one release and report what the parser found. + * + * WHY THIS EXISTS + * + * The unit suite reads hand-written fixtures and, when an EnergyPlus install happens to be present, + * the example files of ONE release. That is not enough. Two classes of defect are only visible + * across releases: + * + * - A file whose content belongs to one version while its `Version` object declares another. The + * schema and the content then disagree, every value after the first added field lands one field + * early, and nothing says so. EnergyPlus ships several: `UnitarySystem_VSCoolingCoil_2.idf` in + * the 25.2 release declares `Version, 24.2` and uses a field added in 25.2. + * - A parser change that is safe on the newest schema and wrong on an older one, because field + * lists, extensible groups and sentinel spellings all moved between releases. + * + * Neither is reachable from a single version, which is why this sweeps them all. The Python + * library runs the same sweep over the same files, so a divergence between the two shows up as a + * different count on the same release. + * + * Usage: node scripts/sweep-example-files.mjs [--max-findings N] [--max-errors N] + */ + +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; + +import { getIdfVersion, parseIdf } from '../packages/core/dist/index.js'; +import { schemaFor } from '../packages/core/dist/node.js'; +import { localBundle } from '../packages/schemas/dist/node.js'; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + 'max-findings': { type: 'string', default: '0' }, + 'max-errors': { type: 'string', default: '0' }, + }, +}); + +const directory = positionals[0]; +if (directory === undefined) { + console.error('::error::usage: sweep-example-files.mjs [--max-findings N] [--max-errors N]'); + process.exit(2); +} +if (!statSync(directory, { throwIfNoEntry: false })?.isDirectory()) { + console.error(`::error::${directory} is not a directory`); + process.exit(2); +} + +const files = readdirSync(directory) + .filter((name) => name.endsWith('.idf')) + .sort(); + +if (files.length === 0) { + // An empty sweep passes every threshold while proving nothing, which is the one outcome this + // must never report as success. + console.error(`::error::no .idf files under ${directory}; the sweep proved nothing`); + process.exit(2); +} + +const bundle = localBundle(); +/** Schemas are shared across files, and loading one per file would dominate the run. */ +const schemas = new Map(); + +const errors = []; +const findings = []; +const byCode = new Map(); + +for (const name of files) { + try { + const text = readFileSync(join(directory, name)).toString('latin1'); + const version = getIdfVersion(text); + if (version === undefined) { + errors.push(`${name}: no Version object found`); + continue; + } + // `schemaFor`, not `bundle.load`, because that is the path `loadIdf` takes and it is the one + // that resolves a declared version onto a bundled one. Loading directly demands an exact + // match, so every file declaring `9.0` was reported unreadable against a bundle carrying + // 9.0.1: a defect in this script that looked exactly like one in the library. + if (!schemas.has(version)) schemas.set(version, await schemaFor(version, { bundle })); + + // strict off, because the point is to collect what a file reports rather than to stop at the + // first thing wrong with it. + const result = parseIdf(text, schemas.get(version), { strict: false }); + for (const diagnostic of result.diagnostics) { + byCode.set(diagnostic.code ?? 'none', (byCode.get(diagnostic.code ?? 'none') ?? 0) + 1); + findings.push(`${name}:${diagnostic.line}: [${diagnostic.code}] ${diagnostic.message}`); + } + } catch (error) { + // A file that will not read at all is the louder problem. + errors.push(`${name}: ${error instanceof Error ? `${error.name}: ${error.message}` : String(error)}`); + } +} + +console.log(`files read ${files.length}`); +console.log(`unreadable ${errors.length}`); +console.log(`diagnostics ${findings.length}`); +for (const [code, count] of [...byCode.entries()].sort()) { + console.log(` ${code.padEnd(20)} ${count}`); +} + +if (errors.length > 0) { + console.log('\nunreadable files:'); + for (const line of errors.slice(0, 40)) console.log(` ${line}`); +} +if (findings.length > 0) { + console.log('\ndiagnostics:'); + for (const line of findings.slice(0, 60)) console.log(` ${line}`); + if (findings.length > 60) console.log(` ... ${findings.length} in total, 60 shown`); +} + +const maxFindings = Number(values['max-findings']); +const maxErrors = Number(values['max-errors']); +let failed = false; +if (errors.length > maxErrors) { + console.log(`\n::error::${errors.length} files failed to read, budget is ${maxErrors}`); + failed = true; +} +if (findings.length > maxFindings) { + console.log(`\n::error::${findings.length} diagnostics, budget is ${maxFindings}`); + failed = true; +} + +if (failed) process.exit(1); +console.log('\nPASSED: every example file read, within budget.');