diff --git a/docs/issue-128-large-schematic-import.md b/docs/issue-128-large-schematic-import.md new file mode 100644 index 0000000..83653f9 --- /dev/null +++ b/docs/issue-128-large-schematic-import.md @@ -0,0 +1,37 @@ +# Issue #128: bounded large schematic import evidence + +## Root cause + +MinePilot Probe B's Sponge schematic is 512×64×512 with 997,632 occupied blocks. Its gzip file is 107,799 bytes and its NBT `BlockData` byte array is exactly 16,777,216 bytes. `prismarine-nbt@2.8.0` delegates to `protodef@1.19.0`, whose default array reader rejects counts above `0xffffff` (16,777,215). The default parser rejection is covered by the importer regression test. + +## Safety boundary + +The importer now bounds work before using Prismarine's supported `parseUncompressed(data, "big", { noArraySizeCheck: true })` option: + +- compressed schematic input: at most 16 MiB; +- uncompressed NBT: at most 32 MiB, enforced by `gunzipSync({ maxOutputLength })`; +- dimensions: positive and at most 16,777,216 total cells; +- root `BlockData`: at most 24 MiB and at least one encoded byte per cell; +- all other NBT lists and arrays: at most 1,000,000 elements total, no more than 100,000 tags, and at most 64 levels deep; +- large-array imports: at most 1,000,000 decoded non-air blocks. + +A no-allocation NBT preflight validates tag structure, declared collection lengths, dimensions, and the single permitted large array before the large-array parser option is enabled. The parser option is only used when the top-level `BlockData` exceeds the default `0xffffff` limit and the preflight passes. These limits intentionally bound the supported large-schematic envelope to the measured Probe B case and nearby inputs; larger volumes, larger BlockData, or over one million occupied blocks remain rejected. + +## Probe B result + +Exact MinePilot artifact: `packages/importer-schem/test/fixtures/probe-b-512.schem`. + +| Dimensions | Bounding volume | Occupied | BlockData bytes | Compressed bytes | Import | Peak RSS after import | Exact position check | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 512×64×512 | 16,777,216 | 997,632 | 16,777,216 | 107,799 | 540.9 ms | 902,381,568 B (860.6 MiB) | Pass; all occupied coordinates and block names match the deterministic source geometry | + +The separate exact-position verification took 17.0 ms and did not raise the process high-water mark beyond 902,381,568 B (860.6 MiB). This is a successful but memory-heavy import (`PASS_HEAVY`), not a comfortable-scale claim. + +Environment: macOS 27.0 arm64, Node.js v22.22.3, pnpm 8.15.0. Reproduce after building the workspace: + +```sh +pnpm build +node scripts/bench-probe-b-512.mjs +``` + +The benchmark script checks dimensions, occupied count, every occupied position, and material against the deterministic MinePilot Probe B generator formula without allocating a second million-position set. diff --git a/packages/importer-schem/src/index.ts b/packages/importer-schem/src/index.ts index 709fe57..f4ceb18 100644 --- a/packages/importer-schem/src/index.ts +++ b/packages/importer-schem/src/index.ts @@ -14,6 +14,204 @@ interface SchematicData { blockIds: number[]; } +// Probe B establishes a concrete supported large-schematic envelope. Keep +// the larger parser path bounded to that volume and a modest NBT payload. +const MAX_COMPRESSED_SCHEMATIC_BYTES = 16 * 1024 * 1024; +const MAX_UNCOMPRESSED_NBT_BYTES = 32 * 1024 * 1024; +const MAX_SCHEMATIC_VOLUME = 16_777_216; +const MAX_BLOCK_DATA_BYTES = 24 * 1024 * 1024; +const MAX_OTHER_COLLECTION_ITEMS = 1_000_000; +const MAX_TOTAL_COLLECTION_ITEMS = 1_000_000; +const MAX_NBT_TAGS = 100_000; +const MAX_NBT_DEPTH = 64; +const MAX_LARGE_IMPORT_OCCUPIED_BLOCKS = 1_000_000; + +interface SchematicPreflight { + width: number; + height: number; + length: number; + blockDataLength: number; +} + +/** + * Walk NBT tag headers and lengths without allocating tag arrays. This runs + * before opting out of ProtoDef's array guard, so only the bounded root + * BlockData byte array may exceed the ordinary collection limits. + */ +function preflightSchematic(uncompressed: Buffer): SchematicPreflight { + let offset = 0; + let tagCount = 0; + let collectionItems = 0; + let width: number | undefined; + let height: number | undefined; + let length: number | undefined; + let blockDataLength: number | undefined; + + function requireBytes(count: number, description: string): void { + if (!Number.isSafeInteger(count) || count < 0 || offset + count > uncompressed.length) { + throw new Error(`Invalid schematic NBT: truncated ${description}`); + } + } + + function readByte(description: string): number { + requireBytes(1, description); + return uncompressed.readUInt8(offset++); + } + + function readShort(description: string): number { + requireBytes(2, description); + const value = uncompressed.readInt16BE(offset); + offset += 2; + return value; + } + + function readInt(description: string): number { + requireBytes(4, description); + const value = uncompressed.readInt32BE(offset); + offset += 4; + return value; + } + + function readName(): string { + const byteLength = readUnsignedShort("tag name length"); + requireBytes(byteLength, "tag name"); + const name = uncompressed.toString("utf8", offset, offset + byteLength); + offset += byteLength; + return name; + } + + function readUnsignedShort(description: string): number { + requireBytes(2, description); + const value = uncompressed.readUInt16BE(offset); + offset += 2; + return value; + } + + function consumeItems(count: number, description: string): void { + if (count < 0 || count > MAX_OTHER_COLLECTION_ITEMS) { + throw new Error(`Schematic safety limit exceeded: ${description} exceeds ${MAX_OTHER_COLLECTION_ITEMS} items`); + } + collectionItems += count; + if (collectionItems > MAX_TOTAL_COLLECTION_ITEMS) { + throw new Error(`Schematic safety limit exceeded: NBT collections exceed ${MAX_TOTAL_COLLECTION_ITEMS} total items`); + } + } + + function scanPayload(type: number, name: string | undefined, isRootTag: boolean, depth: number): void { + if (depth > MAX_NBT_DEPTH) { + throw new Error(`Schematic safety limit exceeded: NBT nesting exceeds ${MAX_NBT_DEPTH}`); + } + + switch (type) { + case 1: // byte + requireBytes(1, "byte tag"); + offset += 1; + return; + case 2: { // short + const value = readShort("short tag"); + if (isRootTag && name === "Width") width = value; + if (isRootTag && name === "Height") height = value; + if (isRootTag && name === "Length") length = value; + return; + } + case 3: // int + case 5: // float + requireBytes(4, "32-bit tag"); + offset += 4; + return; + case 4: // long + case 6: // double + requireBytes(8, "64-bit tag"); + offset += 8; + return; + case 7: { // byte array + const count = readInt("byte-array length"); + if (count < 0) throw new Error("Invalid schematic NBT: negative byte-array length"); + if (isRootTag && name === "BlockData") { + if (blockDataLength !== undefined) throw new Error("Invalid schematic NBT: duplicate BlockData tag"); + if (count > MAX_BLOCK_DATA_BYTES) { + throw new Error(`Schematic safety limit exceeded: BlockData exceeds ${MAX_BLOCK_DATA_BYTES} bytes`); + } + blockDataLength = count; + } else { + consumeItems(count, "non-BlockData byte array"); + } + requireBytes(count, "byte-array payload"); + offset += count; + return; + } + case 8: { // string + const byteLength = readUnsignedShort("string length"); + requireBytes(byteLength, "string payload"); + offset += byteLength; + return; + } + case 9: { // list + const elementType = readByte("list element type"); + const count = readInt("list length"); + consumeItems(count, "NBT list"); + if (elementType < 0 || elementType > 12 || (elementType === 0 && count > 0)) { + throw new Error("Invalid schematic NBT: invalid list element type or length"); + } + for (let i = 0; i < count; i++) scanPayload(elementType, undefined, false, depth + 1); + return; + } + case 10: { // compound + for (;;) { + const childType = readByte("compound tag type"); + if (childType === 0) return; + if (childType < 1 || childType > 12) throw new Error("Invalid schematic NBT: unknown tag type"); + const childName = readName(); + tagCount += 1; + if (tagCount > MAX_NBT_TAGS) { + throw new Error(`Schematic safety limit exceeded: NBT contains more than ${MAX_NBT_TAGS} tags`); + } + scanPayload(childType, childName, false, depth + 1); + } + } + case 11: // int array + case 12: { // long array + const count = readInt("numeric-array length"); + consumeItems(count, "NBT numeric array"); + const elementBytes = type === 11 ? 4 : 8; + const byteLength = count * elementBytes; + requireBytes(byteLength, "numeric-array payload"); + offset += byteLength; + return; + } + default: + throw new Error(`Invalid schematic NBT: unsupported tag type ${type}`); + } + } + + if (readByte("root tag type") !== 10) throw new Error("Invalid schematic NBT: root tag must be a compound"); + readName(); + for (;;) { + const type = readByte("root tag type"); + if (type === 0) break; + if (type < 1 || type > 12) throw new Error("Invalid schematic NBT: unknown root tag type"); + const name = readName(); + tagCount += 1; + if (tagCount > MAX_NBT_TAGS) { + throw new Error(`Schematic safety limit exceeded: NBT contains more than ${MAX_NBT_TAGS} tags`); + } + scanPayload(type, name, true, 1); + } + + if (offset !== uncompressed.length) throw new Error("Invalid schematic NBT: unexpected trailing bytes"); + if (!Number.isInteger(width) || !Number.isInteger(height) || !Number.isInteger(length) || width! <= 0 || height! <= 0 || length! <= 0) { + throw new Error("Invalid schematic: Width, Height, and Length must be positive short values"); + } + const volume = width! * height! * length!; + if (volume > MAX_SCHEMATIC_VOLUME) { + throw new Error(`Schematic safety limit exceeded: volume ${volume} exceeds ${MAX_SCHEMATIC_VOLUME} cells`); + } + if (blockDataLength === undefined) throw new Error("Invalid schematic NBT: missing BlockData byte array"); + if (blockDataLength < volume) throw new Error(`Invalid schematic: BlockData has ${blockDataLength} bytes for ${volume} cells`); + + return { width: width!, height: height!, length: length!, blockDataLength }; +} + function parseBlockState(str: string): BlockState { const bracketIdx = str.indexOf("["); if (bracketIdx === -1) return { name: str }; @@ -28,8 +226,31 @@ function parseBlockState(str: string): BlockState { } function parseSchematic(buffer: Buffer): SchematicData { - const uncompressed = zlib.gunzipSync(buffer); - const parsed = nbt.parseUncompressed(uncompressed); + if (buffer.byteLength > MAX_COMPRESSED_SCHEMATIC_BYTES) { + throw new Error(`Schematic safety limit exceeded: compressed input exceeds ${MAX_COMPRESSED_SCHEMATIC_BYTES} bytes`); + } + + let uncompressed: Buffer; + try { + uncompressed = zlib.gunzipSync(buffer, { maxOutputLength: MAX_UNCOMPRESSED_NBT_BYTES }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") { + throw new Error(`Schematic safety limit exceeded: uncompressed NBT exceeds ${MAX_UNCOMPRESSED_NBT_BYTES} bytes`); + } + throw error; + } + const preflight = preflightSchematic(uncompressed); + const oversizedBlockData = preflight.blockDataLength > 0xffffff; + let parsed: ReturnType; + if (oversizedBlockData) { + try { + parsed = nbt.parseUncompressed(uncompressed, "big", { noArraySizeCheck: true }); + } catch (error) { + throw new Error(`Unable to parse bounded large schematic NBT: ${(error as Error).message}`); + } + } else { + parsed = nbt.parseUncompressed(uncompressed); + } const root = (parsed as any).value; const version = root.Version?.value ?? 2; @@ -49,6 +270,7 @@ function parseSchematic(buffer: Buffer): SchematicData { const blockDataBytes: number[] = root.BlockData?.value ?? []; const blockIds: number[] = []; + let occupiedBlockCount = 0; let i = 0; while (i < blockDataBytes.length) { let value = 0; @@ -60,6 +282,18 @@ function parseSchematic(buffer: Buffer): SchematicData { shift += 7; } while ((byte & 0x80) !== 0 && i < blockDataBytes.length); blockIds.push(value); + const blockState = palette.get(value); + if (oversizedBlockData && blockState && blockState.name !== "minecraft:air") { + occupiedBlockCount += 1; + if (occupiedBlockCount > MAX_LARGE_IMPORT_OCCUPIED_BLOCKS) { + throw new Error(`Schematic safety limit exceeded: large imports may contain at most ${MAX_LARGE_IMPORT_OCCUPIED_BLOCKS} occupied blocks`); + } + } + } + + const expectedCells = width * height * length; + if (blockIds.length !== expectedCells) { + throw new Error(`Invalid schematic: BlockData decoded to ${blockIds.length} cells; expected ${expectedCells}`); } return { version, dataVersion, width, height, length, offset, paletteMax, palette, blockIds }; diff --git a/packages/importer-schem/test/fixtures/probe-b-512.schem b/packages/importer-schem/test/fixtures/probe-b-512.schem new file mode 100644 index 0000000..79fd685 Binary files /dev/null and b/packages/importer-schem/test/fixtures/probe-b-512.schem differ diff --git a/packages/importer-schem/test/importer.test.ts b/packages/importer-schem/test/importer.test.ts index e546ac2..63667db 100644 --- a/packages/importer-schem/test/importer.test.ts +++ b/packages/importer-schem/test/importer.test.ts @@ -1,3 +1,6 @@ +import { readFileSync } from "node:fs"; +import zlib from "node:zlib"; +import nbt from "prismarine-nbt"; import { describe, it, expect } from "vitest"; import { exportToSchematic } from "../../exporter-schem/src/index.js"; import { importFromSchematic, importerVersion } from "../src/index.js"; @@ -39,6 +42,61 @@ const samplePlan = { ], }; +const probeB512Fixture = readFileSync(new URL("./fixtures/probe-b-512.schem", import.meta.url)); +const probeBColumns = 512 * 512; +const probeBTargetOccupied = 1_000_000; +const probeBExtraColumns = probeBTargetOccupied % probeBColumns; + +function probeBHash2(x: number, z: number): number { + let n = (Math.imul(x >> 3, 0x45d9f3b) ^ Math.imul(z >> 3, 0x119de1f3) ^ 0x5f3759df) >>> 0; + n = Math.imul(n ^ (n >>> 16), 0x45d9f3b) >>> 0; + n = Math.imul(n ^ (n >>> 16), 0x45d9f3b) >>> 0; + return (n ^ (n >>> 16)) >>> 0; +} + +function probeBExpectedBlock(x: number, y: number, z: number): string | undefined { + const hash = probeBHash2(x, z); + const landHeight = 5 + ((hash >>> 8) % 28); + const layerCount = 3 + (hash % probeBColumns < probeBExtraColumns ? 1 : 0); + const layer = y - (landHeight - layerCount); + if (layer < 0 || layer >= layerCount) return undefined; + if (layer === layerCount - 1) return landHeight > 23 ? "minecraft:stone" : "minecraft:grass_block"; + if (layer === layerCount - 2) return "minecraft:dirt"; + return "minecraft:stone"; +} + +function expectedProbeBOccupiedCount(): number { + let count = 0; + for (let z = 0; z < 512; z++) { + for (let x = 0; x < 512; x++) { + const hash = probeBHash2(x, z); + count += 3 + (hash % probeBColumns < probeBExtraColumns ? 1 : 0); + } + } + return count; +} + +function makeSchematic(size: [number, number, number], blockData: number[] = [0], withEmptyBlockEntitiesList = false): Buffer { + const [width, height, length] = size; + const raw = nbt.writeUncompressed({ + type: "compound", + name: "Schematic", + value: { + Version: { type: "int", value: 2 }, + DataVersion: { type: "int", value: 3463 }, + Width: { type: "short", value: width }, + Height: { type: "short", value: height }, + Length: { type: "short", value: length }, + Offset: { type: "intArray", value: [0, 0, 0] }, + PaletteMax: { type: "int", value: 1 }, + Palette: { type: "compound", value: { "minecraft:air": { type: "int", value: 0 } } }, + BlockData: { type: "byteArray", value: blockData }, + ...(withEmptyBlockEntitiesList ? { BlockEntities: { type: "list", value: { type: "compound", value: [] } } } : {}), + }, + } as any); + return zlib.gzipSync(raw); +} + describe("Schematic Importer metadata", () => { it("reports the current package version", () => { expect(importerVersion).toBe("0.2.5"); @@ -79,4 +137,82 @@ describe("schematic importer", () => { expect(imported.blocks.length).toBe(0); expect(imported.size).toEqual([4, 4, 4]); }); + + it("reproduces the prismarine-nbt default 0xffffff array guard on the Probe B fixture", () => { + const uncompressed = zlib.gunzipSync(probeB512Fixture); + expect(() => nbt.parseUncompressed(uncompressed)).toThrow(/array size is abnormally large.*16777216/); + }); + + it("imports and exactly matches the Probe B 512×64×512 occupied coordinates", () => { + const imported = importFromSchematic(probeB512Fixture, { name: "Probe B 512" }); + expect(imported.size).toEqual([512, 64, 512]); + expect(imported.blocks).toHaveLength(997_632); + expect(imported.blocks).toHaveLength(expectedProbeBOccupiedCount()); + + // The importer emits each voxel position at most once from the full grid. + // Matching every output position/material plus the expected count proves + // the exact expected occupied set without keeping a second million-key Set. + for (const block of imported.blocks) { + const expectedBlock = probeBExpectedBlock(...block.pos); + if (expectedBlock !== block.block.name) { + throw new Error(`unexpected block ${block.block.name} at ${block.pos}; expected ${expectedBlock ?? "air"}`); + } + } + }, 120_000); + + it("rejects a schematic volume above the measured safety bound", () => { + const tooLarge = makeSchematic([512, 64, 513]); + expect(() => importFromSchematic(tooLarge)).toThrow(/volume .* exceeds 16777216 cells/); + }); + + it("rejects compressed schematics above the input safety bound before gunzip", () => { + const tooLarge = Buffer.alloc(16 * 1024 * 1024 + 1); + expect(() => importFromSchematic(tooLarge)).toThrow(/compressed input exceeds 16777216 bytes/); + }); + + it("rejects uncompressed NBT above the gunzip safety bound", () => { + const tooLarge = zlib.gzipSync(Buffer.alloc(32 * 1024 * 1024 + 1)); + expect(() => importFromSchematic(tooLarge)).toThrow(/uncompressed NBT exceeds 33554432 bytes/); + }); + + it("rejects oversized BlockData even when the compressed payload is small", () => { + const uncompressed = zlib.gunzipSync(probeB512Fixture); + const blockDataName = Buffer.from("BlockData"); + const nameIndex = uncompressed.indexOf(blockDataName); + expect(nameIndex).toBeGreaterThan(0); + uncompressed.writeInt32BE(24 * 1024 * 1024 + 1, nameIndex + blockDataName.length); + const malformed = zlib.gzipSync(uncompressed); + expect(() => importFromSchematic(malformed)).toThrow(/BlockData exceeds 25165824 bytes/); + }); + + it("does not lift the normal list bound when allowing large BlockData", () => { + const uncompressed = zlib.gunzipSync(makeSchematic([1, 1, 1], [0], true)); + const listName = Buffer.from("BlockEntities"); + const nameIndex = uncompressed.indexOf(listName); + expect(nameIndex).toBeGreaterThan(0); + // NBT list layout is element type byte followed by signed 32-bit length. + uncompressed.writeInt32BE(1_000_001, nameIndex + listName.length + 1); + const malformed = zlib.gzipSync(uncompressed); + expect(() => importFromSchematic(malformed)).toThrow(/NBT list exceeds 1000000 items/); + }); + + it("rejects large schematics whose decoded occupied count exceeds the safety bound", () => { + const uncompressed = zlib.gunzipSync(probeB512Fixture); + const blockDataName = Buffer.from("BlockData"); + const blockDataNameIndex = uncompressed.indexOf(blockDataName); + const blockDataLength = uncompressed.readInt32BE(blockDataNameIndex + blockDataName.length); + const payloadOffset = blockDataNameIndex + blockDataName.length + 4; + let changed = 0; + const additionalBlocks = 1_000_001 - 997_632; + for (let i = payloadOffset; i < payloadOffset + blockDataLength && changed < additionalBlocks; i++) { + if (uncompressed[i] === 0) { + uncompressed[i] = 1; + changed += 1; + } + } + expect(changed).toBe(additionalBlocks); + const overOccupiedLimit = zlib.gzipSync(uncompressed); + expect(() => importFromSchematic(overOccupiedLimit)).toThrow(/at most 1000000 occupied blocks/); + }, 120_000); + }); diff --git a/scripts/bench-probe-b-512.mjs b/scripts/bench-probe-b-512.mjs new file mode 100644 index 0000000..b54a772 --- /dev/null +++ b/scripts/bench-probe-b-512.mjs @@ -0,0 +1,54 @@ +import { readFile } from "node:fs/promises"; +import { performance } from "node:perf_hooks"; +import { importFromSchematic } from "../packages/importer-schem/dist/index.js"; + +const fixturePath = new URL("../packages/importer-schem/test/fixtures/probe-b-512.schem", import.meta.url); +const fixture = await readFile(fixturePath); + +function hash2(x, z) { + let n = (Math.imul(x >> 3, 0x45d9f3b) ^ Math.imul(z >> 3, 0x119de1f3) ^ 0x5f3759df) >>> 0; + n = Math.imul(n ^ (n >>> 16), 0x45d9f3b) >>> 0; + n = Math.imul(n ^ (n >>> 16), 0x45d9f3b) >>> 0; + return (n ^ (n >>> 16)) >>> 0; +} + +function expectedBlock(x, y, z) { + const hash = hash2(x, z); + const landHeight = 5 + ((hash >>> 8) % 28); + const columns = 512 * 512; + const extraColumns = 1_000_000 % columns; + const layers = 3 + (hash % columns < extraColumns ? 1 : 0); + const layer = y - (landHeight - layers); + if (layer < 0 || layer >= layers) return undefined; + if (layer === layers - 1) return landHeight > 23 ? "minecraft:stone" : "minecraft:grass_block"; + if (layer === layers - 2) return "minecraft:dirt"; + return "minecraft:stone"; +} + +const importStart = performance.now(); +const plan = importFromSchematic(fixture, { name: "Probe B deterministic wide landscape 512" }); +const importMs = performance.now() - importStart; +const peakRssBytesAfterImport = process.resourceUsage().maxRSS * 1024; + +if (JSON.stringify(plan.size) !== JSON.stringify([512, 64, 512])) throw new Error(`unexpected dimensions: ${plan.size}`); +if (plan.blocks.length !== 997_632) throw new Error(`unexpected occupied count: ${plan.blocks.length}`); + +const validationStart = performance.now(); +for (const block of plan.blocks) { + const expected = expectedBlock(...block.pos); + if (expected !== block.block.name) { + throw new Error(`unexpected block ${block.block.name} at ${block.pos}; expected ${expected ?? "air"}`); + } +} +const exactPositionValidationMs = performance.now() - validationStart; + +console.log(JSON.stringify({ + fixtureBytes: fixture.byteLength, + dimensions: plan.size, + occupiedBlocks: plan.blocks.length, + exactPositionAndMaterialMatch: true, + importMs, + exactPositionValidationMs, + peakRssBytesAfterImport, + peakRssBytesAfterValidation: process.resourceUsage().maxRSS * 1024, +}));