diff --git a/electron.vite.config.ts b/electron.vite.config.ts index a718730..0b11905 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -12,7 +12,15 @@ export default defineConfig({ }, build: { rollupOptions: { - input: { index: resolve(__dirname, 'src/main/index.ts') } + // The tile worker is a SECOND entry, not an import: it runs on its own + // thread and `new Worker()` needs a real file beside the main bundle + // (#160). Importing it instead would fold its module graph back into + // the main bundle, which is the opposite of the point. + input: { + index: resolve(__dirname, 'src/main/index.ts'), + tileWorker: resolve(__dirname, 'src/main/core/tileWorker.ts') + }, + output: { entryFileNames: '[name].js' } } } }, diff --git a/src/main/core/regionParse.ts b/src/main/core/regionParse.ts new file mode 100644 index 0000000..917bbc1 --- /dev/null +++ b/src/main/core/regionParse.ts @@ -0,0 +1,331 @@ +/** + * Turning a region file into tiles, with nothing but the file (#160). + * + * Split out of `worldTiles.ts` so a WORKER THREAD can do it. That module + * reaches for the server registry, the app paths and the logger, all of which + * pull in Electron; this one knows about a buffer, a dimension name and the + * colour table, so it runs anywhere. + * + * THE COLOUR TABLE IS THE TRAP. `blockColour` reads module-level state that + * `core/clientAssets.ts` fills at runtime from the operator's client jar, and a + * worker starts with it EMPTY. A worker that is not given the table renders the + * fallback palette, and `writeCachedRegion` then persists those colours to disk + * where they outlive the process — a wrong map that survives restarts, with a + * cache version that still matches. `tileWorker.ts` is handed the table before + * it parses anything, and the pool re-sends it whenever it changes. + */ +import { gunzipSync, inflateSync } from 'node:zlib' +import * as nbt from 'prismarine-nbt' +import { + bitsPerIndex, + blockColour, + indexAt, + packingFor, + parseLocationTable, + prepareIndices, + scanRuleFor, + seeThrough, + structureKind, + CHUNK_AXIS, + INVISIBLE +} from '@shared/regionFormat' +import type { StructureMark } from '@shared/regionFormat' + +/** A chunk's surface: 256 columns, row-major (x fastest). */ +export interface ChunkTile { + /** Packed 0xRRGGBB per column. */ + colour: number[] + /** World Y of the drawn block, for cross-chunk shading. */ + height: number[] + /** + * Structures starting in this chunk (#131). + * + * Read from the same NBT the surface came from, so it costs one more object + * lookup rather than a second pass over the world. Absent on most chunks. + */ + marks?: StructureMark[] +} + +/** Longs out of prismarine-nbt, which gives signed 64-bit values as [hi, lo]. */ +function toLongs(raw: unknown): bigint[] { + if (!Array.isArray(raw)) return [] + return raw.map((v) => { + if (typeof v === 'bigint') return v + if (Array.isArray(v) && v.length === 2) { + // [high, low], both signed 32-bit. The high word carries the sign. + return BigInt.asIntN(64, (BigInt(v[0]) << 32n) | (BigInt(v[1] >>> 0) & 0xffffffffn)) + } + if (typeof v === 'number') return BigInt(Math.trunc(v)) + return 0n + }) +} + +/* eslint-disable @typescript-eslint/no-explicit-any */ +function tag(v: any): any { + return v && typeof v === 'object' && 'value' in v ? v.value : v +} + +/** + * Unwrap an NBT *list*, which prismarine-nbt wraps twice. + * + * A list arrives as `{type:'list', value:{type:'compound', value:[...]}}` — the + * outer wrapper says "list", the inner one says what the elements are. One + * `tag()` leaves you holding the inner descriptor object, not the array. + * + * This is not a nicety. Reading `sections` happened to work because the code + * unwrapped it twice by accident, while `palette` was unwrapped once — so + * `Array.isArray(palette)` was false for every section of every chunk, every + * section was skipped, and the world renderer produced nothing at all. The + * smoke tested the bit decoding and never a real chunk, so nothing caught it. + */ +function listOf(v: any): any[] { + const once = tag(v) + const twice = tag(once) + return Array.isArray(twice) ? twice : Array.isArray(once) ? once : [] +} + +/** + * The topmost visible block of every column in one chunk. + * + * Sections are walked from the highest down, and within a section from y=15 + * down, stopping at the first block that is not air. A column that is nothing + * but air the whole way — under an unlit sky, or a chunk that is only partly + * generated — is left transparent rather than drawn as the void. + */ +export function tileFromChunk(chunk: any, dim = 'overworld'): ChunkTile | null { + const v = tag(chunk) + if (!v) return null + const dataVersion = tag(v.DataVersion) + const packing = packingFor(typeof dataVersion === 'number' ? dataVersion : undefined) + // 1.18+ uses `sections`; 1.13-1.17 used `Level.Sections`. + const sections = listOf(v.sections).length ? listOf(v.sections) : listOf(tag(v.Level)?.Sections) + if (!sections.length) return null + + const withY = sections + .map((s: any) => ({ s: tag(s), y: Number(tag(tag(s)?.Y)) })) + .filter((x) => Number.isFinite(x.y)) + .sort((a, b) => b.y - a.y) + + const colour = new Array(CHUNK_AXIS * CHUNK_AXIS).fill(-1) + const height = new Array(CHUNK_AXIS * CHUNK_AXIS).fill(0) + let remaining = colour.length + + // The nether has a bedrock roof: a top-down scan finds it in every column and + // paints the whole dimension one flat grey. `sawAir` per column is how a map + // gets under it — solid blocks are skipped until an air gap has been seen. + const rule = scanRuleFor(dim) + const sawAir = rule.underRoof ? new Array(colour.length).fill(false) : null + // A column that is solid from the ceiling all the way down — a netherrack + // pillar joining floor to roof — never shows an air gap, so the under-roof + // rule would skip every block in it and leave a hole. The highest solid block + // seen while skipping is kept as the answer for exactly that case. + const fallbackColour = sawAir ? new Array(colour.length).fill(-1) : null + const fallbackHeight = sawAir ? new Array(colour.length).fill(0) : null + + for (const { s, y: sectionY } of withY) { + if (remaining === 0) break + // Above the ceiling there is nothing worth looking at, and on the nether + // that is most of the sections. + if (rule.ceiling !== null && sectionY * CHUNK_AXIS > rule.ceiling) continue + const states = tag(s.block_states) ?? tag(s.BlockStates) + // A list, so unwrapped twice. See `listOf`. + const paletteRaw = listOf(states?.palette).length ? listOf(states?.palette) : listOf(s.Palette) + if (!paletteRaw.length) continue + const names: string[] = paletteRaw.map((p: any) => String(tag(tag(p)?.Name) ?? '')) + + /** + * The block rules, resolved once per PALETTE ENTRY. + * + * This is the whole cost of a map (#157). A section is 4096 positions and + * its palette is at most a few dozen entries, and every one of these used + * to run per position: a regex to strip the namespace, a Set lookup for + * air, `seeThrough` (a Set miss then ten `endsWith` calls), and + * `blockColour` with a second regex inside it. A fully generated chunk is + * about 53 thousand of those to render 256 columns — 13 ms a chunk, 14 + * seconds a region, and the reason a viewport could take over a minute. + * + * Resolved per entry it is a few dozen, and the inner loop is array + * indexing. Same answers: the arrays are built by the same functions in the + * same order. + */ + const invisible = names.map((n) => { + // Air, and the plants a map looks through — see `seeThrough`. Without it + // the surface is whatever is standing ON the ground rather than the + // ground, which is how a bamboo jungle rendered as a maroon smear. + const short = n.replace(/^minecraft:/, '') + return !short || INVISIBLE.has(short) || seeThrough(short) + }) + // A section a map sees nothing in — and above the surface that is most of + // them, fifteen or so single-entry air palettes per chunk. Skipped HERE, + // before the colours are looked up, before the indices are unpacked and + // before anything is allocated, because the point is not to make those + // sections cheaper but to stop touching them at all. + if (invisible.every(Boolean)) { + // Except under a roof, where an air section is not nothing: it is the gap + // the scan is looking for. Recorded for every column in one pass instead + // of by walking 4096 positions to reach the same conclusion. Every + // section still standing here has its bottom layer at or below the + // ceiling, so a full air layer covers all 256 columns. + if (sawAir) sawAir.fill(true) + continue + } + + // Only now, for the sections that can actually contribute a colour. + const packedColour = names.map((n) => { + const c = blockColour(n.replace(/^minecraft:/, '')) + return (c.r << 16) | (c.g << 8) | c.b + }) + + // A section whose palette is one entry has no data array at all — it is + // 4096 of that block, which is how a solid stone or all-air section is + // stored. Reading `data` there would skip the section entirely. + // `data` is a longArray, which is wrapped once — unlike the palette beside + // it, which is a list and wrapped twice. + const longs = toLongs(tag(states?.data) ?? tag(s.BlockStates)) + const bits = bitsPerIndex(names.length) + // `null` rather than 4096 zeroes: a uniform section reads palette entry 0 + // at every position, so the array only existed to say so. + // + // And prepared rather than unpacked: the loop below walks down from the top + // layer and stops the moment every column has an answer, which on real + // terrain is after two or three of the sixteen. Unpacking all 4096 did the + // rest for nothing. + const indices = + names.length === 1 || !longs.length ? null : prepareIndices(longs, bits, packing) + + for (let y = CHUNK_AXIS - 1; y >= 0 && remaining > 0; y--) { + // Neither of these depends on the column, and both used to be recomputed + // 256 times a layer. + const worldY = sectionY * CHUNK_AXIS + y + if (rule.ceiling !== null && worldY > rule.ceiling) continue + const base = y * 256 + // `x + z * CHUNK_AXIS` IS the column index, so the two loops the original + // had over x and z collapse into one over the column in the same order. + for (let col = 0; col < 256; col++) { + if (colour[col] >= 0) continue + const pi = indices ? indexAt(indices, base + col) : 0 + // An index past the end of its palette: the width comes from + // `bitsPerIndex`, which rounds up, so a three-entry palette is read + // four bits wide and corrupt data can address entry 15. The old code + // resolved that to an empty name and treated it as invisible. + const inv = pi >= names.length || invisible[pi] + if (sawAir) { + // Under a roof: remember the gap, and skip everything solid until + // one has been seen. Without this the first hit is the roof itself. + if (inv) sawAir[col] = true + if (!sawAir[col]) { + if (!inv && fallbackColour && fallbackColour[col] < 0) { + fallbackColour[col] = packedColour[pi] + if (fallbackHeight) fallbackHeight[col] = worldY + } + continue + } + } + if (inv) continue + colour[col] = packedColour[pi] + height[col] = worldY + remaining-- + } + } + } + // Columns the under-roof rule skipped entirely fall back to the highest solid + // block, so a floor-to-ceiling pillar is drawn rather than punched out. + if (fallbackColour && fallbackHeight) { + for (let i = 0; i < colour.length; i++) { + if (colour[i] < 0 && fallbackColour[i] >= 0) { + colour[i] = fallbackColour[i] + height[i] = fallbackHeight[i] + remaining-- + } + } + } + // Nothing at all: an ungenerated or empty chunk, which is not a tile. + if (remaining === colour.length) return null + const marks = structuresOf(v) + return { colour, height, ...(marks.length ? { marks } : {}) } +} + +/** + * Structures whose start is in this chunk. + * + * `structures.starts` is keyed by structure id and each entry carries the chunk + * it starts in — `ChunkX`/`ChunkZ` in chunk units, which is why they are + * multiplied here rather than used raw. A chunk that merely CONTAINS part of a + * structure lists it in `References`, not `starts`, so this yields one mark per + * structure rather than one per chunk it sprawls across. + */ +function structuresOf(v: any): StructureMark[] { + const starts = tag(tag(v.structures)?.starts) ?? tag(tag(tag(v.Level)?.Structures)?.Starts) + if (!starts || typeof starts !== 'object') return [] + const out: StructureMark[] = [] + for (const [id, raw] of Object.entries(starts)) { + const s = tag(raw) + if (!s || typeof s !== 'object') continue + const cx = Number(tag((s as any).ChunkX)) + const cz = Number(tag((s as any).ChunkZ)) + if (!Number.isFinite(cx) || !Number.isFinite(cz)) continue + out.push({ + kind: structureKind(id), + id: String(id).replace(/^minecraft:/, ''), + x: cx * CHUNK_AXIS + CHUNK_AXIS / 2, + z: cz * CHUNK_AXIS + CHUNK_AXIS / 2 + }) + } + return out +} + +export function decompress(buf: Buffer, kind: number): Buffer | null { + try { + if (kind === 1) return gunzipSync(buf) + if (kind === 2) return inflateSync(buf) + if (kind === 3) return buf + } catch { + /* a truncated or corrupt chunk is skipped, never fatal */ + } + return null +} + +/** A region header is two 4 KiB tables: locations, then timestamps. */ +export const SECTOR_HEADER = 8192 + +/** + * One chunk out of a region file, into `tiles`. + * + * Pulled out of the parse loop so the same work can be done in one pass or in + * slices with the event loop running in between. + */ +export function parseSlot( + file: Buffer, + table: ReturnType, + slot: number, + tiles: Map, + dim: string +): void { + const loc = table[slot] + if (!loc || !loc.offset || loc.offset + 5 > file.length) return + const length = file.readUInt32BE(loc.offset) + const kind = file[loc.offset + 4] + const end = loc.offset + 5 + Math.max(0, length - 1) + if (length <= 0 || end > file.length) return + const raw = decompress(file.subarray(loc.offset + 5, end), kind) + if (!raw) return + try { + tiles.set(slot, tileFromChunk(nbt.parseUncompressed(raw), dim)) + } catch { + /* one unreadable chunk must not lose the region */ + } +} + +/** + * Every chunk of one region file, in a single pass. + * + * For a caller that is not on the thread the interface lives on — which is the + * whole point of the worker. On the main thread use the sliced form. + */ +export function parseRegionBuffer(file: Buffer, dim: string): Map { + const tiles = new Map() + if (file.length < SECTOR_HEADER) return tiles + const table = parseLocationTable(file.subarray(0, 4096)) + for (let slot = 0; slot < table.length; slot++) parseSlot(file, table, slot, tiles, dim) + return tiles +} diff --git a/src/main/core/tilePool.ts b/src/main/core/tilePool.ts new file mode 100644 index 0000000..7725341 --- /dev/null +++ b/src/main/core/tilePool.ts @@ -0,0 +1,167 @@ +/** + * A small pool of threads that parse region files (#160). + * + * A region costs about 1.4 s after #157, and it was being spent on the thread + * that answers every IPC call, serves the web panel and reads the console. + * Slicing made that interruptible, not free: four cold regions still measured + * six seconds, of which three quarters of a second was the politeness gap + * between them — a gap that exists only because the parse was in the way. + * + * Off-thread there is nothing to be polite to, and the parses run at once. + * + * Small on purpose. This is a server manager, and the machine's cores belong to + * the Minecraft server it is running, not to drawing its map. + */ +import { Worker } from 'node:worker_threads' +import { cpus } from 'node:os' +import { join } from 'node:path' +import { app } from 'electron' +import { textureColourEpoch, textureColours_ } from '@shared/regionFormat' +import { log } from '../logger' +import type { TileJob, TileJobResult } from './tileWorker' + +/** Half the cores, at least one, never more than four. */ +export function poolSize(): number { + const n = Math.floor((cpus()?.length ?? 2) / 2) + return Math.max(1, Math.min(4, n)) +} + +interface Slot { + worker: Worker + /** Which colour table this thread has been given. -1 = none yet. */ + epoch: number + busy: boolean +} + +interface Waiting { + path: string + dim: string + resolve: (r: TileJobResult) => void +} + +let slots: Slot[] = [] +const queue: Waiting[] = [] +const inFlight = new Map void>() +let nextId = 1 +let broken = false + +/** + * Where the worker bundle is. + * + * electron-vite emits it beside the main bundle, so it is a sibling of + * `__dirname` in both dev and a packaged build. Resolved rather than imported: + * a static import would pull the worker's module graph into the main bundle, + * which is the opposite of the point. + */ +function workerPath(): string { + return join(__dirname, 'tileWorker.js') +} + +function spawn(): Slot | null { + try { + const worker = new Worker(workerPath()) + const slot: Slot = { worker, epoch: -1, busy: false } + worker.on('message', (r: TileJobResult) => { + slot.busy = false + const done = inFlight.get(r.id) + inFlight.delete(r.id) + done?.(r) + pump() + }) + worker.on('error', (err: Error) => { + log.warn('Tile worker failed: ' + String(err?.message ?? err)) + slot.busy = false + // A thread that died takes its job with it. Everything waiting on it is + // answered as unreadable rather than left hanging — a map that draws + // nothing is recoverable, a promise that never settles is not. + for (const [id, done] of inFlight) { + inFlight.delete(id) + done({ id, buf: null, mtimeMs: 0, chunks: 0, error: 'worker-died' }) + } + slots = slots.filter((s) => s !== slot) + void worker.terminate() + if (!slots.length) broken = true + pump() + }) + worker.unref() + return slot + } catch (err) { + log.warn('Tile worker could not start: ' + String((err as Error)?.message ?? err)) + return null + } +} + +/** + * Switched off, so the on-thread path can still be reached. + * + * An escape hatch for a machine where spawning threads is a problem, and the + * only way the smoke can exercise the sliced parse now that the worker handles + * it instead — a gate that cannot reach the code it asserts about is a gate + * that has stopped asserting anything. + */ +let enabled = !process.env['MSMS_NO_TILE_WORKERS'] + +export function _setTileWorkersEnabled(on: boolean): void { + enabled = on +} + +/** Whether the pool can be used at all. False falls the caller back on-thread. */ +export function poolReady(): boolean { + if (!enabled || broken) return false + if (!slots.length) { + for (let i = 0; i < poolSize(); i++) { + const s = spawn() + if (s) slots.push(s) + } + if (!slots.length) { + broken = true + return false + } + log.info(`Tile workers: ${slots.length} thread(s) parsing regions off the main thread`) + } + return true +} + +function pump(): void { + for (const slot of slots) { + if (slot.busy || !queue.length) continue + const job = queue.shift() + if (!job) return + const id = nextId++ + inFlight.set(id, job.resolve) + slot.busy = true + const epoch = textureColourEpoch() + const msg: TileJob = { id, path: job.path, dim: job.dim, colourEpoch: epoch } + // Sent only when this thread's copy is out of date — the table is about a + // thousand entries and re-cloning it for every region would cost more than + // some of the parses do. + if (slot.epoch !== epoch) { + msg.colours = textureColours_() + slot.epoch = epoch + } + slot.worker.postMessage(msg) + } +} + +/** Parse one region on a worker. Rejects nothing; failure comes back as `error`. */ +export function parseOnWorker(path: string, dim: string): Promise { + return new Promise((resolve) => { + queue.push({ path, dim, resolve }) + pump() + }) +} + +/** How many jobs are waiting or running. The warmer reads this to stay behind. */ +export function poolBacklog(): number { + return queue.length + inFlight.size +} + +export async function stopTileWorkers(): Promise { + const all = slots + slots = [] + queue.length = 0 + await Promise.all(all.map((s) => s.worker.terminate())) +} + +// A packaged app that quits with threads alive can leave the process resident. +app?.on?.('before-quit', () => void stopTileWorkers()) diff --git a/src/main/core/tileWarm.ts b/src/main/core/tileWarm.ts new file mode 100644 index 0000000..1f964da --- /dev/null +++ b/src/main/core/tileWarm.ts @@ -0,0 +1,188 @@ +/** + * Reading a world into the tile cache before anyone looks at it (#161). + * + * Nothing parsed a region until somebody looked at it, so the map was cold the + * first time it was opened on a world MSMS had not seen — and the two paths are + * far apart: about 1.4 s to parse a region against about 13 ms to read one back + * from the cache. A viewport is four to nine regions, so first-open was seconds + * and every open after it was a tenth of one. + * + * This closes that gap by doing the parsing when nobody is waiting. + * + * Three things it must not do, all of which it would do naively: + * + * - fight the map. A visitor looking at the map has to win, so the warmer + * stands aside whenever the pool has interactive work queued. + * - stall the start. A big explored world is thousands of regions; this is + * bounded per pass and resumable, never a loop that has to finish. + * - ignore the operator. `cache: false` means the operator asked for nothing + * to be written, so there is nothing to warm. + */ +import { existsSync, readdirSync, statSync } from 'node:fs' +import { join } from 'node:path' +import { normalizeMapPerf } from '@shared/tileCache' +import { getServer } from './serverRegistry' +import { log } from '../logger' +import { poolBacklog, poolReady } from './tilePool' +import { + cachedRegionIsCurrent, + regionDirsForServer, + tileCacheBytes, + warmOneRegion +} from './worldTiles' + +/** + * Regions read per pass, per server. + * + * The point is that this finishes eventually while never being the reason + * something else is slow. A pass of 24 at roughly a second each is under half a + * minute of background work, then it stands down until the next tick. + */ +const PER_PASS = 24 + +/** How often a pass starts. Long, because this is housekeeping. */ +const TICK_MS = 60_000 + +/** Servers already fully warmed, so a finished world costs one directory scan. */ +const done = new Set() +/** Servers whose warming stopped because the cache is full, so it is said once. */ +const full = new Set() +let timer: NodeJS.Timeout | null = null +let running = false + +export function _resetTileWarm(): void { + done.clear() + full.clear() + running = false +} + +/** + * Region files worth reading, nearest to the origin first. + * + * Spawn is where players are and where a map is first pointed, so warming + * outward from it means the useful part is ready long before the far corners a + * world accumulates from one player who once walked a long way. + */ +export function warmOrder(dir: string): string[] { + let names: string[] + try { + names = readdirSync(dir) + } catch { + return [] + } + const out: { name: string; d: number }[] = [] + for (const name of names) { + const m = /^r\.(-?\d+)\.(-?\d+)\.mca$/.exec(name) + if (!m) continue + const rx = Number(m[1]) + const rz = Number(m[2]) + if (!Number.isFinite(rx) || !Number.isFinite(rz)) continue + out.push({ name, d: Math.max(Math.abs(rx), Math.abs(rz)) }) + } + // Then by name, so two runs over the same world agree on the order and a + // resumed pass picks up where the last one stopped rather than at random. + out.sort((a, b) => a.d - b.d || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + return out.map((o) => o.name) +} + +/** One pass over one server. Returns how many regions it actually parsed. */ +export async function warmServer(serverId: string, budget = PER_PASS): Promise { + const s = getServer(serverId) + if (!s) return 0 + const perf = normalizeMapPerf(s.map) + // Nothing to warm INTO. Parsing without a cache to write to would be pure + // cost: the work would be thrown away the moment it left memory. + if (!perf.cache) return 0 + if (!poolReady()) return 0 + + // A full cache means STOP, not "make room". The warmer writes nearest to + // spawn first and the sweep evicts oldest first, so carrying on would spend + // the pass deleting the area around spawn — the part a map is pointed at — + // and keeping whichever far corner was written last. An interactive parse + // may still evict; that one is answering somebody. + // + // A limit of zero is not "no limit", it is no room — the sweep would delete + // whatever was written the moment it landed. + const limit = perf.cacheLimitMB * 1024 * 1024 + if (tileCacheBytes() >= limit) { + if (!full.has(serverId)) { + full.add(serverId) + log.info( + `Tile cache: full at ${perf.cacheLimitMB} MB, so "${s.name}" will not be warmed further. ` + + 'Raise the limit in the map performance settings to cache more of this world.' + ) + } + return 0 + } + full.delete(serverId) + + let parsed = 0 + let looked = 0 + for (const { dim, dir } of regionDirsForServer(serverId)) { + if (!existsSync(dir)) continue + for (const name of warmOrder(dir)) { + if (parsed >= budget) return parsed + // The map is being used. Whatever a visitor is waiting for matters more + // than this does, so stand aside and pick up on the next tick. + if (poolBacklog() > 0) return parsed + const path = join(dir, name) + looked++ + let mtimeMs = 0 + try { + mtimeMs = statSync(path).mtimeMs + } catch { + continue + } + // Already cached and still current — the common case on the second run, + // and it costs one stat and one small read rather than a parse. + if (cachedRegionIsCurrent(serverId, path, mtimeMs)) continue + await warmOneRegion(serverId, path, dim, perf) + parsed++ + } + } + // A whole sweep that parsed nothing means this world is cached. Remembered so + // a finished world is one directory listing a minute, not a stat per region. + if (parsed === 0 && looked > 0) done.add(serverId) + return parsed +} + +async function tick(): Promise { + if (running) return + running = true + try { + const { listServers } = await import('./serverRegistry') + for (const s of listServers()) { + if (done.has(s.id)) continue + const n = await warmServer(s.id) + if (n > 0) log.info(`Tile cache: warmed ${n} region(s) of "${s.name}" in the background`) + } + } catch (err) { + log.warn('Tile warming pass failed: ' + String((err as Error)?.message ?? err)) + } finally { + running = false + } +} + +/** + * Start warming. + * + * The first pass waits: a map that is opened in the first few seconds should + * have the threads to itself, and nothing about this is urgent. + */ +export function startTileWarming(): void { + if (timer) return + timer = setInterval(() => void tick(), TICK_MS) + // Not the reason the process stays alive. + timer.unref?.() +} + +export function stopTileWarming(): void { + if (!timer) return + clearInterval(timer) + timer = null +} + +/** A world that changed is worth looking at again. */ +export function tileWarmingWake(serverId: string): void { + done.delete(serverId) +} diff --git a/src/main/core/tileWorker.ts b/src/main/core/tileWorker.ts new file mode 100644 index 0000000..efc3205 --- /dev/null +++ b/src/main/core/tileWorker.ts @@ -0,0 +1,88 @@ +/** + * A worker thread that turns one region file into encoded tiles (#160). + * + * Deliberately tiny, and deliberately importing only `regionParse` — anything + * that reaches for Electron, the server registry or the app paths would drag + * the main process's world in here, and there is no main process here. + * + * THE COLOUR TABLE. `blockColour` reads module-level state that the main + * process fills at runtime from the operator's client jar, and a worker starts + * with it empty. A worker that parsed without it would render the fallback + * palette, and the caller would write those colours into the on-disk cache, + * where they would outlive the process and keep serving a wrong map with a + * cache version that still matched. So a job carries the table whenever the + * pool believes this worker's copy is stale, and refuses to parse until it has + * been given one at least once. + */ +import { readFileSync, statSync } from 'node:fs' +import { gzipSync } from 'node:zlib' +import { parentPort } from 'node:worker_threads' +import { setTextureColours } from '@shared/regionFormat' +import type { Rgb } from '@shared/regionFormat' +import { encodeRegionTiles } from '@shared/tileCache' +import { parseRegionBuffer } from './regionParse' +import type { ChunkTile } from './regionParse' + +export interface TileJob { + id: number + path: string + dim: string + /** Present only when the pool thinks this worker's table is out of date. */ + colours?: Record + colourEpoch: number +} + +export interface TileJobResult { + id: number + /** gzipped `encodeRegionTiles` output, or null when there was nothing to read. */ + buf: Uint8Array | null + mtimeMs: number + chunks: number + error?: string +} + +/** -1 until the pool has sent one, which is what makes "never given" detectable. */ +let epoch = -1 + +function run(job: TileJob): TileJobResult { + if (job.colours) { + setTextureColours(job.colours) + epoch = job.colourEpoch + } + if (epoch !== job.colourEpoch) { + // The pool got the bookkeeping wrong. Refusing is the only safe answer: a + // parse now would look identical and be quietly the wrong colours. + return { id: job.id, buf: null, mtimeMs: 0, chunks: 0, error: 'stale-colours' } + } + let mtimeMs = 0 + let file: Buffer + try { + mtimeMs = statSync(job.path).mtimeMs + file = readFileSync(job.path) + } catch { + return { id: job.id, buf: null, mtimeMs: 0, chunks: 0, error: 'unreadable' } + } + const tiles = parseRegionBuffer(file, job.dim) + const usable = new Map() + for (const [slot, tile] of tiles) if (tile) usable.set(slot, tile) + // Encoded and gzipped HERE, on this thread. The alternative is posting a Map + // of 1024 objects across the thread boundary, where structured clone would + // rebuild every one of them on the main thread — which is the work this is + // supposed to be taking off it. + const buf = gzipSync(encodeRegionTiles({ mtimeMs, tiles: usable }), { level: 6 }) + return { id: job.id, buf, mtimeMs, chunks: usable.size } +} + +parentPort?.on('message', (job: TileJob) => { + try { + parentPort?.postMessage(run(job)) + } catch (err) { + parentPort?.postMessage({ + id: job.id, + buf: null, + mtimeMs: 0, + chunks: 0, + error: String((err as Error)?.message ?? err) + } satisfies TileJobResult) + } +}) diff --git a/src/main/core/worldTiles.ts b/src/main/core/worldTiles.ts index f5a4409..f664306 100644 --- a/src/main/core/worldTiles.ts +++ b/src/main/core/worldTiles.ts @@ -23,48 +23,25 @@ import { writeFileSync } from 'node:fs' import { createHash } from 'node:crypto' -import { inflateSync, gunzipSync, gzipSync } from 'node:zlib' +import { gunzipSync, gzipSync } from 'node:zlib' import { join } from 'node:path' import { cacheDir } from '../paths' import { MAX_TILES_PER_REQUEST } from '@shared/livemap' import { decodeRegionTiles, encodeRegionTiles, normalizeMapPerf } from '@shared/tileCache' import type { MapPerfConfig } from '@shared/tileCache' -import * as nbt from 'prismarine-nbt' import { getServer } from './serverRegistry' import { readProperties } from './serverFiles' import { log } from '../logger' -import { - bitsPerIndex, - blockColour, - chunkSlot, - indexAt, - localChunk, - packingFor, - parseLocationTable, - prepareIndices, - regionOf, - scanRuleFor, - seeThrough, - structureKind, - CHUNK_AXIS, - INVISIBLE -} from '@shared/regionFormat' +import { chunkSlot, localChunk, parseLocationTable, regionOf } from '@shared/regionFormat' +import { parseSlot, SECTOR_HEADER } from './regionParse' +import { parseOnWorker, poolReady } from './tilePool' +import type { ChunkTile } from './regionParse' import type { StructureMark } from '@shared/regionFormat' -/** A chunk's surface: 256 columns, row-major (x fastest). */ -export interface ChunkTile { - /** Packed 0xRRGGBB per column. */ - colour: number[] - /** World Y of the drawn block, for cross-chunk shading. */ - height: number[] - /** - * Structures starting in this chunk (#131). - * - * Read from the same NBT the surface came from, so it costs one more object - * lookup rather than a second pass over the world. Absent on most chunks. - */ - marks?: StructureMark[] -} +// The parse itself lives in `regionParse.ts`, which knows nothing about +// Electron so a worker thread can run it (#160). +export type { ChunkTile } from './regionParse' +export { tileFromChunk } from './regionParse' interface RegionEntry { at: number @@ -145,7 +122,24 @@ function writeCachedRegion(serverId: string, path: string, entry: RegionEntry): try { const usable = new Map() for (const [slot, tile] of entry.tiles) if (tile) usable.set(slot, tile) - const buf = gzipSync(encodeRegionTiles({ mtimeMs: entry.mtimeMs, tiles: usable }), { level: 6 }) + writeCachedBuffer( + serverId, + path, + gzipSync(encodeRegionTiles({ mtimeMs: entry.mtimeMs, tiles: usable }), { level: 6 }) + ) + } catch { + /* a cache that cannot be written still leaves a working map */ + } +} + +/** + * The same write, given the encoded bytes rather than the tiles. + * + * A worker already produced exactly this buffer, and re-encoding it here would + * be doing on the main thread the work the worker exists to take off it (#160). + */ +function writeCachedBuffer(serverId: string, path: string, buf: Uint8Array): void { + try { writtenSinceSweep += buf.length const f = cacheFileFor(serverId, path) // Through a temp file: a reader hitting a half-written cache would decode @@ -290,244 +284,6 @@ function regionPath(serverId: string, dim: string, rx: number, rz: number): stri return dir ? join(dir, `r.${rx}.${rz}.mca`) : null } -/** Longs out of prismarine-nbt, which gives signed 64-bit values as [hi, lo]. */ -function toLongs(raw: unknown): bigint[] { - if (!Array.isArray(raw)) return [] - return raw.map((v) => { - if (typeof v === 'bigint') return v - if (Array.isArray(v) && v.length === 2) { - // [high, low], both signed 32-bit. The high word carries the sign. - return BigInt.asIntN(64, (BigInt(v[0]) << 32n) | (BigInt(v[1] >>> 0) & 0xffffffffn)) - } - if (typeof v === 'number') return BigInt(Math.trunc(v)) - return 0n - }) -} - -/* eslint-disable @typescript-eslint/no-explicit-any */ -function tag(v: any): any { - return v && typeof v === 'object' && 'value' in v ? v.value : v -} - -/** - * Unwrap an NBT *list*, which prismarine-nbt wraps twice. - * - * A list arrives as `{type:'list', value:{type:'compound', value:[...]}}` — the - * outer wrapper says "list", the inner one says what the elements are. One - * `tag()` leaves you holding the inner descriptor object, not the array. - * - * This is not a nicety. Reading `sections` happened to work because the code - * unwrapped it twice by accident, while `palette` was unwrapped once — so - * `Array.isArray(palette)` was false for every section of every chunk, every - * section was skipped, and the world renderer produced nothing at all. The - * smoke tested the bit decoding and never a real chunk, so nothing caught it. - */ -function listOf(v: any): any[] { - const once = tag(v) - const twice = tag(once) - return Array.isArray(twice) ? twice : Array.isArray(once) ? once : [] -} - -/** - * The topmost visible block of every column in one chunk. - * - * Sections are walked from the highest down, and within a section from y=15 - * down, stopping at the first block that is not air. A column that is nothing - * but air the whole way — under an unlit sky, or a chunk that is only partly - * generated — is left transparent rather than drawn as the void. - */ -export function tileFromChunk(chunk: any, dim = 'overworld'): ChunkTile | null { - const v = tag(chunk) - if (!v) return null - const dataVersion = tag(v.DataVersion) - const packing = packingFor(typeof dataVersion === 'number' ? dataVersion : undefined) - // 1.18+ uses `sections`; 1.13-1.17 used `Level.Sections`. - const sections = listOf(v.sections).length ? listOf(v.sections) : listOf(tag(v.Level)?.Sections) - if (!sections.length) return null - - const withY = sections - .map((s: any) => ({ s: tag(s), y: Number(tag(tag(s)?.Y)) })) - .filter((x) => Number.isFinite(x.y)) - .sort((a, b) => b.y - a.y) - - const colour = new Array(CHUNK_AXIS * CHUNK_AXIS).fill(-1) - const height = new Array(CHUNK_AXIS * CHUNK_AXIS).fill(0) - let remaining = colour.length - - // The nether has a bedrock roof: a top-down scan finds it in every column and - // paints the whole dimension one flat grey. `sawAir` per column is how a map - // gets under it — solid blocks are skipped until an air gap has been seen. - const rule = scanRuleFor(dim) - const sawAir = rule.underRoof ? new Array(colour.length).fill(false) : null - // A column that is solid from the ceiling all the way down — a netherrack - // pillar joining floor to roof — never shows an air gap, so the under-roof - // rule would skip every block in it and leave a hole. The highest solid block - // seen while skipping is kept as the answer for exactly that case. - const fallbackColour = sawAir ? new Array(colour.length).fill(-1) : null - const fallbackHeight = sawAir ? new Array(colour.length).fill(0) : null - - for (const { s, y: sectionY } of withY) { - if (remaining === 0) break - // Above the ceiling there is nothing worth looking at, and on the nether - // that is most of the sections. - if (rule.ceiling !== null && sectionY * CHUNK_AXIS > rule.ceiling) continue - const states = tag(s.block_states) ?? tag(s.BlockStates) - // A list, so unwrapped twice. See `listOf`. - const paletteRaw = listOf(states?.palette).length ? listOf(states?.palette) : listOf(s.Palette) - if (!paletteRaw.length) continue - const names: string[] = paletteRaw.map((p: any) => String(tag(tag(p)?.Name) ?? '')) - - /** - * The block rules, resolved once per PALETTE ENTRY. - * - * This is the whole cost of a map (#157). A section is 4096 positions and - * its palette is at most a few dozen entries, and every one of these used - * to run per position: a regex to strip the namespace, a Set lookup for - * air, `seeThrough` (a Set miss then ten `endsWith` calls), and - * `blockColour` with a second regex inside it. A fully generated chunk is - * about 53 thousand of those to render 256 columns — 13 ms a chunk, 14 - * seconds a region, and the reason a viewport could take over a minute. - * - * Resolved per entry it is a few dozen, and the inner loop is array - * indexing. Same answers: the arrays are built by the same functions in the - * same order. - */ - const invisible = names.map((n) => { - // Air, and the plants a map looks through — see `seeThrough`. Without it - // the surface is whatever is standing ON the ground rather than the - // ground, which is how a bamboo jungle rendered as a maroon smear. - const short = n.replace(/^minecraft:/, '') - return !short || INVISIBLE.has(short) || seeThrough(short) - }) - // A section a map sees nothing in — and above the surface that is most of - // them, fifteen or so single-entry air palettes per chunk. Skipped HERE, - // before the colours are looked up, before the indices are unpacked and - // before anything is allocated, because the point is not to make those - // sections cheaper but to stop touching them at all. - if (invisible.every(Boolean)) { - // Except under a roof, where an air section is not nothing: it is the gap - // the scan is looking for. Recorded for every column in one pass instead - // of by walking 4096 positions to reach the same conclusion. Every - // section still standing here has its bottom layer at or below the - // ceiling, so a full air layer covers all 256 columns. - if (sawAir) sawAir.fill(true) - continue - } - - // Only now, for the sections that can actually contribute a colour. - const packedColour = names.map((n) => { - const c = blockColour(n.replace(/^minecraft:/, '')) - return (c.r << 16) | (c.g << 8) | c.b - }) - - // A section whose palette is one entry has no data array at all — it is - // 4096 of that block, which is how a solid stone or all-air section is - // stored. Reading `data` there would skip the section entirely. - // `data` is a longArray, which is wrapped once — unlike the palette beside - // it, which is a list and wrapped twice. - const longs = toLongs(tag(states?.data) ?? tag(s.BlockStates)) - const bits = bitsPerIndex(names.length) - // `null` rather than 4096 zeroes: a uniform section reads palette entry 0 - // at every position, so the array only existed to say so. - // - // And prepared rather than unpacked: the loop below walks down from the top - // layer and stops the moment every column has an answer, which on real - // terrain is after two or three of the sixteen. Unpacking all 4096 did the - // rest for nothing. - const indices = - names.length === 1 || !longs.length ? null : prepareIndices(longs, bits, packing) - - for (let y = CHUNK_AXIS - 1; y >= 0 && remaining > 0; y--) { - // Neither of these depends on the column, and both used to be recomputed - // 256 times a layer. - const worldY = sectionY * CHUNK_AXIS + y - if (rule.ceiling !== null && worldY > rule.ceiling) continue - const base = y * 256 - // `x + z * CHUNK_AXIS` IS the column index, so the two loops the original - // had over x and z collapse into one over the column in the same order. - for (let col = 0; col < 256; col++) { - if (colour[col] >= 0) continue - const pi = indices ? indexAt(indices, base + col) : 0 - // An index past the end of its palette: the width comes from - // `bitsPerIndex`, which rounds up, so a three-entry palette is read - // four bits wide and corrupt data can address entry 15. The old code - // resolved that to an empty name and treated it as invisible. - const inv = pi >= names.length || invisible[pi] - if (sawAir) { - // Under a roof: remember the gap, and skip everything solid until - // one has been seen. Without this the first hit is the roof itself. - if (inv) sawAir[col] = true - if (!sawAir[col]) { - if (!inv && fallbackColour && fallbackColour[col] < 0) { - fallbackColour[col] = packedColour[pi] - if (fallbackHeight) fallbackHeight[col] = worldY - } - continue - } - } - if (inv) continue - colour[col] = packedColour[pi] - height[col] = worldY - remaining-- - } - } - } - // Columns the under-roof rule skipped entirely fall back to the highest solid - // block, so a floor-to-ceiling pillar is drawn rather than punched out. - if (fallbackColour && fallbackHeight) { - for (let i = 0; i < colour.length; i++) { - if (colour[i] < 0 && fallbackColour[i] >= 0) { - colour[i] = fallbackColour[i] - height[i] = fallbackHeight[i] - remaining-- - } - } - } - // Nothing at all: an ungenerated or empty chunk, which is not a tile. - if (remaining === colour.length) return null - const marks = structuresOf(v) - return { colour, height, ...(marks.length ? { marks } : {}) } -} - -/** - * Structures whose start is in this chunk. - * - * `structures.starts` is keyed by structure id and each entry carries the chunk - * it starts in — `ChunkX`/`ChunkZ` in chunk units, which is why they are - * multiplied here rather than used raw. A chunk that merely CONTAINS part of a - * structure lists it in `References`, not `starts`, so this yields one mark per - * structure rather than one per chunk it sprawls across. - */ -function structuresOf(v: any): StructureMark[] { - const starts = tag(tag(v.structures)?.starts) ?? tag(tag(tag(v.Level)?.Structures)?.Starts) - if (!starts || typeof starts !== 'object') return [] - const out: StructureMark[] = [] - for (const [id, raw] of Object.entries(starts)) { - const s = tag(raw) - if (!s || typeof s !== 'object') continue - const cx = Number(tag((s as any).ChunkX)) - const cz = Number(tag((s as any).ChunkZ)) - if (!Number.isFinite(cx) || !Number.isFinite(cz)) continue - out.push({ - kind: structureKind(id), - id: String(id).replace(/^minecraft:/, ''), - x: cx * CHUNK_AXIS + CHUNK_AXIS / 2, - z: cz * CHUNK_AXIS + CHUNK_AXIS / 2 - }) - } - return out -} - -function decompress(buf: Buffer, kind: number): Buffer | null { - try { - if (kind === 1) return gunzipSync(buf) - if (kind === 2) return inflateSync(buf) - if (kind === 3) return buf - } catch { - /* a truncated or corrupt chunk is skipped, never fatal */ - } - return null -} /** * The smallest gap between two region parses. @@ -557,33 +313,6 @@ export function parseBudgetReady(serverId?: string, now = Date.now()): boolean { * Synchronous and slow by design — the callers are expected to keep this off * any request path, and to respect `parseBudgetReady`. */ -/** - * One chunk out of a region file, into `tiles`. - * - * Pulled out of the parse loop so the same work can be done in one pass or in - * slices with the event loop running in between — see `SLICE_SLOTS`. - */ -function parseSlot( - file: Buffer, - table: ReturnType, - slot: number, - tiles: Map, - dim: string -): void { - const loc = table[slot] - if (!loc || !loc.offset || loc.offset + 5 > file.length) return - const length = file.readUInt32BE(loc.offset) - const kind = file[loc.offset + 4] - const end = loc.offset + 5 + Math.max(0, length - 1) - if (length <= 0 || end > file.length) return - const raw = decompress(file.subarray(loc.offset + 5, end), kind) - if (!raw) return - try { - tiles.set(slot, tileFromChunk(nbt.parseUncompressed(raw), dim)) - } catch { - /* one unreadable chunk must not lose the region */ - } -} /** * How many of a region's 1024 chunks to parse before letting the event loop @@ -723,6 +452,38 @@ async function loadRegionSliced( regions.set(path, empty) return empty } + + // A worker thread if there is one (#160). The parse is about 1.4 s and this + // thread answers every IPC call, serves the web panel and reads the console; + // slicing made that interruptible, not absent. Off-thread it is absent, and + // several regions parse at once. + // + // The worker hands back the ENCODED region, already gzipped, which is also + // exactly what the disk cache stores — so a hit costs one decode either way + // and nothing rebuilds 1024 objects across a thread boundary. + if (poolReady()) { + const r = await parseOnWorker(path, dim) + if (!r.error && r.buf) { + const decoded = decodeRegionTiles(gunzipSync(r.buf)) + if (decoded) { + const entry: RegionEntry = { at: Date.now(), mtimeMs: r.mtimeMs, tiles: new Map(decoded.tiles) } + regions.set(path, entry) + trimMemory(perf.memoryRegions) + if (perf.cache) { + writeCachedBuffer(serverId, path, r.buf) + sweepCache(perf.cacheLimitMB) + } + // Nothing blocked this thread, so there is no slice to report. + lastSliceMs = 0 + log.info(`World tiles: parsed ${r.chunks} chunks from ${path.split(/[\/]/).pop()} on a worker`) + return entry + } + } + if (r.error === 'unreadable') return giveUp() + // Anything else falls through to the on-thread parse below rather than + // showing the operator an empty map. + } + let file: Buffer try { file = readFileSync(path) @@ -788,7 +549,6 @@ function trimMemory(keep: number): void { } } -const SECTOR_HEADER = 8192 /** * A tile ONLY if its region is already parsed. @@ -982,3 +742,89 @@ export function chunkTile( return region.tiles.get(chunkSlot(localChunk(chunkX), localChunk(chunkZ))) ?? null } + +// ---- what the background warmer needs (#161) ---- + +/** + * Every dimension directory this server actually has on disk. + * + * The warmer cannot ask for "the region folder" — a Paper server keeps the + * nether and the end in sibling folders, and a custom world is a top-level one + * of its own, which is the layout `regionDirCandidates` exists to resolve. + */ +export function regionDirsForServer(serverId: string): { dim: string; dir: string }[] { + const out: { dim: string; dir: string }[] = [] + const seen = new Set() + for (const dim of ['overworld', 'nether', 'end']) { + const dir = regionDirFor(serverId, dim) + if (!dir || seen.has(dir) || !existsSync(dir)) continue + seen.add(dir) + out.push({ dim, dir }) + } + return out +} + +/** + * Whether the cache already holds this region as it stands on disk. + * + * Cheap on purpose: the warmer asks this once per region per pass, and on a + * warmed world the answer is yes for every one of them. Reading the file and + * checking its mtime is what makes a second pass cost a directory walk rather + * than a world. + */ +export function cachedRegionIsCurrent(serverId: string, path: string, mtimeMs: number): boolean { + const hit = regions.get(path) + if (hit && hit.mtimeMs === mtimeMs) return true + return !!readCachedRegion(serverId, path, mtimeMs) +} + +/** + * Parse one region into the cache, without touching the in-memory working set. + * + * Deliberately NOT `loadRegionSliced`: warming a thousand regions through that + * would evict the handful an operator is actually looking at, over and over. + * The disk cache is the point here; memory belongs to whoever is looking. + */ +export async function warmOneRegion( + serverId: string, + path: string, + dim: string, + perf: MapPerfConfig +): Promise { + if (!poolReady()) return false + const r = await parseOnWorker(path, dim) + if (r.error || !r.buf) return false + if (perf.cache) { + writeCachedBuffer(serverId, path, r.buf) + sweepCache(perf.cacheLimitMB) + } + return true +} + +/** + * Bytes currently held in the on-disk tile cache. + * + * The warmer reads this to know when to stop. It matters because the two + * policies point in opposite directions: the warmer writes NEAREST TO SPAWN + * FIRST, and `sweepCache` evicts OLDEST FIRST — so on a world bigger than the + * cache limit, warming would have spent its time deleting the area around + * spawn, the part a map is actually pointed at, and keeping the far corners it + * happened to write last. + */ +export function tileCacheBytes(): number { + try { + const dir = cacheDirFor() + let total = 0 + for (const name of readdirSync(dir)) { + if (!name.endsWith('.tiles')) continue + try { + total += statSync(join(dir, name)).size + } catch { + /* a file that vanished mid-scan is not in the cache */ + } + } + return total + } catch { + return 0 + } +} diff --git a/src/main/index.ts b/src/main/index.ts index f1f4c08..55a8916 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -15,6 +15,7 @@ import { initAlerts } from './core/alerts' import { initBlockColours } from './core/clientAssets' import { resolveBaseDir } from './paths' import { log } from './logger' +import { startTileWarming } from './core/tileWarm' import { runSmoke, runWizardSmoke, @@ -387,6 +388,10 @@ if (!gotLock) { // Before the web server, because the public map draws from the same table. initBlockColours() initWebServer() + // AFTER the colour table, never before: a warmer that ran first would parse + // the world with the fallback palette and write those colours into the + // cache, where they would outlive the process (#160, #161). + startTileWarming() createWindow() app.on('activate', () => { diff --git a/src/main/smoke.ts b/src/main/smoke.ts index 80e2fa0..05a0979 100644 --- a/src/main/smoke.ts +++ b/src/main/smoke.ts @@ -131,11 +131,15 @@ import * as worldsMod from './core/worlds' import * as areasMod from '@shared/chunkAreas' import * as areasMod2 from './core/chunkAreas' import * as tilesMod from './core/worldTiles' +import { parseRegionBuffer } from './core/regionParse' +import type { ChunkTile } from './core/regionParse' +import * as warmMod from './core/tileWarm' +import { parseOnWorker, poolReady, _setTileWorkersEnabled } from './core/tilePool' import * as tex from '@shared/textures' import { MAP_CSS, MAP_HTML, MAP_JS } from '@shared/mapUi' import { getMapPageHtml } from './web/mapPageHtml' import * as pngMod from './core/png' -import { deflateSync } from 'node:zlib' +import { deflateSync, gunzipSync } from 'node:zlib' import * as assetsMod from './core/clientAssets' import { isValidMcName, @@ -2489,7 +2493,14 @@ export async function runWorldsSmoke(): Promise { // The queue's path parses in slices. What matters is not that it is // faster — it is the same work — but that no single uninterrupted block // is long enough to be felt: this thread answers every IPC call, so a - // 180 ms block is 180 ms of frozen interface while the map loads. + // long block is that long with the interface frozen while the map loads. + // + // Workers OFF for this one. They are the normal path now (#160) and the + // main thread does not block at all with them on, which would leave this + // gate measuring nothing — the same way its fixture used to (#157). The + // sliced parse is still what runs when a thread cannot be spawned, so it + // still has to hold. + _setTileWorkersEnabled(false) tilesMod.clearTileCache(SID) const t0 = Date.now() await tilesMod.chunkTileSliced(SID, 'overworld', 0, 0) @@ -2507,8 +2518,103 @@ export async function runWorldsSmoke(): Promise { } if (slice > 60) return fail('a parse slice blocked for ' + slice + ' ms; the interface will stutter') console.log( - 'WORLDS-SMOKE: region parsed in ' + total + ' ms, longest uninterrupted slice ' + slice + ' ms' + 'WORLDS-SMOKE: region parsed on-thread in ' + total + ' ms, longest uninterrupted slice ' + slice + ' ms' ) + _setTileWorkersEnabled(true) + } + + // --- 12a-ii. the worker parses, and parses the SAME map (#160) ---------- + { + // The hazard this whole design is arranged around: `blockColour` reads + // module-level state that the client jar fills at runtime, and a worker + // thread starts with it EMPTY. A worker that was not given the table + // renders the fallback palette, and the result is written to the on-disk + // cache — a wrong map that survives restarts, with a cache version that + // still matches. So the colours are set to something unmistakable here + // and the worker's answer has to carry them. + const level = + /^level-name=(.+)$/m.exec(readFileSync(join(root, 'server.properties'), 'utf-8'))?.[1]?.trim() || + 'world' + const rpath = join(root, level, 'region', 'r.0.0.mca') + setTextureColours({ stone: { r: 7, g: 11, b: 13 } }) + tilesMod.clearTileCache(SID) + tilesMod._resetWorldTiles() + + if (!poolReady()) return fail('the tile worker pool would not start') + const t0 = Date.now() + const r = await parseOnWorker(rpath, 'overworld') + const took = Date.now() - t0 + if (r.error) return fail('the worker refused the region: ' + r.error) + if (!r.buf) return fail('the worker returned no tiles') + if (r.chunks !== 1024) return fail('the worker parsed ' + r.chunks + ' chunks, expected 1024') + + const decoded = decodeRegionTiles(gunzipSync(r.buf)) + if (!decoded) return fail('the worker produced a region that does not decode') + const first = decoded.tiles.get(0) + if (!first) return fail('the worker produced no tile for chunk 0') + // The fixture's surface band is a 16-block palette, so not every column + // is stone — but every column must be a colour this table could produce, + // and the fallback palette contains none of these. + const wantStone = (7 << 16) | (11 << 8) | 13 + if (!first.colour.includes(wantStone)) { + return fail('the worker did not use the colour table it was given') + } + + // And byte for byte what this thread would have produced. Two renderers + // that agree on a smoke fixture and disagree on a real world is the + // failure mode; comparing the ENCODED region compares every column of + // every chunk rather than a sample. + const here = parseRegionBuffer(readFileSync(rpath), 'overworld') + const usable = new Map() + for (const [slot, tile] of here) if (tile) usable.set(slot, tile) + const mine = encodeRegionTiles({ mtimeMs: r.mtimeMs, tiles: usable }) + const theirs = gunzipSync(r.buf) + if (Buffer.compare(Buffer.from(mine), Buffer.from(theirs)) !== 0) { + return fail('the worker and the main thread rendered different maps') + } + setTextureColours({}) + console.log('WORLDS-SMOKE: worker parsed 1024 chunks in ' + took + ' ms, identical to this thread') + } + + // --- 12a-iii. the background warmer (#161) ------------------------------ + { + // Warming is only worth doing into a cache, and the operator can turn the + // cache off — in which case a warmer would be pure cost, parsing a world + // and throwing every bit of it away. + const restore = registry.getServer(SID)?.map + registry.updateServer(SID, { map: normalizeMapPerf({ ...(restore ?? {}), cache: false }) }) + tilesMod.clearTileCache(SID) + warmMod._resetTileWarm() + const off = await warmMod.warmServer(SID, 4) + if (off !== 0) return fail('warming ran with the cache off: ' + off) + + // With it on it really warms, and what it wrote is what the map reads: + // the second pass finds everything current and parses nothing. + registry.updateServer(SID, { map: normalizeMapPerf({ ...(restore ?? {}), cache: true }) }) + warmMod._resetTileWarm() + const first = await warmMod.warmServer(SID, 4) + if (first < 1) return fail('warming with the cache on parsed nothing') + const second = await warmMod.warmServer(SID, 4) + if (second !== 0) { + return fail('warming re-parsed regions it had already cached: ' + second) + } + // A full cache means STOP. The warmer writes nearest to spawn first and + // the sweep evicts oldest first, so carrying on would delete the area + // around spawn and keep whichever far corner was written last — warming + // would actively make the map worse on a world bigger than the limit. + registry.updateServer(SID, { map: normalizeMapPerf({ ...(restore ?? {}), cache: true, cacheLimitMB: 0 }) }) + warmMod._resetTileWarm() + tilesMod.clearTileCache(SID) + const capped = await warmMod.warmServer(SID, 4) + if (capped !== 0) return fail('warming ran past a full cache: ' + capped) + if (tilesMod.tileCacheBytes() !== 0) return fail('warming wrote into a cache it had no room in') + if (restore) registry.updateServer(SID, { map: restore }) + + // Nearest the origin first: spawn is where a map is first pointed, and a + // world accumulates far corners from one player who once walked a long way. + const order = warmMod.warmOrder(join(root, 'paper_world', 'region')) + const seen = order.filter((n) => /^r\.-?\d+\.-?\d+\.mca$/.test(n)) + if (order.length !== seen.length) return fail('warmOrder returned a non-region file') } // --- 12b. item textures from the client jar (#127) ---------------------- diff --git a/src/shared/regionFormat.ts b/src/shared/regionFormat.ts index de74e3b..594b69f 100644 --- a/src/shared/regionFormat.ts +++ b/src/shared/regionFormat.ts @@ -428,8 +428,29 @@ export const WATERY = new Set(['water', 'bubble_column']) */ let textureColours: Record = {} +/** + * Bumped on every change, so a WORKER can be told whether its copy is current + * (#160). + * + * A worker thread starts with an empty table and renders the fallback palette, + * and the caller writes what it produces into the on-disk cache — a wrong map + * that outlives the process, with a cache version that still matches. Comparing + * a number is how the pool knows to re-send the table instead of assuming. + */ +let colourEpoch = 0 + export function setTextureColours(map: Record): void { textureColours = map || {} + colourEpoch++ +} + +export function textureColourEpoch(): number { + return colourEpoch +} + +/** The table itself, for handing to a worker. */ +export function textureColours_(): Record { + return textureColours } export function textureColourCount(): number {