Add type-strip mode: run Harper directly from .ts sources

.github/workflows/review-coverage.yml

@@ -122,6 +122,7 @@ jobs:
             dataLayer/backupRepository.ts
             dataLayer/harperBridge/ResourceBridge.ts
             dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js
+            dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts
             dataLayer/restoreMarker.ts
             dataLayer/rocksdbBackup.ts
             dataLayer/schemaDescribe.ts

DESIGN.md

@@ -121,6 +121,7 @@ Index of the design notes for the harper core: one line per note, grouped by the
 
 - [Three HTTP stacks coexist — know which one](server/DESIGN.md#three-http-stacks-coexist--know-which-one) — The native `http.ts` stack, the Fastify operations API and the legacy Fastify custom-functions server, and which one a request lands on.
 - [File overview](server/DESIGN.md#file-overview) — What each file in `server/` owns.
+- [Native and compiled runtime startup](server/DESIGN.md#native-and-compiled-runtime-startup) — Runtime paths stay canonical; cyclic worker registration remains synchronous, and only an HTTP worker entry starts listeners.
 - [`http.ts` — symbol map](server/DESIGN.md#httpts--symbol-map) — Symbol index for the native HTTP entry file.
 - [Operations authorization boundary](server/DESIGN.md#operations-authorization-boundary) — Operations request bodies are untrusted: trusted dispatch state rides async context, and a worker-registered fact reaches main only through `OPERATION_REGISTERED`.
 - [Resource ↔ HTTP boundary](server/DESIGN.md#resource--http-boundary) — How a request becomes a Resource call and a result becomes a response.
@@ -136,13 +137,13 @@ Index of the design notes for the harper core: one line per note, grouped by the
 - [`universalHeaders` (`http.securityHeaders`): ownership, precedence, and per-thread scope](server/DESIGN.md#universalheaders-httpsecurityheaders-ownership-precedence-and-per-thread-scope) — `applySecurityHeaders` splices only the tuples it owns; only the root `http` scope owns them; a header the app or route set wins; the array is per thread.
 - [Under Bun, the main HTTP port is served by `node:http`, not `Bun.serve`](server/DESIGN.md#under-bun-the-main-http-port-is-served-by-nodehttp-not-bunserve) — Under Bun the default port is a `node:http` server because WebSockets need it; no `Bun.serve` config is registered for it.
 - [Per-worker UDS mirrors are separate server instances — port-keyed wiring does not reach them (`server/http.ts`)](server/DESIGN.md#per-worker-uds-mirrors-are-separate-server-instances--port-keyed-wiring-does-not-reach-them-serverhttpts) — Per-worker UDS mirrors are separate `http.Server` instances; port-keyed wiring must be propagated through `server.udsMirror`.
-- [A per-thread UDS mirror is bound at a temp name and renamed over its published path (`server/threads/threadServer.js`)](server/DESIGN.md#a-per-thread-uds-mirror-is-bound-at-a-temp-name-and-renamed-over-its-published-path-serverthreadsthreadserverjs) — libuv unlinks a pipe server's path on close regardless of owner; rename keeps the listening inode under the published name so an overlapping restart's outgoing worker cannot delete the replacement's mirror.
-- [A worker that misses an ITC ack gets its OS thread state logged (`server/threads/manageThreads.js`)](server/DESIGN.md#a-worker-that-misses-an-itc-ack-gets-its-os-thread-state-logged-serverthreadsmanagethreadsjs) — A worker that misses a broadcast ack gets its OS thread state logged.
-- [A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.js`)](server/DESIGN.md#a-workers-parentport-close-is-not-the-main-threads-exit-serverthreadsmanagethreadsjs) — A worker's `parentPort` closing is never announced as the main thread's exit.
+- [A per-thread UDS mirror is bound at a temp name and renamed over its published path (`server/threads/threadServer.ts`)](server/DESIGN.md#a-per-thread-uds-mirror-is-bound-at-a-temp-name-and-renamed-over-its-published-path-serverthreadsthreadserverts) — libuv unlinks a pipe server's path on close regardless of owner; rename keeps the listening inode under the published name so an overlapping restart's outgoing worker cannot delete the replacement's mirror.
+- [A worker that misses an ITC ack gets its OS thread state logged (`server/threads/manageThreads.ts`)](server/DESIGN.md#a-worker-that-misses-an-itc-ack-gets-its-os-thread-state-logged-serverthreadsmanagethreadsts) — A worker that misses a broadcast ack gets its OS thread state logged.
+- [A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.ts`)](server/DESIGN.md#a-workers-parentport-close-is-not-the-main-threads-exit-serverthreadsmanagethreadsts) — A worker's `parentPort` closing is never announced as the main thread's exit.
 - [`chooseOperation` authorizes the invoked operation against the authenticated principal (`server/serverHelpers/serverUtilities.ts`)](server/DESIGN.md#chooseoperation-authorizes-the-invoked-operation-against-the-authenticated-principal-serverserverhelpersserverutilitiests) — The principal comes only from top-level `hdb_user`; `search_operation` is the subject only for export ops; `parsed_sql_object` is never client input.
 - [`withNodeAdapter()`'s response is the body `PassThrough` it resolves with (`server/serverHelpers/NodeAdapterResponse.ts`)](server/DESIGN.md#withnodeadapters-response-is-the-body-passthrough-it-resolves-with-serverserverhelpersnodeadapterresponsets) — The adapter response is the `PassThrough` it resolves with; headers commit once through `writeHead`; Express is not a target.
 - [Dedicated worker pools own their listeners and load no applications (`server/threads/workerPools.ts`)](server/DESIGN.md#dedicated-worker-pools-own-their-listeners-and-load-no-applications-serverthreadsworkerpoolsts) — A `replication` pool loads only trusted built-ins, binds only the listeners claimed with `threadType`, stays active while it restarts, and restarts only on an operator restart.
-- [`manageThreads` has two different `workerCount`s (`server/threads/manageThreads.js`)](server/DESIGN.md#managethreads-has-two-different-workercounts-serverthreadsmanagethreadsjs) — The module-global `workerCount` sizes only the rolling-restart throttle; `getWorkerCount()` reads `workerData.workerCount`, frozen at spawn.
+- [`manageThreads` has two different `workerCount`s (`server/threads/manageThreads.ts`)](server/DESIGN.md#managethreads-has-two-different-workercounts-serverthreadsmanagethreadsts) — The module-global `workerCount` sizes only the rolling-restart throttle; `getWorkerCount()` reads `workerData.workerCount`, frozen at spawn.
 - [A WebSocket close reason must be bounded to 123 bytes (`server/serverHelpers/webSocketCloseReason.ts`)](server/DESIGN.md#a-websocket-close-reason-must-be-bounded-to-123-bytes-serverserverhelperswebsocketclosereasonts) — `ws` throws past 123 bytes from a rejection handler; every dynamic reason goes through `toCloseReason()`, and all three terminal handlers render the error code alike.
 - [`serverErrorHandler` skips what `handlePostRequest` already logged (`server/serverHelpers/serverHandlers.js`)](server/DESIGN.md#servererrorhandler-skips-what-handlepostrequest-already-logged-serverserverhelpersserverhandlersjs) — One Error line per failure raised before an operation runs; an error thrown while it runs is also logged by `OperationFunctionCaller`.
 - [A request-queue shed is a 503 the server never logged (`server/throttle.ts`)](server/DESIGN.md#a-request-queue-shed-is-a-503-the-server-never-logged-serverthrottlets) — the HTTP request-queue throttle answers a literal 503 body without throwing, so no error-path log line exists; its per-instance warn names the queue.
@@ -204,6 +205,8 @@ Index of the design notes for the harper core: one line per note, grouped by the
 
 ## utility/
 
+- [Core runtime and product paths (`packageUtils.js`)](utility/DESIGN.md#core-runtime-and-product-paths-packageutilsjs) — Core files and schema assets follow the module tree; metadata, product startup and public relative-worker paths retain their package owner.
+
 - [Every path handed to a native file watch must be canonicalized (`utility/watchPath.ts`)](utility/DESIGN.md#every-path-handed-to-a-native-file-watch-must-be-canonicalized-utilitywatchpathts) — Every native file-watch path is canonicalized first: libuv aborts the process on a Windows 8.3 short-path mismatch.
 - [Interactive CLI prompts go through `utility/interactivePrompts.ts`](utility/DESIGN.md#interactive-cli-prompts-go-through-utilityinteractivepromptsts) — Every `@inquirer` prompt uses this seam, which lazy-loads packages off the boot path, exits 130 on Ctrl-C and gives tests a stubbable raw layer.
 - [An HdbError's `message` is a string; the structured body is `http_resp_msg` (`utility/errors/hdbError.ts`)](utility/DESIGN.md#an-hdberrors-message-is-a-string-the-structured-body-is-http_resp_msg-utilityerrorshdberrorts) — A report object passed to `handleHDBError` is the response body and the job message; `message` is a string derived from it.

agent/agent.ts

@@ -21,7 +21,7 @@ import * as env from '../utility/environment/environmentManager.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
 import { Models } from '../resources/models/Models.ts';
 import type { AuthedUser } from '../components/mcp/toolRegistry.ts';
-import { workers } from '../server/threads/manageThreads.js';
+import { workers } from '../server/threads/manageThreads.ts';
 import { composeToolset } from './toolset.ts';
 import { buildInspectorTools } from './tools/inspectorTool.ts';
 import { buildHttpFetchTool, resolveHttpFetchConfig } from './tools/httpFetchTool.ts';

bin/agentCli.ts

@@ -18,7 +18,7 @@ import * as readline from 'node:readline';
 import { loadCredentials, normalizeTarget, extractTargetCredentials } from './cliCredentials.ts';
 import { refreshExpiredOperationToken } from './cliOperations.ts';
 import { httpRequest } from '../utility/common_utils.ts';
-import { getHdbPid } from '../utility/processManagement/processManagement.js';
+import { getHdbPid } from '../utility/processManagement/processManagement.ts';
 import { initConfig, getConfigPath } from '../config/configUtils.ts';
 import * as terms from '../utility/hdbTerms.ts';
 

bin/backup.ts

@@ -1,4 +1,5 @@
-'use strict';
+import dotenv from 'dotenv';
+('use strict');
 
 import { createWriteStream } from 'node:fs';
 import { rename, unlink } from 'node:fs/promises';
@@ -12,7 +13,7 @@ import { buildRequest, cliOperations, resolveRequestOptions } from './cliOperati
 import { loadCredentials } from './cliCredentials.ts';
 import { httpRequest } from '../utility/common_utils.ts';
 import { initConfig } from '../config/configUtils.ts';
-import { getHdbPid } from '../utility/processManagement/processManagement.js';
+import { getHdbPid } from '../utility/processManagement/processManagement.ts';
 import {
 	createBackupOffline,
 	deleteBackupOffline,
@@ -37,7 +38,7 @@ export async function runBackupCommand(command: string): Promise<void> {
 	// the local-vs-remote routing off HARPER_CLI_TARGET/CLI_TARGET, and without this a `.env`-
 	// configured remote target would be invisible, silently running a destructive op locally
 	// (cliOperations does the same on its first line).
-	require('dotenv').config();
+	dotenv.config();
 	const request = buildRequest();
 	const databaseName = request.database || 'data';
 

bin/cliOperations.ts

@@ -3,11 +3,11 @@
 import { loadCredentials, saveCredentials, normalizeTarget, extractTargetCredentials } from './cliCredentials.ts';
 import { isJWTExpired } from '../security/tokenAuthentication.ts';
 import * as envMgr from '../utility/environment/environmentManager.ts';
-envMgr.initSync();
 import * as terms from '../utility/hdbTerms.ts';
 import { httpRequest } from '../utility/common_utils.ts';
 import { workloadIdentityAvailable, exchangeWorkloadIdentityForToken } from './workloadIdentity.ts';
-import * as fs from 'fs-extra';
+import fs from 'fs-extra';
+import dotenv from 'dotenv';
 import * as YAML from 'yaml';
 import { Readable } from 'node:stream';
 import { execFileSync } from 'node:child_process';
@@ -16,13 +16,18 @@ import { encode as encodeCbor } from 'cbor-x';
 import { buildMultipartBody } from './multipartBuilder.ts';
 import { parseSSE } from './sseConsumer.ts';
 import { DeployRenderer } from './deployRenderer.ts';
-import { getHdbPid } from '../utility/processManagement/processManagement.js';
+import { getHdbPid } from '../utility/processManagement/processManagement.ts';
 import { initConfig, getConfigPath } from '../config/configUtils.ts';
 // The `deploy setup` seal and this by-reference flag both name the same hdb_secret row, and the
 // server re-derives it from its own request — so the derivation lives in one dependency-free module
 // rather than being restated per caller (it also keeps `components/` off the CLI's import graph).
 import { deriveGitSecretName, directoryProjectName, normalizeGitHost } from '../utility/componentNames.ts';
 
+try {
+	envMgr.initSync();
+} catch {
+	/* tolerate ESM cycle TDZ; bin entry will re-call later */
+}
 const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' };
 
 // Shown for any local-instance connection failure (missing pid, missing/stale domain
@@ -881,7 +886,7 @@ export async function resolveRequestOptions(req: any): Promise<{ options: any; t
 }
 
 async function cliOperations(req: any, skipResponseLog = false) {
-	require('dotenv').config();
+	dotenv.config();
 
 	// Resolve target/auth inside the try so a credential or connection error (e.g. an incomplete
 	// `auth_username=`/`auth_password=` pair, which resolveRequestOptions throws on) is mapped to the

bin/copyDb.ts

@@ -1,3 +1,4 @@
+import { loadNativePackage } from '../utility/packageUtils.js';
 import {
 	getDatabases,
 	getDefaultCompression,
@@ -7,7 +8,8 @@ import {
 } from '../resources/databases.ts';
 import { asBinary } from '../resources/asBinary.ts';
 import { isAbsolute, join, relative, sep } from 'node:path';
-import { move, remove } from 'fs-extra';
+import _typestrip_fs_extra from 'fs-extra';
+const { move, remove } = _typestrip_fs_extra;
 import { existsSync, mkdirSync } from 'node:fs';
 import { rename, writeFile } from 'node:fs/promises';
 import { get } from '../utility/environment/environmentManager.ts';
@@ -344,7 +346,9 @@ async function copyDbEnvironment(
 	// this contains the list of all the dbis
 	const sourceDbisDb = rootStore.dbisDb;
 	const sourceAuditStore = rootStore.auditStore;
-	const targetEnv = (require('lmdb') as typeof import('lmdb')).open(new OpenEnvironmentObject(targetDatabasePath));
+	const targetEnv = (loadNativePackage('lmdb') as typeof import('lmdb')).open(
+		new OpenEnvironmentObject(targetDatabasePath)
+	);
 	const targetDbisDb = targetEnv.openDB({ name: INTERNAL_DBIS_NAME });
 	let written;
 	let outstandingWrites = 0;