diff --git a/.github/workflows/review-coverage.yml b/.github/workflows/review-coverage.yml index 66cb77891d..04df45c3fa 100644 --- a/.github/workflows/review-coverage.yml +++ b/.github/workflows/review-coverage.yml @@ -122,6 +122,7 @@ jobs: dataLayer/backupRepository.ts dataLayer/harperBridge/ResourceBridge.ts dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js + dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts dataLayer/restoreMarker.ts dataLayer/rocksdbBackup.ts dataLayer/schemaDescribe.ts diff --git a/DESIGN.md b/DESIGN.md index c065563fda..73577c4740 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -121,6 +121,7 @@ Index of the design notes for the harper core: one line per note, grouped by the - [Three HTTP stacks coexist — know which one](server/DESIGN.md#three-http-stacks-coexist--know-which-one) — The native `http.ts` stack, the Fastify operations API and the legacy Fastify custom-functions server, and which one a request lands on. - [File overview](server/DESIGN.md#file-overview) — What each file in `server/` owns. +- [Native and compiled runtime startup](server/DESIGN.md#native-and-compiled-runtime-startup) — Runtime paths stay canonical; cyclic worker registration remains synchronous, and only an HTTP worker entry starts listeners. - [`http.ts` — symbol map](server/DESIGN.md#httpts--symbol-map) — Symbol index for the native HTTP entry file. - [Operations authorization boundary](server/DESIGN.md#operations-authorization-boundary) — Operations request bodies are untrusted: trusted dispatch state rides async context, and a worker-registered fact reaches main only through `OPERATION_REGISTERED`. - [Resource ↔ HTTP boundary](server/DESIGN.md#resource--http-boundary) — How a request becomes a Resource call and a result becomes a response. @@ -136,13 +137,13 @@ Index of the design notes for the harper core: one line per note, grouped by the - [`universalHeaders` (`http.securityHeaders`): ownership, precedence, and per-thread scope](server/DESIGN.md#universalheaders-httpsecurityheaders-ownership-precedence-and-per-thread-scope) — `applySecurityHeaders` splices only the tuples it owns; only the root `http` scope owns them; a header the app or route set wins; the array is per thread. - [Under Bun, the main HTTP port is served by `node:http`, not `Bun.serve`](server/DESIGN.md#under-bun-the-main-http-port-is-served-by-nodehttp-not-bunserve) — Under Bun the default port is a `node:http` server because WebSockets need it; no `Bun.serve` config is registered for it. - [Per-worker UDS mirrors are separate server instances — port-keyed wiring does not reach them (`server/http.ts`)](server/DESIGN.md#per-worker-uds-mirrors-are-separate-server-instances--port-keyed-wiring-does-not-reach-them-serverhttpts) — Per-worker UDS mirrors are separate `http.Server` instances; port-keyed wiring must be propagated through `server.udsMirror`. -- [A per-thread UDS mirror is bound at a temp name and renamed over its published path (`server/threads/threadServer.js`)](server/DESIGN.md#a-per-thread-uds-mirror-is-bound-at-a-temp-name-and-renamed-over-its-published-path-serverthreadsthreadserverjs) — libuv unlinks a pipe server's path on close regardless of owner; rename keeps the listening inode under the published name so an overlapping restart's outgoing worker cannot delete the replacement's mirror. -- [A worker that misses an ITC ack gets its OS thread state logged (`server/threads/manageThreads.js`)](server/DESIGN.md#a-worker-that-misses-an-itc-ack-gets-its-os-thread-state-logged-serverthreadsmanagethreadsjs) — A worker that misses a broadcast ack gets its OS thread state logged. -- [A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.js`)](server/DESIGN.md#a-workers-parentport-close-is-not-the-main-threads-exit-serverthreadsmanagethreadsjs) — A worker's `parentPort` closing is never announced as the main thread's exit. +- [A per-thread UDS mirror is bound at a temp name and renamed over its published path (`server/threads/threadServer.ts`)](server/DESIGN.md#a-per-thread-uds-mirror-is-bound-at-a-temp-name-and-renamed-over-its-published-path-serverthreadsthreadserverts) — libuv unlinks a pipe server's path on close regardless of owner; rename keeps the listening inode under the published name so an overlapping restart's outgoing worker cannot delete the replacement's mirror. +- [A worker that misses an ITC ack gets its OS thread state logged (`server/threads/manageThreads.ts`)](server/DESIGN.md#a-worker-that-misses-an-itc-ack-gets-its-os-thread-state-logged-serverthreadsmanagethreadsts) — A worker that misses a broadcast ack gets its OS thread state logged. +- [A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.ts`)](server/DESIGN.md#a-workers-parentport-close-is-not-the-main-threads-exit-serverthreadsmanagethreadsts) — A worker's `parentPort` closing is never announced as the main thread's exit. - [`chooseOperation` authorizes the invoked operation against the authenticated principal (`server/serverHelpers/serverUtilities.ts`)](server/DESIGN.md#chooseoperation-authorizes-the-invoked-operation-against-the-authenticated-principal-serverserverhelpersserverutilitiests) — The principal comes only from top-level `hdb_user`; `search_operation` is the subject only for export ops; `parsed_sql_object` is never client input. - [`withNodeAdapter()`'s response is the body `PassThrough` it resolves with (`server/serverHelpers/NodeAdapterResponse.ts`)](server/DESIGN.md#withnodeadapters-response-is-the-body-passthrough-it-resolves-with-serverserverhelpersnodeadapterresponsets) — The adapter response is the `PassThrough` it resolves with; headers commit once through `writeHead`; Express is not a target. - [Dedicated worker pools own their listeners and load no applications (`server/threads/workerPools.ts`)](server/DESIGN.md#dedicated-worker-pools-own-their-listeners-and-load-no-applications-serverthreadsworkerpoolsts) — A `replication` pool loads only trusted built-ins, binds only the listeners claimed with `threadType`, stays active while it restarts, and restarts only on an operator restart. -- [`manageThreads` has two different `workerCount`s (`server/threads/manageThreads.js`)](server/DESIGN.md#managethreads-has-two-different-workercounts-serverthreadsmanagethreadsjs) — The module-global `workerCount` sizes only the rolling-restart throttle; `getWorkerCount()` reads `workerData.workerCount`, frozen at spawn. +- [`manageThreads` has two different `workerCount`s (`server/threads/manageThreads.ts`)](server/DESIGN.md#managethreads-has-two-different-workercounts-serverthreadsmanagethreadsts) — The module-global `workerCount` sizes only the rolling-restart throttle; `getWorkerCount()` reads `workerData.workerCount`, frozen at spawn. - [A WebSocket close reason must be bounded to 123 bytes (`server/serverHelpers/webSocketCloseReason.ts`)](server/DESIGN.md#a-websocket-close-reason-must-be-bounded-to-123-bytes-serverserverhelperswebsocketclosereasonts) — `ws` throws past 123 bytes from a rejection handler; every dynamic reason goes through `toCloseReason()`, and all three terminal handlers render the error code alike. - [`serverErrorHandler` skips what `handlePostRequest` already logged (`server/serverHelpers/serverHandlers.js`)](server/DESIGN.md#servererrorhandler-skips-what-handlepostrequest-already-logged-serverserverhelpersserverhandlersjs) — One Error line per failure raised before an operation runs; an error thrown while it runs is also logged by `OperationFunctionCaller`. - [A request-queue shed is a 503 the server never logged (`server/throttle.ts`)](server/DESIGN.md#a-request-queue-shed-is-a-503-the-server-never-logged-serverthrottlets) — the HTTP request-queue throttle answers a literal 503 body without throwing, so no error-path log line exists; its per-instance warn names the queue. @@ -204,6 +205,8 @@ Index of the design notes for the harper core: one line per note, grouped by the ## utility/ +- [Core runtime and product paths (`packageUtils.js`)](utility/DESIGN.md#core-runtime-and-product-paths-packageutilsjs) — Core files and schema assets follow the module tree; metadata, product startup and public relative-worker paths retain their package owner. + - [Every path handed to a native file watch must be canonicalized (`utility/watchPath.ts`)](utility/DESIGN.md#every-path-handed-to-a-native-file-watch-must-be-canonicalized-utilitywatchpathts) — Every native file-watch path is canonicalized first: libuv aborts the process on a Windows 8.3 short-path mismatch. - [Interactive CLI prompts go through `utility/interactivePrompts.ts`](utility/DESIGN.md#interactive-cli-prompts-go-through-utilityinteractivepromptsts) — Every `@inquirer` prompt uses this seam, which lazy-loads packages off the boot path, exits 130 on Ctrl-C and gives tests a stubbable raw layer. - [An HdbError's `message` is a string; the structured body is `http_resp_msg` (`utility/errors/hdbError.ts`)](utility/DESIGN.md#an-hdberrors-message-is-a-string-the-structured-body-is-http_resp_msg-utilityerrorshdberrorts) — A report object passed to `handleHDBError` is the response body and the job message; `message` is a string derived from it. diff --git a/agent/agent.ts b/agent/agent.ts index 04ea8d40f7..8a1c619e1a 100644 --- a/agent/agent.ts +++ b/agent/agent.ts @@ -21,7 +21,7 @@ import * as env from '../utility/environment/environmentManager.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { Models } from '../resources/models/Models.ts'; import type { AuthedUser } from '../components/mcp/toolRegistry.ts'; -import { workers } from '../server/threads/manageThreads.js'; +import { workers } from '../server/threads/manageThreads.ts'; import { composeToolset } from './toolset.ts'; import { buildInspectorTools } from './tools/inspectorTool.ts'; import { buildHttpFetchTool, resolveHttpFetchConfig } from './tools/httpFetchTool.ts'; diff --git a/bin/agentCli.ts b/bin/agentCli.ts index 9bfbe8c2cd..e10b3de3ee 100644 --- a/bin/agentCli.ts +++ b/bin/agentCli.ts @@ -18,7 +18,7 @@ import * as readline from 'node:readline'; import { loadCredentials, normalizeTarget, extractTargetCredentials } from './cliCredentials.ts'; import { refreshExpiredOperationToken } from './cliOperations.ts'; import { httpRequest } from '../utility/common_utils.ts'; -import { getHdbPid } from '../utility/processManagement/processManagement.js'; +import { getHdbPid } from '../utility/processManagement/processManagement.ts'; import { initConfig, getConfigPath } from '../config/configUtils.ts'; import * as terms from '../utility/hdbTerms.ts'; diff --git a/bin/backup.ts b/bin/backup.ts index 0dd8eb24fa..82a37a704e 100644 --- a/bin/backup.ts +++ b/bin/backup.ts @@ -1,4 +1,5 @@ -'use strict'; +import dotenv from 'dotenv'; +('use strict'); import { createWriteStream } from 'node:fs'; import { rename, unlink } from 'node:fs/promises'; @@ -12,7 +13,7 @@ import { buildRequest, cliOperations, resolveRequestOptions } from './cliOperati import { loadCredentials } from './cliCredentials.ts'; import { httpRequest } from '../utility/common_utils.ts'; import { initConfig } from '../config/configUtils.ts'; -import { getHdbPid } from '../utility/processManagement/processManagement.js'; +import { getHdbPid } from '../utility/processManagement/processManagement.ts'; import { createBackupOffline, deleteBackupOffline, @@ -37,7 +38,7 @@ export async function runBackupCommand(command: string): Promise { // the local-vs-remote routing off HARPER_CLI_TARGET/CLI_TARGET, and without this a `.env`- // configured remote target would be invisible, silently running a destructive op locally // (cliOperations does the same on its first line). - require('dotenv').config(); + dotenv.config(); const request = buildRequest(); const databaseName = request.database || 'data'; diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 7a089895e6..8367822e50 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -3,11 +3,11 @@ import { loadCredentials, saveCredentials, normalizeTarget, extractTargetCredentials } from './cliCredentials.ts'; import { isJWTExpired } from '../security/tokenAuthentication.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; -envMgr.initSync(); import * as terms from '../utility/hdbTerms.ts'; import { httpRequest } from '../utility/common_utils.ts'; import { workloadIdentityAvailable, exchangeWorkloadIdentityForToken } from './workloadIdentity.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; +import dotenv from 'dotenv'; import * as YAML from 'yaml'; import { Readable } from 'node:stream'; import { execFileSync } from 'node:child_process'; @@ -16,13 +16,18 @@ import { encode as encodeCbor } from 'cbor-x'; import { buildMultipartBody } from './multipartBuilder.ts'; import { parseSSE } from './sseConsumer.ts'; import { DeployRenderer } from './deployRenderer.ts'; -import { getHdbPid } from '../utility/processManagement/processManagement.js'; +import { getHdbPid } from '../utility/processManagement/processManagement.ts'; import { initConfig, getConfigPath } from '../config/configUtils.ts'; // The `deploy setup` seal and this by-reference flag both name the same hdb_secret row, and the // server re-derives it from its own request — so the derivation lives in one dependency-free module // rather than being restated per caller (it also keeps `components/` off the CLI's import graph). import { deriveGitSecretName, directoryProjectName, normalizeGitHost } from '../utility/componentNames.ts'; +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' }; // Shown for any local-instance connection failure (missing pid, missing/stale domain @@ -881,7 +886,7 @@ export async function resolveRequestOptions(req: any): Promise<{ options: any; t } async function cliOperations(req: any, skipResponseLog = false) { - require('dotenv').config(); + dotenv.config(); // Resolve target/auth inside the try so a credential or connection error (e.g. an incomplete // `auth_username=`/`auth_password=` pair, which resolveRequestOptions throws on) is mapped to the diff --git a/bin/copyDb.ts b/bin/copyDb.ts index 5c0f8e2584..ffbc8ce8e1 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -1,3 +1,4 @@ +import { loadNativePackage } from '../utility/packageUtils.js'; import { getDatabases, getDefaultCompression, @@ -7,7 +8,8 @@ import { } from '../resources/databases.ts'; import { asBinary } from '../resources/asBinary.ts'; import { isAbsolute, join, relative, sep } from 'node:path'; -import { move, remove } from 'fs-extra'; +import _typestrip_fs_extra from 'fs-extra'; +const { move, remove } = _typestrip_fs_extra; import { existsSync, mkdirSync } from 'node:fs'; import { rename, writeFile } from 'node:fs/promises'; import { get } from '../utility/environment/environmentManager.ts'; @@ -344,7 +346,9 @@ async function copyDbEnvironment( // this contains the list of all the dbis const sourceDbisDb = rootStore.dbisDb; const sourceAuditStore = rootStore.auditStore; - const targetEnv = (require('lmdb') as typeof import('lmdb')).open(new OpenEnvironmentObject(targetDatabasePath)); + const targetEnv = (loadNativePackage('lmdb') as typeof import('lmdb')).open( + new OpenEnvironmentObject(targetDatabasePath) + ); const targetDbisDb = targetEnv.openDB({ name: INTERNAL_DBIS_NAME }); let written; let outstandingWrites = 0; diff --git a/bin/harper.ts b/bin/harper.ts index f504b466a9..c4c7de8193 100644 --- a/bin/harper.ts +++ b/bin/harper.ts @@ -4,9 +4,8 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import logger from '../utility/logging/harper_logger.ts'; -import * as cliOperations from './cliOperations.ts'; import { help } from './help.ts'; -import { packageJson } from '../utility/packageUtils.js'; +import { packageJson, RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT } from '../utility/packageUtils.js'; import checkNode from '../launchServiceScripts/utility/checkNodeVersion.js'; import * as hdbTerms from '../utility/hdbTerms.ts'; const { SERVICE_ACTIONS_ENUM, OPERATIONS_ENUM } = hdbTerms as any; @@ -36,6 +35,25 @@ export function wantsTopLevelHelp(argv: readonly string[], service: string | und return !delegatesHelp && (argv.includes('-h') || argv.includes('--help')); } +// Load configuration before a subcommand's module-level config reads. +async function initEnv() { + const env = await import('../utility/environment/environmentManager.ts'); + env.initSync(); +} + +/** + * Resolve a default-exported function from a dynamically-imported module that + * may have been loaded from a .ts source (ESM: `mod.default` is the function) + * or a tsc-compiled .js (CJS-wrapped: `mod.default` is the CJS exports object + * and `mod.default.default` is the function). + */ +function getDefaultExport(mod: any): any { + if (typeof mod === 'function') return mod; + if (typeof mod.default === 'function') return mod.default; + if (typeof mod.default?.default === 'function') return mod.default.default; + return mod.default ?? mod; +} + async function harper() { let nodeResults = checkNode(); @@ -63,60 +81,75 @@ async function harper() { switch (service) { case SERVICE_ACTIONS_ENUM.HELP: return help(); - case SERVICE_ACTIONS_ENUM.START: - return require('./run').launch(); - case SERVICE_ACTIONS_ENUM.INSTALL: - return (require('./install').default || require('./install'))(); - case SERVICE_ACTIONS_ENUM.STOP: - return (require('./stop').default || require('./stop'))().then(() => { + case SERVICE_ACTIONS_ENUM.START: { + await initEnv(); + const mod = await import('./run.ts'); + // runStartup() is drained inside run.ts main() after initialize() + // completes, so server.X singletons and auth's table() resolution + // see a populated env / installed hdbBasePath. + return mod.launch(); + } + case SERVICE_ACTIONS_ENUM.INSTALL: { + return getDefaultExport(await import('./install.ts'))(); + } + case SERVICE_ACTIONS_ENUM.STOP: { + await initEnv(); + return getDefaultExport(await import('./stop.ts'))().then(() => { process.exit(0); }); + } case SERVICE_ACTIONS_ENUM.RESTART: - return require('./restart').restart({}); + await initEnv(); + return (await import('./restart.ts')).restart({}); case SERVICE_ACTIONS_ENUM.VERSION: return packageJson.version; case SERVICE_ACTIONS_ENUM.UPGRADE: + await initEnv(); logger.setLogLevel(hdbTerms.LOG_LEVELS.INFO); // The require is here to better control the flow of imports when this module is called. - return require('./upgrade.js') - .upgrade(null) - .then(() => 'Your instance of Harper is up to date!'); - case SERVICE_ACTIONS_ENUM.STATUS: - return (require('./status').default || require('./status'))(); + return (await import('./upgrade.ts')).upgrade(null).then(() => 'Your instance of Harper is up to date!'); + case SERVICE_ACTIONS_ENUM.STATUS: { + await initEnv(); + return getDefaultExport(await import('./status.ts'))(); + } case SERVICE_ACTIONS_ENUM.LOGIN: { const args = process.argv.slice(3); const forCi = args.includes('--for-ci'); // Flags are filtered out so they can appear anywhere without being mistaken for the // positional target/username. const [target, username] = args.filter((arg) => !arg.startsWith('-')); - const { login } = require('./login'); + const { login } = await import('./login.ts'); return login(target, username, { forCi }); } case SERVICE_ACTIONS_ENUM.LOGOUT: { const target = process.argv[3]; - const { logout } = require('./logout'); + const { logout } = await import('./logout.ts'); return logout(target); } case SERVICE_ACTIONS_ENUM.MCP: { - const { runMcpCli } = require('./mcp'); + await initEnv(); + const { runMcpCli } = await import('./mcp/index.ts'); const code = await runMcpCli(process.argv.slice(3)); process.exit(code); } case SERVICE_ACTIONS_ENUM.CHAT: case SERVICE_ACTIONS_ENUM.AGENT: { - const { runAgentCli } = require('./agentCli'); + await initEnv(); + const { runAgentCli } = await import('./agentCli.ts'); const code = await runAgentCli(process.argv.slice(3)); process.exit(code); } // eslint-disable-next-line no-fallthrough case SERVICE_ACTIONS_ENUM.RENEWCERTS: - return require('../security/keys') + await initEnv(); + return (await import('../security/keys.ts')) .renewSelfSigned() .then(() => 'Successfully renewed self-signed certificates'); case SERVICE_ACTIONS_ENUM.COPYDB: { + await initEnv(); let sourceDb = process.argv[3]; let targetDbPath = process.argv[4]; - return require('./copyDb').copyDb(sourceDb, targetDbPath, { blobs: 'copy' }); + return (await import('./copyDb.ts')).copyDb(sourceDb, targetDbPath, { blobs: 'copy' }); } case OPERATIONS_ENUM.CREATE_BACKUP: case OPERATIONS_ENUM.LIST_BACKUPS: @@ -125,7 +158,8 @@ async function harper() { case OPERATIONS_ENUM.PURGE_BACKUPS: case OPERATIONS_ENUM.GET_BACKUP: case OPERATIONS_ENUM.RESTORE_BACKUP: - return require('./backup').runBackupCommand(service); + await initEnv(); + return (await import('./backup.ts')).runBackupCommand(service); case SERVICE_ACTIONS_ENUM.DEV: process.env.DEV_MODE = 'true'; // fall through @@ -163,14 +197,21 @@ async function harper() { } } // fall through - case undefined: // run harperdb in the foreground in standard mode - return require('./run').main(); - default: + case undefined: { + // run harperdb in the foreground in standard mode. + // runStartup() is drained inside run.ts main() after initialize(). + await initEnv(); + const mod = await import('./run.ts'); + return mod.main(); + } + default: { + await initEnv(); + const cliOperations = await import('./cliOperations.ts'); const cliApiOp = cliOperations.buildRequest(); // `harper deploy setup=true` provisions an encrypted deploy credential (client-side sealed // token) rather than deploying — an interactive flow, not a single operation call. if (cliApiOp.operation === 'deploy_component' && cliApiOp.setup) { - const { deploySetup } = require('./deploySetup'); + const { deploySetup } = await import('./deploySetup.ts'); await deploySetup(cliApiOp); return; } @@ -190,10 +231,16 @@ async function harper() { logger.trace('calling cli operations with:', cliOperations.redactCredentials(cliApiOp)); await cliOperations.cliOperations(cliApiOp); return; + } } } export { harper }; -if (require.main === module) { +const entryPath = + process.argv[1] && + [process.argv[1], process.argv[1] + RUNTIME_FILE_EXT].find((candidate) => fs.existsSync(candidate)); +const isEntry = + entryPath && fs.realpathSync(entryPath) === path.join(RUNTIME_SRC_ROOT, `bin/harper${RUNTIME_FILE_EXT}`); +if (isEntry) { harper() .then((message) => { if (message) { diff --git a/bin/mcp/client.ts b/bin/mcp/client.ts index 204ce6e3fe..42244f8f33 100644 --- a/bin/mcp/client.ts +++ b/bin/mcp/client.ts @@ -1,3 +1,5 @@ +import * as configUtilsRuntimeModule from '../../config/configUtils.ts'; +import * as hdbTermsRuntimeModule from '../../utility/hdbTerms.ts'; /** * Streamable HTTP MCP client over stdio. Implements the MCP rev 2025-06-18 * transport from the client side: each line on stdin is a JSON-RPC frame, @@ -364,8 +366,8 @@ export function resolveConnection(opts: McpCliOptions): HttpOptions { ); } // UDS path resolution mirrors bin/cliOperations.ts:150. - const { getConfigPath } = require('../../config/configUtils'); - const terms = require('../../utility/hdbTerms'); + const { getConfigPath } = configUtilsRuntimeModule; + const terms = hdbTermsRuntimeModule; const socketPath = getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET); return { protocol: 'http:', diff --git a/bin/restart.ts b/bin/restart.ts index ea12b8d5a8..256ef88108 100644 --- a/bin/restart.ts +++ b/bin/restart.ts @@ -1,10 +1,10 @@ 'use strict'; import minimist from 'minimist'; -import { isMainThread, parentPort } from 'worker_threads'; +import { isMainThread, parentPort } from 'node:worker_threads'; import * as hdbTerms from '../utility/hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; -import * as processMan from '../utility/processManagement/processManagement.js'; +import * as processMan from '../utility/processManagement/processManagement.ts'; import { compactOnStart } from './copyDb.ts'; import { beginProcessShutdown, @@ -14,7 +14,7 @@ import { getRunningIsolatedApplications, onMessageByType, shutdownWorkersNow, -} from '../server/threads/manageThreads.js'; +} from '../server/threads/manageThreads.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; import * as envMgr from '../utility/environment/environmentManager.ts'; @@ -25,8 +25,12 @@ import { awaitRestart } from '../components/awaitRestart.ts'; import { rmSync } from 'node:fs'; import { getThisNodeName } from '../server/nodeName.ts'; import { armRestartExitWatchdog } from './restartExitWatchdog.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const RESTART_RESPONSE = `Restarting Harper. This may take up to ${hdbTerms.RESTART_TIMEOUT_MS / 1000} seconds.`; const INVALID_SERVICE_ERR = 'Invalid service'; const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000; @@ -125,7 +129,7 @@ async function restart(req: any) { if (calledFromCli) { const hdbPid = processMan.getHdbPid(); console.error(hdbPid ? 'Restarting Harper...' : 'Starting Harper...'); - require('./run').launch(true); + (await import('./run.ts')).launch(true); return RESTART_RESPONSE; } @@ -157,7 +161,7 @@ async function restart(req: any) { // and shut down. hdbLogger.debug('Shutdown workers'); await shutdownWorkersNow(); - const { closeServers } = require('../server/threads/threadServer.js'); + const { closeServers } = await import('../server/threads/threadServer.ts'); await closeServers(); await processMan.cleanupChildrenProcesses(false); // remove pid file so it doesn't trip up the launch @@ -170,7 +174,7 @@ async function restart(req: any) { process.exit(0); } // now launch the new process and exit this process - await require('./run').launch(true); + await (await import('./run.ts')).launch(true); } catch (error) { hdbLogger.fatal('Restart teardown failed; exiting Harper', error); process.exit(1); @@ -238,7 +242,7 @@ async function activateDeploymentOnPeers(activation: any) { async function activateEachPeer(peers: any[], { project, deploymentId, deploymentRow }, results: any[]) { const replication = (global as any).server.replication; replication.monitorNodeCAs(); - const { peerDeployAnswerTimeoutMs } = await import('../components/operations.js'); + const { peerDeployAnswerTimeoutMs } = await import('../components/operations.ts'); const timeoutMs = peerDeployAnswerTimeoutMs({ restart: true }); for (const node of peers) { try { diff --git a/bin/run.ts b/bin/run.ts index cb155ad8d6..5e7ddadcfb 100644 --- a/bin/run.ts +++ b/bin/run.ts @@ -1,16 +1,19 @@ 'use strict'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); - +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} // This unused restart import is here so that main thread loads ITC event listener defined in restart file. Do not remove. import './restart.ts'; import * as terms from '../utility/hdbTerms.ts'; const { CONFIG_PARAMS } = terms; import hdbLogger from '../utility/logging/harper_logger.ts'; -import * as fs from 'fs-extra'; -import * as path from 'path'; -import checkJwtTokens from '../utility/install/checkJWTTokensExist.js'; +import fs from 'fs-extra'; +import * as path from 'node:path'; +import checkJwtTokens from '../utility/install/checkJWTTokensExist.ts'; import { install } from '../utility/install/installer.ts'; import chalk from 'chalk'; import { packageJson } from '../utility/packageUtils.js'; @@ -18,19 +21,19 @@ import * as hdbUtils from '../utility/common_utils.ts'; import * as installation from '../utility/installation.ts'; import * as configUtils from '../config/configUtils.ts'; import assignCMDENVVariables from '../utility/assignCmdEnvVariables.ts'; -import * as upgrade from './upgrade.js'; +import * as upgrade from './upgrade.ts'; import { compactOnStart, migrateOnStart } from './copyDb.ts'; import minimist from 'minimist'; import * as keys from '../security/keys.ts'; import { startHTTPThreads } from '../server/threads/socketRouter.ts'; -import { beginProcessShutdown } from '../server/threads/manageThreads.js'; +import { beginProcessShutdown } from '../server/threads/manageThreads.ts'; import * as hdbInfoController from '../dataLayer/hdbInfoController.ts'; import { isReadOnlyMode } from '../resources/databases.ts'; import { declareTokenUseTable } from '../security/authn/oidc/tokenUseTable.ts'; import { ensureCertificateVerificationTables } from '../security/certificateVerification/verificationTables.ts'; import { getThisNodeName, getThisNodeHostname } from '../server/nodeName.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; -import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.js'; +import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.ts'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; let pmUtils; @@ -117,7 +120,7 @@ async function initialize(calledByInstall = false, calledByMain = false) { // If HARPER_SET_CONFIG is present, filter out any config keys that are set in it // to prevent individual env vars from overriding explicit runtime configuration - const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars'); + const { filterArgsAgainstRuntimeConfig } = await import('../config/harperConfigEnvVars.ts'); parsedArgs = filterArgsAgainstRuntimeConfig(parsedArgs); if (!hdbUtils.isEmpty(parsedArgs) && !hdbUtils.isEmptyOrZeroLength(Object.keys(parsedArgs))) { @@ -236,6 +239,12 @@ async function main(calledByInstall = false) { if (process.env.HARPER_SAFE_MODE) hdbLogger.notify(SAFE_MODE_MSG); + // Drain onStartup hooks after install/initialize so server.X singletons + // (server.http, server.getUser, etc.) and auth.ts's onStartup table() + // call see the populated env / installed hdbBasePath. + const lifecycle = await import('../utility/lifecycle.ts'); + await lifecycle.runStartup(); + if (env.get(terms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart(); if (env.get(terms.CONFIG_PARAMS.STORAGE_MIGRATEONSTART)) await migrateOnStart(); @@ -270,7 +279,7 @@ function started() { async function launch(exit = true) { skipExitListeners = !exit; try { - if (pmUtils === undefined) pmUtils = require('../utility/processManagement/processManagement.js'); + if (pmUtils === undefined) pmUtils = await import('../utility/processManagement/processManagement.ts'); hdbLogger.debug('initializing processManagement...'); await initialize(); hdbLogger.debug('Starting new main process'); diff --git a/bin/status.ts b/bin/status.ts index 2b1eb849fe..5258eea29f 100644 --- a/bin/status.ts +++ b/bin/status.ts @@ -1,7 +1,7 @@ 'use strict'; -import * as fs from 'fs-extra'; -import * as path from 'path'; +import fs from 'fs-extra'; +import * as path from 'node:path'; import * as YAML from 'yaml'; import * as hdbTerms from '../utility/hdbTerms.ts'; @@ -10,8 +10,12 @@ import * as systemInformation from '../utility/environment/systemInformation.ts' import * as envMgr from '../utility/environment/environmentManager.ts'; import * as installation from '../utility/installation.ts'; import { prettyDuration } from '../utility/common_utils.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const STATUSES = { RUNNING: 'running', STOPPED: 'stopped', diff --git a/bin/upgrade.js b/bin/upgrade.ts similarity index 86% rename from bin/upgrade.js rename to bin/upgrade.ts index c20bc8c103..6282348220 100644 --- a/bin/upgrade.js +++ b/bin/upgrade.ts @@ -1,31 +1,27 @@ -'use strict'; - /** * The upgrade module is used to facilitate the upgrade process for existing instances of HDB that pull down a new version * of HDB from NPM that requires a specific upgrade script be run - e.g. there are changes required for the settings.js * config file, a data model change requires a re-indexing script is run, etc. */ -const env = require('../utility/environment/environmentManager.ts'); -env.initSync(); - -const chalk = require('chalk'); -const hdbLogger = require('../utility/logging/harper_logger.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const directivesManager = require('../upgrade/directivesManager.ts'); -const installation = require('../utility/installation.ts'); -const hdbInfoController = require('../dataLayer/hdbInfoController.ts'); -const globalSchema = require('../utility/globalSchema.ts'); -const { packageJson } = require('../utility/packageUtils.js'); -const promisify = require('util').promisify; +import * as env from '../utility/environment/environmentManager.ts'; + +import chalk from 'chalk'; +import _hdbLogger from '../utility/logging/harper_logger.ts'; +const hdbLogger = _hdbLogger; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as directivesManager from '../upgrade/directivesManager.ts'; +import * as installation from '../utility/installation.ts'; +import * as hdbInfoController from '../dataLayer/hdbInfoController.ts'; +import * as globalSchema from '../utility/globalSchema.ts'; +import { packageJson } from '../utility/packageUtils.js'; +import { promisify } from 'node:util'; const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal); let pm2Utils; const { UPGRADE_VERSION } = hdbTerms.UPGRADE_JSON_FIELD_NAMES_ENUM; -module.exports = { - upgrade, -}; +export { upgrade }; /** * Runs the upgrade directives, if needed, for an updated version of Harper. @@ -38,7 +34,7 @@ async function upgrade(upgradeObj) { // Requiring the processManagement mod will create the .pm2 dir. This code is here to allow install to set // pm2 env vars before that is done. - if (pm2Utils === undefined) pm2Utils = require('../utility/processManagement/processManagement.js'); + if (pm2Utils === undefined) pm2Utils = await import('../utility/processManagement/processManagement.ts'); //We have to make sure HDB is installed before doing anything else const installed = installation.isHdbInstalled(env, hdbLogger); diff --git a/components/Application.ts b/components/Application.ts index 2071a48000..8854bc1e53 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1,3 +1,4 @@ +import { RUNTIME_SRC_ROOT } from '../utility/packageUtils.js'; import { type Logger } from '../utility/logging/logger.ts'; import { getConfigObj, @@ -26,7 +27,7 @@ import { processIncarnation, registerProcessGroup, unregisterProcessGroup, -} from '../server/threads/manageThreads.js'; +} from '../server/threads/manageThreads.ts'; import { certificationPinsOf, liveCertification, @@ -698,7 +699,7 @@ type ExtractionContext = Pick; // The credential helper git executes for a private git-reference deploy. It ships alongside this // module (both in source and in dist), holds no secret, and is inert without a live session. -export const GIT_CREDENTIAL_HELPER_PATH = join(__dirname, 'gitCredentialHelper.js'); +export const GIT_CREDENTIAL_HELPER_PATH = join(RUNTIME_SRC_ROOT, 'components/gitCredentialHelper.js'); type InstalledPackageMetadata = { files: Map; diff --git a/components/Component.ts b/components/Component.ts index 7f04e57616..8ea23daf27 100644 --- a/components/Component.ts +++ b/components/Component.ts @@ -1,8 +1,8 @@ -import { resolveBaseURLPath } from './resolveBaseURLPath'; -import { deriveCommonPatternBase } from './deriveCommonPatternBase'; -import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions'; -import { scan } from 'micromatch'; - +import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; +import { deriveCommonPatternBase } from './deriveCommonPatternBase.ts'; +import { deriveGlobOptions, type FastGlobOptions, type FilesOption } from './deriveGlobOptions.ts'; +import _micromatch from 'micromatch'; +const { scan } = _micromatch; interface ComponentConfig { files: FilesOption; urlPath?: string; diff --git a/components/ComponentV1.ts b/components/ComponentV1.ts index 4804a271ef..f02bd85517 100644 --- a/components/ComponentV1.ts +++ b/components/ComponentV1.ts @@ -3,12 +3,12 @@ import fg from 'fast-glob'; import { Resources } from '../resources/Resources.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; -import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions.ts'; +import { deriveGlobOptions, type FastGlobOptions, type FilesOption } from './deriveGlobOptions.ts'; import { basename, join } from 'node:path'; import { readFile } from 'node:fs/promises'; import { deriveURLPath } from './deriveURLPath.ts'; -import { scan } from 'micromatch'; - +import _micromatch from 'micromatch'; +const { scan } = _micromatch; interface ComponentV1Config { files: string | string[] | FilesOption; /** @deprecated */ path?: string; diff --git a/components/EntryHandler.ts b/components/EntryHandler.ts index 73acb4661b..30795e81ac 100644 --- a/components/EntryHandler.ts +++ b/components/EntryHandler.ts @@ -3,13 +3,14 @@ import { loggerWithTag } from '../utility/logging/harper_logger.ts'; import { createHash } from 'node:crypto'; import type { Stats } from 'node:fs'; import { EventEmitter } from 'node:events'; -import { Component, FileAndURLPathConfig } from './Component.ts'; -import { FSWatcher, FSWatcherEventMap } from 'chokidar'; +import { Component, type FileAndURLPathConfig } from './Component.ts'; +import { FSWatcher, type FSWatcherEventMap } from 'chokidar'; import { isAbsolute, join } from 'node:path'; import { readFile } from 'node:fs/promises'; -import { FilesOption } from './deriveGlobOptions.ts'; +import { type FilesOption } from './deriveGlobOptions.ts'; import { deriveURLPath } from './deriveURLPath.ts'; -import { isMatch } from 'micromatch'; +import _typestrip_micromatch from 'micromatch'; +const { isMatch } = _typestrip_micromatch; import { DIRECTORY_POLLING_FALLBACK_OPTIONS, claimLostNativeWatchError, diff --git a/components/OptionsWatcher.ts b/components/OptionsWatcher.ts index a99e577305..31d3001a7d 100644 --- a/components/OptionsWatcher.ts +++ b/components/OptionsWatcher.ts @@ -1,9 +1,9 @@ import { type Logger } from '../utility/logging/logger.ts'; import { loggerWithTag } from '../utility/logging/harper_logger.ts'; -import { EventEmitter, once } from 'events'; +import { EventEmitter, once } from 'node:events'; import { type FSWatcher } from 'chokidar'; import { readFile } from 'node:fs/promises'; -import { isDeepStrictEqual } from 'util'; +import { isDeepStrictEqual } from 'node:util'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; import cloneDeep from 'lodash/cloneDeep.js'; import { diff --git a/components/PluginModule.ts b/components/PluginModule.ts index 6b0781018a..47d107193f 100644 --- a/components/PluginModule.ts +++ b/components/PluginModule.ts @@ -1,4 +1,4 @@ -import { Scope } from './Scope'; +import { Scope } from './Scope.ts'; export interface PluginModule { handleApplication: (scope: Scope) => void | Promise; diff --git a/components/Scope.ts b/components/Scope.ts index 6571bedabe..2091b25a73 100644 --- a/components/Scope.ts +++ b/components/Scope.ts @@ -4,11 +4,11 @@ import { EventEmitter, once } from 'node:events'; import { databaseEventsEmitter, scopedTableFactory } from '../resources/databases.ts'; import { server, type Server } from '../server/Server.ts'; import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts'; -import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts'; +import { OptionsWatcher, type OptionsWatcherEventMap } from './OptionsWatcher.ts'; import { resources, type Resources } from '../resources/Resources.ts'; import { Models, models as modelsSingleton } from '../resources/models/Models.ts'; import type { FileAndURLPathConfig } from './Component.ts'; -import { FilesOption } from './deriveGlobOptions.ts'; +import { type FilesOption } from './deriveGlobOptions.ts'; import { requestRestart } from './requestRestart.ts'; import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; import { composeMountedUrlPath, type ScopeMount } from './scopeMount.ts'; diff --git a/components/canaryRollout.ts b/components/canaryRollout.ts index 3dbda1354b..4bc6cacdf1 100644 --- a/components/canaryRollout.ts +++ b/components/canaryRollout.ts @@ -10,7 +10,7 @@ import { certificationRollout, processIncarnation, setCertificationHandler, -} from '../server/threads/manageThreads.js'; +} from '../server/threads/manageThreads.ts'; import { Application, prepareApplication, withSettledComponent, type ActivationCertification } from './Application.ts'; import { liveDeploymentId, diff --git a/components/componentLoader.ts b/components/componentLoader.ts index d5cf9f34f1..46bd05d416 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -1,4 +1,5 @@ -import { onMessageByType } from '../server/threads/manageThreads.js'; +import { onMessageByType } from '../server/threads/manageThreads.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { readdirSync, readFileSync, @@ -14,7 +15,7 @@ import { isMainThread } from 'node:worker_threads'; import { parseDocument } from 'yaml'; import * as env from '../utility/environment/environmentManager.ts'; -import { PACKAGE_ROOT } from '../utility/packageUtils.js'; +import { PACKAGE_ROOT, getRuntimeModuleURL } from '../utility/packageUtils.js'; import { CONFIG_PARAMS, HDB_ROOT_DIR_NAME, ITC_EVENT_TYPES } from '../utility/hdbTerms.ts'; import * as graphqlHandler from '../resources/graphql.ts'; import * as graphqlQueryHandler from '../server/graphqlQuerying.ts'; @@ -27,7 +28,7 @@ import * as loadEnv from '../resources/loadEnv.ts'; import harperLogger, { errorForLog } from '../utility/logging/harper_logger.ts'; import * as dataLoader from '../resources/dataLoader.ts'; import * as scheduler from '../resources/scheduler/scheduler.ts'; -import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.js'; +import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.ts'; import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts'; import { trackScopeClose } from './scopeShutdown.ts'; import { deployLifecycle } from './deployLifecycle.ts'; @@ -60,7 +61,7 @@ import { Status } from '../server/status/index.ts'; import { lifecycle as componentLifecycle, statusForComponent } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; import { materializeGlobalSecrets, processComponentEnv } from './componentSecrets.ts'; -import { PluginModule } from './PluginModule.ts'; +import { type PluginModule } from './PluginModule.ts'; import { getEnvBuiltInComponents, recoverInterruptedActivations, @@ -71,7 +72,7 @@ import { import { ComponentPreparationLockTimeoutError } from './componentPreparationLock.ts'; import { pathToFileURL } from 'node:url'; -const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); +const getComponentsRoot = () => getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); let loadedComponents = new Map(); @@ -307,7 +308,7 @@ export async function loadComponentDirectories( // startup sequencing does not protect a worker that restarts later. Safe to run on any thread: each deployment is settled under // the component preparation lock, which is cross-thread and cross-process, and the pass is idempotent. try { - for (const [component, error] of await recoverInterruptedActivations(CF_ROUTES_DIR)) { + for (const [component, error] of await recoverInterruptedActivations(getComponentsRoot())) { if (!failedRecoveries.has(component)) failedRecoveries.set(component, error); } } catch (error) { @@ -318,7 +319,7 @@ export async function loadComponentDirectories( } // Plus anything a previous pass recorded as unsettleable, which settlement above leaves in place. try { - for (const [component, error] of await unsettleableComponentsFromDisk(CF_ROUTES_DIR)) { + for (const [component, error] of await unsettleableComponentsFromDisk(getComponentsRoot())) { if (!failedRecoveries.has(component)) failedRecoveries.set(component, error); } } catch (error) { @@ -329,7 +330,7 @@ export async function loadComponentDirectories( } } try { - for (const [component, error] of await recoverInterruptedComponentExtractions(CF_ROUTES_DIR)) { + for (const [component, error] of await recoverInterruptedComponentExtractions(getComponentsRoot())) { if (!failedRecoveries.has(component)) failedRecoveries.set(component, error); } } catch (error) { @@ -352,13 +353,13 @@ export async function loadComponentDirectories( [...failedRecoveries].filter(([, error]) => !(error instanceof ComponentPreparationLockTimeoutError)) ); const deferComponentLoad = (appName: string) => { - const appFolder = join(CF_ROUTES_DIR, appName); + const appFolder = join(getComponentsRoot(), appName); const appWasVisible = existsSync(appFolder); if (appWasVisible) { componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); } const deferredLoad = serializeComponentLoad(appName, () => - recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) + recoverInterruptedComponentExtraction(getComponentsRoot(), appName) .then(async () => { if (!existsSync(appFolder)) { if (appWasVisible) { @@ -394,8 +395,8 @@ export async function loadComponentDirectories( ); trackDeferredBootLoad(appName, deferredLoad); }; - if (existsSync(CF_ROUTES_DIR)) { - const cfFolders = readdirSync(CF_ROUTES_DIR, { withFileTypes: true }); + if (existsSync(getComponentsRoot())) { + const cfFolders = readdirSync(getComponentsRoot(), { withFileTypes: true }); for (const appEntry of cfFolders) { if (!appEntry.isDirectory() && !appEntry.isSymbolicLink()) continue; // Skip hidden entries: component names are never dot-prefixed, and this keeps @@ -425,7 +426,7 @@ export async function loadComponentDirectories( continue; } if (!placedOnThisThread(appName)) continue; - const appFolder = join(CF_ROUTES_DIR, appName); + const appFolder = join(getComponentsRoot(), appName); const mountResult = tryRootConfigMount(appName); if (!mountResult.ok) continue; cfsLoaded.push( @@ -487,20 +488,15 @@ export const TRUSTED_RESOURCE_PLUGINS: any = { graphqlSchema: graphqlHandler, roles, jsResource: jsHandler, - get fastifyRoutes() { - return require('../server/fastifyRoutes'); - }, + fastifyRoutes: getRuntimeModuleURL('server/fastifyRoutes'), login, - // String entry: the loader `await import()`s these lazily when the component is actually - // processed, so the gateway's module graph is not pulled into componentLoader's own - // evaluation. `#src/*` is used rather than a relative path because it resolves under both - // conditions (source under --conditions=typestrip, dist otherwise); a relative extensionless - // require would only resolve against dist. + // String entry: the loader imports the gateway only when configured, without resolving + // core-owned files through an embedding product's package imports. // // The component is only *processed* when a `modelsGateway` key exists in the config, since // the root loop skips absent keys (`if (!componentConfig) continue`). `defaultConfig.yaml` // ships no such key, so an instance that never opts in pays nothing for this entry. - modelsGateway: '#src/resources/models/v1/index', + modelsGateway: getRuntimeModuleURL('resources/models/v1/index'), static: staticFiles, customFunctions: {}, http: httpComponent, @@ -516,29 +512,30 @@ export const TRUSTED_RESOURCE_PLUGINS: any = { login: ... */ }; -if (isMainThread) { - TRUSTED_RESOURCE_PLUGINS.operationsApi = require('../server/operationsServer'); - // Built-in agent component (#626). Only loads if the root config carries an `agent:` block; - // the block's `enabled: false` default keeps it inert even when the key is present. - TRUSTED_RESOURCE_PLUGINS.agent = require('../agent/agent'); -} else { - // The HTTP operations API itself only binds in the main thread, but worker threads still - // dispatch operations — most notably, the replication WebSocket handler in workers receives - // inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires - // `server.operation` / `server.registerOperation` to be wired up here too, and the operation - // function map to be initialized, BEFORE component plugins (replication, etc.) load and call - // `server.registerOperation?.({...})` at their module top level. Requiring serverUtilities - // directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in - // workers while still installing the dispatch machinery. - require('../server/serverHelpers/serverUtilities'); -} +onStartup(async () => { + if (isMainThread) { + const operationsApi = await import('../server/operationsServer.ts'); + if (!Object.hasOwn(TRUSTED_RESOURCE_PLUGINS, 'operationsApi')) + TRUSTED_RESOURCE_PLUGINS.operationsApi = operationsApi; + const agent = await import('../agent/agent.ts'); + if (!Object.hasOwn(TRUSTED_RESOURCE_PLUGINS, 'agent')) TRUSTED_RESOURCE_PLUGINS.agent = agent; + } else { + // The HTTP operations API itself only binds in the main thread, but worker threads still + // dispatch operations — most notably, the replication WebSocket handler in workers receives + // inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires + // `server.operation` / `server.registerOperation` to be wired up here too, and the operation + // function map to be initialized, BEFORE component plugins (replication, etc.) load and call + // `server.registerOperation?.({...})` at their module top level. Loading serverUtilities + // directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in + // workers while still installing the dispatch machinery. + await import('../server/serverHelpers/serverUtilities.ts'); + } +}); for (const { name, packageIdentifier } of getEnvBuiltInComponents()) { TRUSTED_RESOURCE_PLUGINS[name] = packageIdentifier; } -const BUILT_INS = Object.keys(TRUSTED_RESOURCE_PLUGINS); - export const loadedPaths = new Map(); // Tracks which components have already had `startOnMainThread` invoked, so it runs at most once @@ -865,6 +862,7 @@ export async function loadComponent( } else { config = DEFAULT_CONFIG; } + if (isRoot) config ??= DEFAULT_CONFIG; applicationScope.config ??= config; // Before any of the application's modules are imported: a branch has to exist by the time its @@ -895,15 +893,8 @@ export async function loadComponent( ); } - // For non-root components with empty/null config (e.g., comment-only YAML), - // don't synthesize DEFAULT_CONFIG. Empty config means the component has nothing - // to load; falling back to DEFAULT_CONFIG would cause OptionsWatcher to wait - // forever for plugins that the file doesn't actually declare. - if (isRoot) config ??= DEFAULT_CONFIG; - if (!config) { - // Empty/comment-only config file on a non-root component: nothing to load. - return undefined; - } + // Empty non-root configurations declare no plugins to load. + if (!config) return undefined; // #629 (Phase 2 of #510): populate the model-backend registry from the root // config's `models:` block before any user `handleApplication(scope)` runs, @@ -1145,7 +1136,7 @@ export async function loadComponent( // Old Extension API (`start` or `startOnMainThread`) if ( - !BUILT_INS.includes(componentName) && + !Object.hasOwn(TRUSTED_RESOURCE_PLUGINS, componentName) && ('startOnMainThread' in extensionModule || 'start' in extensionModule || 'handleFile' in extensionModule || diff --git a/components/deployLifecycle.ts b/components/deployLifecycle.ts index 85cfe98753..ab8248643f 100644 --- a/components/deployLifecycle.ts +++ b/components/deployLifecycle.ts @@ -30,7 +30,7 @@ import { isThreadRunning, onMessageByType, onThreadExit, -} from '../server/threads/manageThreads.js'; +} from '../server/threads/manageThreads.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; const DEPLOY_LIFECYCLE_MSG = 'harper:deploy:lifecycle'; diff --git a/components/mcp/listChanged.ts b/components/mcp/listChanged.ts index 6e2cb0d643..6b79489f94 100644 --- a/components/mcp/listChanged.ts +++ b/components/mcp/listChanged.ts @@ -1,3 +1,5 @@ +import * as serverHandlersRuntimeModule from '../../server/itc/serverHandlers.ts'; +import * as userRuntimeModule from '../../security/user.ts'; /** * MCP server-push notification dispatcher (#619). Subscribes to Harper's * existing role-cache-invalidation + schema-reload event channels and @@ -57,8 +59,8 @@ export function _resetListChangedForTest(): void { function loadItcHandlers(): ItcHandlers | undefined { if (_itcHandlersOverride) return _itcHandlersOverride; try { - const { schemaHandler, resourceHandler } = require('../../server/itc/serverHandlers'); - const { onUserChange } = require('../../security/user'); + const { schemaHandler, resourceHandler } = serverHandlersRuntimeModule; + const { onUserChange } = userRuntimeModule; return { schemaHandler, resourceHandler, userHandler: { addListener: onUserChange } }; } catch (err) { harperLogger.trace(`MCP listChanged: ITC handlers unavailable (${(err as Error).message})`); @@ -78,7 +80,7 @@ async function resolveUser(username: string | undefined): Promise ResourceClass.subscribe!(request, context)); stream = result && typeof (result as ResourceChangeStream)[Symbol.asyncIterator] === 'function' @@ -562,9 +561,10 @@ async function readCustomResource( // Merge onto any ambient store (usually none on the MCP HTTP path) so an // inherited transaction/cache is preserved rather than clobbered; `user` // binds last. Same idiom as processLocalTransaction's `{ ...currentStore, user }`. - // Lazy-require the server-layer machinery (see file-top note on eager init). - const { transaction, contextStorage } = require('../../resources/transaction'); - const result = await transaction({ ...contextStorage.getStore(), user }, () => def.read(params, { user, profile })); + const { transaction, contextStorage } = transactionRuntimeModule; + const result = await transaction({ ...contextStorage.getStore(), user: user as any }, () => + def.read(params, { user, profile }) + ); if (typeof result === 'string') { return { ok: true, contents: [{ uri, mimeType: def.mimeType ?? 'text/plain', text: result }] }; } @@ -937,7 +937,7 @@ function guessAppHttpUrlPrefix(): string | undefined { if (_httpUrlPrefixOverride !== undefined) return _httpUrlPrefixOverride || undefined; let hostname: string | undefined; try { - const { server } = require('../../server/Server'); + const { server } = ServerRuntimeModule; hostname = (server as { hostname?: string })?.hostname; } catch { return undefined; diff --git a/components/mcp/serverRequests.ts b/components/mcp/serverRequests.ts index 555d4c5ab0..691079dbb0 100644 Binary files a/components/mcp/serverRequests.ts and b/components/mcp/serverRequests.ts differ diff --git a/components/mcp/tools/application.ts b/components/mcp/tools/application.ts index 84646fca87..d76c1060a4 100644 --- a/components/mcp/tools/application.ts +++ b/components/mcp/tools/application.ts @@ -1,3 +1,5 @@ +import * as ResourcesCjsModule from '../../../resources/Resources.ts'; +import * as RequestTargetCjsModule from '../../../resources/RequestTarget.ts'; /** * Application-profile tool generation. Walks Harper's `Resources` registry * and registers verb tools (`get_*`, `search_*`, `create_*`, `update_*`, @@ -344,7 +346,7 @@ let _resourcesModule: { resources?: ResourcesRegistry } | undefined; function loadResources(): ResourcesRegistry | undefined { if (_resourcesOverride) return _resourcesOverride; try { - _resourcesModule ??= require('../../../resources/Resources'); + _resourcesModule ??= ResourcesCjsModule as any; return _resourcesModule!.resources as ResourcesRegistry; } catch (err) { harperLogger.trace(`MCP application tools: Resources registry unavailable (${(err as Error).message})`); @@ -355,7 +357,7 @@ function loadResources(): ResourcesRegistry | undefined { function loadRequestTarget(): RequestTargetCtor | undefined { if (_requestTargetCtorOverride) return _requestTargetCtorOverride; try { - return require('../../../resources/RequestTarget').RequestTarget as RequestTargetCtor; + return RequestTargetCjsModule.RequestTarget as unknown as RequestTargetCtor; } catch { // Tests that don't use the real RequestTarget can supply a fake. return undefined; diff --git a/components/mcp/tools/operations.ts b/components/mcp/tools/operations.ts index 5ff48c4a4f..3ef1c5a8e1 100644 --- a/components/mcp/tools/operations.ts +++ b/components/mcp/tools/operations.ts @@ -1,3 +1,4 @@ +import * as serverUtilitiesRuntimeModule from '../../../server/serverHelpers/serverUtilities.ts'; /** * Operations-profile tool generation. Exposes one MCP tool per Harper * operation that survives the `mcp.operations.allow` / `deny` filter, computed @@ -64,8 +65,7 @@ interface OperationsConfig { deny?: readonly string[]; } -// Test seams. Avoids importing Harper's heavy server-helpers graph from unit -// tests that only want to exercise the registration logic. +// Test seams replace production dispatch when exercising registry behavior. let _opMapOverride: OperationFunctionMap | undefined; let _chooseOperationOverride: ChooseOperation | undefined; let _processLocalTransactionOverride: ProcessLocalTransaction | undefined; @@ -88,11 +88,7 @@ function loadServerUtilities(): } | undefined { try { - // Lazy require: Harper's server-helpers graph initializes eagerly - // (RocksDB lock acquisition, schema preload). Loading it from a unit - // test that hasn't booted Harper throws; treat that as "we're not in - // a Harper process" and let callers gracefully no-op. - return require('../../../server/serverHelpers/serverUtilities'); + return serverUtilitiesRuntimeModule as any; } catch (err) { harperLogger.trace(`MCP operations tools: serverUtilities unavailable (${(err as Error).message})`); return undefined; diff --git a/components/operations.js b/components/operations.ts similarity index 91% rename from components/operations.js rename to components/operations.ts index 0b55c28f73..fe223fb8d8 100644 --- a/components/operations.js +++ b/components/operations.ts @@ -1,66 +1,70 @@ -'use strict'; - -const path = require('node:path'); -const { isMainThread, parentPort } = require('node:worker_threads'); -const fs = require('fs-extra'); -const fg = require('fast-glob'); -const normalize = require('normalize-path'); -const validator = require('./operationsValidation.js'); -const log = require('../utility/logging/harper_logger.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const env = require('../utility/environment/environmentManager.ts'); -const configUtils = require('../config/configUtils.ts'); -const hdbUtils = require('../utility/common_utils.ts'); -const { +import { loadRuntimeModule } from '../utility/packageUtils.js'; +import * as awaitRestartRuntimeModule from './awaitRestart.ts'; +import * as isolatedApplicationsRuntimeModule from '../server/threads/isolatedApplications.ts'; +import * as secretOperationsRuntimeModule from './secretOperations.ts'; +import * as serverUtilitiesRuntimeModule from '../server/serverHelpers/serverUtilities.ts'; +import * as requestRestartRuntimeModule from './requestRestart.ts'; +import * as componentSecretsRuntimeModule from './componentSecrets.ts'; +import * as indexRuntimeModule from './status/index.ts'; +import * as branchDatabaseRuntimeModule from '../resources/branchDatabase.ts'; +import * as path from 'node:path'; +import { isMainThread, parentPort } from 'node:worker_threads'; +import fs from 'fs-extra'; +import fg from 'fast-glob'; +import normalize from 'normalize-path'; +import * as validator from './operationsValidation.ts'; +import log from '../utility/logging/harper_logger.ts'; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as env from '../utility/environment/environmentManager.ts'; +import * as configUtils from '../config/configUtils.ts'; +import * as hdbUtils from '../utility/common_utils.ts'; +import { isEnvFile, isProtectedEnvFile, parseEnvKeys, renderMaskedEnv, upsertEnvValues, removeEnvKeys, -} = require('../utility/envFile.ts'); -const { canonicalProjectName, projectNameFromPackage } = require('../utility/componentNames.ts'); -const { handleHDBError, ClientError, ServerError, hdbErrors } = require('../utility/errors/hdbError.ts'); +} from '../utility/envFile.ts'; +import { canonicalProjectName, projectNameFromPackage } from '../utility/componentNames.ts'; +import { handleHDBError, ClientError, ServerError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const manageThreads = require('../server/threads/manageThreads.js'); -const { - packageDirectory, - scanPackageDirectory, - streamPackagedDirectory, -} = require('../components/packageComponent.ts'); -const { +import * as manageThreads from '../server/threads/manageThreads.ts'; +import { packageDirectory, scanPackageDirectory, streamPackagedDirectory } from '../components/packageComponent.ts'; +import { Application, assertNotCertifying, - prepareApplication, + prepareApplication as _prepareApplication, componentPreparationBudgetMs, ASIDE_STAGING_DIR, DEPLOY_STAGING_DIR, dropComponentDirectory, retireComponentDirectory, -} = require('./Application.ts'); -const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require('./componentPreparationLock.ts'); -const { DEPLOYMENT_PROVENANCE_FILE } = require('./deploymentProvenance.ts'); -const { compareInstallFingerprints, describeInstallDrift } = require('./installFingerprint.ts'); -const { +} from './Application.ts'; +const prepareApplication = _prepareApplication; +import { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } from './componentPreparationLock.ts'; +import { DEPLOYMENT_PROVENANCE_FILE } from './deploymentProvenance.ts'; +import { compareInstallFingerprints, describeInstallDrift } from './installFingerprint.ts'; +import { applyRootConfigEffect, assertRootConfigEffectPublishable, hasRootConfigEntry, withRootConfigPublicationLock, -} = require('./rootConfigPublication.ts'); -const { server } = require('../server/Server.ts'); -const { +} from './rootConfigPublication.ts'; +import { server } from '../server/Server.ts'; +import { DeploymentRecorder, awaitDeploymentRow, readPayloadBlobWithRetry, coerceTimeoutMs, DEFAULT_AWAIT_ROW_TIMEOUT_MS, -} = require('./deploymentRecorder.ts'); -const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts'); +} from './deploymentRecorder.ts'; +import { ProgressEmitter } from '../server/serverHelpers/progressEmitter.ts'; const DROP_COMPONENT_LOCK_TIMEOUT_MS = 5 * 60 * 1000; const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000; -function componentDropLockOptions(project) { +function componentDropLockOptions(project?: any) { return { timeoutMs: DROP_COMPONENT_LOCK_TIMEOUT_MS, onWait: (owner) => @@ -80,7 +84,7 @@ function componentDropLockOptions(project) { */ function customFunctionsStatus() { log.trace(`getting custom api status`); - let response = {}; + let response: any = {}; try { response = { @@ -93,7 +97,7 @@ function customFunctionsStatus() { new Error(), HDB_ERROR_MSGS.FUNCTION_STATUS, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -107,7 +111,7 @@ function customFunctionsStatus() { */ function getCustomFunctions() { log.trace(`getting custom api endpoints`); - let response = {}; + let response: any = {}; const dir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); try { @@ -129,7 +133,7 @@ function getCustomFunctions() { new Error(), HDB_ERROR_MSGS.GET_FUNCTIONS, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -142,7 +146,7 @@ function getCustomFunctions() { * @param {NodeObject} req * @returns {string} */ -function getCustomFunction(req) { +function getCustomFunction(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } @@ -168,7 +172,7 @@ function getCustomFunction(req) { new Error(), HDB_ERROR_MSGS.GET_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -180,7 +184,7 @@ function getCustomFunction(req) { * @param {NodeObject} req * @returns {{message:string}} */ -async function setCustomFunction(req) { +async function setCustomFunction(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } @@ -200,7 +204,7 @@ async function setCustomFunction(req) { try { fs.outputFileSync(path.join(cfDir, project, type, file + '.js'), function_content); - let response = await server.replication.replicateOperation(req); + let response: any = await server.replication.replicateOperation(req); response.message = `Successfully updated custom function: ${file}.js`; return response; } catch (err) { @@ -208,7 +212,7 @@ async function setCustomFunction(req) { new Error(), HDB_ERROR_MSGS.SET_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -220,7 +224,7 @@ async function setCustomFunction(req) { * @param {NodeObject} req * @returns {{message:string}} */ -async function dropCustomFunction(req) { +async function dropCustomFunction(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } @@ -240,7 +244,7 @@ async function dropCustomFunction(req) { try { fs.unlinkSync(path.join(cfDir, project, type, file + '.js')); - let response = await server.replication.replicateOperation(req); + let response: any = await server.replication.replicateOperation(req); response.message = `Successfully deleted custom function: ${file}.js`; return response; } catch (err) { @@ -248,7 +252,7 @@ async function dropCustomFunction(req) { new Error(), HDB_ERROR_MSGS.DROP_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -259,7 +263,7 @@ async function dropCustomFunction(req) { * @param {NodeObject} req * @returns {{message:string}} */ -async function addComponent(req) { +async function addComponent(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } @@ -276,7 +280,7 @@ async function addComponent(req) { try { await fs.mkdir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), { recursive: true }); - const application = new Application({ + const application: any = new Application({ name: project, packageIdentifier: template, install: { @@ -286,7 +290,7 @@ async function addComponent(req) { }, }); await prepareApplication(application); - let response = await server.replication.replicateOperation(req); + let response: any = await server.replication.replicateOperation(req); response.message = `Successfully added project: ${project}`; return response; } catch (err) { @@ -294,7 +298,7 @@ async function addComponent(req) { new Error(), HDB_ERROR_MSGS.ADD_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -306,7 +310,7 @@ async function addComponent(req) { * @param {NodeObject} req * @returns {string} */ -async function dropCustomFunctionProject(req) { +async function dropCustomFunctionProject(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } @@ -320,7 +324,7 @@ async function dropCustomFunctionProject(req) { const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); const { project } = req; - let apps = env.get(hdbTerms.CONFIG_PARAMS.APPS); + let apps = env.get((hdbTerms.CONFIG_PARAMS as any).APPS); if (!hdbUtils.isEmptyOrZeroLength(apps)) { let appFound = false; for (const [i, app] of apps.entries()) { @@ -332,7 +336,9 @@ async function dropCustomFunctionProject(req) { } if (appFound) { - await withRootConfigPublicationLock(async () => configUtils.updateConfigValue(hdbTerms.CONFIG_PARAMS.APPS, apps)); + await withRootConfigPublicationLock(async () => + configUtils.updateConfigValue((hdbTerms.CONFIG_PARAMS as any).APPS, apps) + ); return `Successfully deleted project: ${project}`; } @@ -349,7 +355,7 @@ async function dropCustomFunctionProject(req) { }, componentDropLockOptions(project) ); - const response = await server.replication.replicateOperation(req); + const response: any = await server.replication.replicateOperation(req); response.message = `Successfully deleted project: ${project}`; return response; } catch (err) { @@ -357,7 +363,7 @@ async function dropCustomFunctionProject(req) { new Error(), HDB_ERROR_MSGS.DROP_FUNCTION_PROJECT, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -390,7 +396,7 @@ async function dropCustomFunctionProject(req) { * @returns {Promise<{payload: string, project: string}|{project: string, total_size: number, * dangling_symlinks: string[]}|Readable>} */ -async function packageComponent(req) { +async function packageComponent(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } @@ -430,8 +436,8 @@ async function packageComponent(req) { } if (req.stream) { - const stream = streamPackagedDirectory(pathToProject, req); - const headers = new Map(); + const stream: any = streamPackagedDirectory(pathToProject, req); + const headers: any = new Map(); headers.set('content-type', 'application/gzip'); // `project` is narrowed to a bare filename by path.parse().name above and validated // against PROJECT_FILE_NAME_REGEX, so it cannot inject header content. @@ -452,8 +458,8 @@ const BRANCH_STORAGE_RETAINED = '. Any branched database storage this application owns was left in place; drop it again with restart: true to discard that data'; /** Report a restart outcome the operation's own success message cannot convey. */ -function logRestartOutcome(restart, what) { - const { RESTART_IDLE_TIMEOUT_MS, RESTART_WAIT_CEILING_MS } = require('./awaitRestart.ts'); +function logRestartOutcome(restart?: any, what?: any) { + const { RESTART_IDLE_TIMEOUT_MS, RESTART_WAIT_CEILING_MS } = awaitRestartRuntimeModule; if (restart.replacementsNotStarted) log.warn( `${restart.replacementsNotStarted} replacement worker thread(s) did not report starting after ${what}; the pool is short until they are restarted` @@ -487,8 +493,8 @@ const PEER_DEPLOY_ACTIVATION_MARGIN_MS = 10 * 60 * 1000; * preparation, or behind several, can take longer; so can one validating plugins whose configured timeouts outlast * the margin. Such a peer is reported as not answering, which says nothing of its outcome. */ -function peerDeployAnswerTimeoutMs(req) { - const { RESTART_WAIT_CEILING_MS } = require('./awaitRestart.ts'); +function peerDeployAnswerTimeoutMs(req?: any) { + const { RESTART_WAIT_CEILING_MS } = awaitRestartRuntimeModule; const payloadWaitMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); const installTimeoutMs = coerceTimeoutMs(req.install_timeout, undefined); return Math.min( @@ -508,7 +514,7 @@ function peerDeployAnswerTimeoutMs(req) { * @param req * @returns {Promise} */ -async function deployComponent(req) { +async function deployComponent(req?: any) { if (req.project) { req.project = canonicalProjectName(req.project); } else if (req.package) { @@ -524,7 +530,7 @@ async function deployComponent(req) { // replicated ciphertext (reference, not embed); already-reference entries pass through, and with // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers // re-running a replicated deploy already carry references and never re-ingest. - const { isIsolatedApplication } = require('../server/threads/isolatedApplications.ts'); + const { isIsolatedApplication } = isolatedApplicationsRuntimeModule; const requestedIsolation = req.isolated; const isReplicatedExecution = typeof req._deploymentId === 'string'; const mode = req.deployment_id ? 'activate' : req.activate === false ? 'stage' : 'deploy'; @@ -550,11 +556,8 @@ async function deployComponent(req) { // then report any node-local inability to run it through component lifecycle status. const assertIsolationAdmission = async (isolated) => { if (!isolated || isReplicatedExecution) return; - const { - isolatedApplicationRefusal, - isolatedApplicationCapacityRefusal, - presentIsolatedApplicationNames, - } = require('../server/threads/isolatedApplications.ts'); + const { isolatedApplicationRefusal, isolatedApplicationCapacityRefusal, presentIsolatedApplicationNames } = + isolatedApplicationsRuntimeModule; let runningApplications; try { runningApplications = await manageThreads.getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS); @@ -584,7 +587,7 @@ async function deployComponent(req) { // refuse an activation whose artifact turns isolation OFF for a component whose present isolated state is // itself the refusal — a 409 for the request that would fix it. if (!isActivation) await assertIsolationAdmission(requestedIsolation ?? initiallyIsolated); - const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); + const { ingestCredentials, resolveCredentials } = secretOperationsRuntimeModule; // An activation resolves, fetches and installs nothing, so there is no credential for it to carry. // The validator rejects one; this keeps the ingest itself off the path rather than relying on that. if (!isActivation) req.credentials = await ingestCredentials(req, req.credentials, req.project); @@ -595,7 +598,7 @@ async function deployComponent(req) { if (req.package) { // Check if trying to overwrite a core component (requires force) // Lazy-load to avoid circular dependency with componentLoader - const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts'); + const { TRUSTED_RESOURCE_PLUGINS } = await import('./componentLoader.ts'); if (TRUSTED_RESOURCE_PLUGINS[req.project] && !req.force) { throw handleHDBError( new Error(), @@ -707,7 +710,7 @@ async function deployComponent(req) { waitMs: credentialsWaitMs, }); - const application = new Application({ + const application: any = new Application({ name: req.project, payload: extractionPayload, packageIdentifier: req.package, @@ -746,7 +749,7 @@ async function deployComponent(req) { let declaredRootConfig = null; const certification = mode !== 'stage' && (req.restart === true || req.restart === 'rolling') - ? require('./canaryRollout.ts').deployCertification({ + ? loadRuntimeModule('components/canaryRollout').deployCertification({ component: req.project, deploymentId: req.deployment_id ?? req._deploymentId, eligible: () => (wasIsolated ?? initiallyIsolated) === (nowIsolated ?? initiallyIsolated), @@ -796,7 +799,7 @@ async function deployComponent(req) { await assertIsolationAdmission(nowIsolated); if (!isReplicatedExecution && req.package) req.isolated = nowIsolated; if (!req.package) return; - const applicationConfig = { package: req.package }; + const applicationConfig: any = { package: req.package }; if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) { applicationConfig.install = { command: req.install_command, @@ -883,7 +886,7 @@ async function deployComponent(req) { recorder?.seal(); emit('phase', { phase: 'replicate', status: 'start' }); // A rolling activation's peers already hold the artifact; the job activates it on each. - let response = + let response: any = rollingActivation && mode === 'activate' ? { message: '' } : await server.replication.replicateOperation(req, { @@ -909,7 +912,7 @@ async function deployComponent(req) { // restartWorkers starts or stops the moving application's own worker. An already-live retry cannot know // which workers loaded the previous release, so it restarts them all. const restartScope = application.alreadyActive ? '*' : wasIsolated && nowIsolated ? application.name : undefined; - const { awaitRestart } = require('./awaitRestart.ts'); + const { awaitRestart } = loadRuntimeModule('components/awaitRestart'); // Main started this release's rollout at its commit; the origin only waits for it. const awaitCertifiedRollout = async () => { if (!isMainThread) return; @@ -932,7 +935,7 @@ async function deployComponent(req) { emit('phase', { phase: 'restart', status: 'start' }); await awaitCertifiedRollout(); if (recorder && !req.ignore_replication_errors) assertPeersStaged(application.name, recorder, response); - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + const serverUtilities = loadRuntimeModule('server/serverHelpers/serverUtilities'); const jobResponse = await serverUtilities.executeJob({ operation: 'restart_service', service: 'http', @@ -960,16 +963,16 @@ async function deployComponent(req) { logRestartOutcome(restart, `deploying ${application.name}`); response.message = `Successfully deployed: ${application.name}, restarting Harper`; } else if (rollingRestart) { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + const serverUtilities = serverUtilitiesRuntimeModule; emit('phase', { phase: 'restart', status: 'start' }); - const jobResponse = await serverUtilities.executeJob({ + const jobResponse: any = await serverUtilities.executeJob({ operation: 'restart_service', service: 'http', scope: manageThreads.encodeRestartScope(restartScope), scopeFallback: restartScope === undefined || restartScope === '*' ? undefined : manageThreads.encodeRestartScope(undefined), replicated: true, - }); + } as any); emit('phase', { phase: 'restart', status: 'done' }); response.restartJobId = jobResponse.job_id; @@ -987,7 +990,7 @@ async function deployComponent(req) { // An existing component's watched files are handled by Scope/EntryHandler. Package // metadata is deliberately outside most plugin globs, so compare it across the atomic // swap as well: a dependency or module-entry change also invalidates loaded code. - const { requestRestartAfterDeploy } = require('./requestRestart.ts'); + const { requestRestartAfterDeploy } = requestRestartRuntimeModule; requestRestartAfterDeploy( application.isNewComponent, application.packageMetadataChanged, @@ -1007,9 +1010,9 @@ async function deployComponent(req) { const failedPeers = recorder.getFailedPeers(); if (failedPeers.length > 0) { const detail = failedPeers - .map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`) + .map((peer) => `${peer.node ?? 'unknown'} (${(peer.error as any)?.message ?? 'unknown error'})`) .join(', '); - const replicationError = new ServerError( + const replicationError: any = new ServerError( `Component '${application.name}' was ${mode === 'stage' ? 'staged' : 'deployed'} on the origin node ` + `but failed to replicate to ${failedPeers.length} of ${recorder.row.peer_results.length} peer ` + `node(s): ${detail}. See deployment ${recorder.deploymentId} (get_deployment) for details, or ` + @@ -1031,7 +1034,7 @@ async function deployComponent(req) { const unconfirmed = unconfirmedStagingPeers(response?.replicated); if (unconfirmed.length > 0) { const detail = unconfirmed.map((peer) => peer.node ?? 'unknown').join(', '); - const unconfirmedError = new ServerError( + const unconfirmedError: any = new ServerError( `Component '${application.name}' was staged on the origin node, but ${unconfirmed.length} peer ` + `node(s) did not confirm staging: ${detail}. Either they are unreachable, or they run a build ` + `that predates staged deploys — which treats this request as an ordinary deploy, so the ` + @@ -1087,7 +1090,7 @@ async function deployComponent(req) { const capture = installCapture.snapshot(); const phase = recorder?.row.phase; const baseMessage = err?.message ?? String(err); - const structured = { error: baseMessage }; + const structured: any = { error: baseMessage }; if (phase) structured.phase = phase; if (capture.lines.length > 0) structured.install_output = capture; if (recorder?.deploymentId) structured.deployment_id = recorder.deploymentId; @@ -1102,7 +1105,7 @@ async function deployComponent(req) { // Wrap as a ServerError so the Fastify error handler picks a 500 by default; preserve // an upstream statusCode (e.g. a ClientError from payload validation) if present. - const outErr = new ServerError(baseMessage, err?.statusCode); + const outErr: any = new ServerError(baseMessage, err?.statusCode); outErr.http_resp_msg = structured; emit('error', { @@ -1145,7 +1148,7 @@ function isConfirmedStaging(peer) { return peer?.staged === true || peer?.value?.staged === true || peer?.body?.staged === true; } -function unconfirmedStagingPeers(replicated) { +function unconfirmedStagingPeers(replicated?: any) { if (!Array.isArray(replicated)) return []; return replicated.filter((peer) => peer && !isConfirmedStaging(peer)); } @@ -1180,8 +1183,8 @@ function drainWhileDeploying( undecidedBoundMs = Infinity, interruptUndecided = async () => false ) { - const { registerShutdownDrain } = require('./shutdownDrain.ts'); - const answered = Promise.withResolvers(); + const { registerShutdownDrain } = loadRuntimeModule('components/shutdownDrain'); + const answered = Promise.withResolvers(); const unregister = registerShutdownDrain({ hasWork: () => Boolean(isDeploying()), drain: (deadlineMs) => { @@ -1192,7 +1195,7 @@ function drainWhileDeploying( timer = setTimeout(resolve, Math.max(0, deadlineMs - Date.now())); timer.unref(); }); - const undecided = new Promise((resolve) => { + const undecided = new Promise((resolve) => { if (!Number.isFinite(undecidedBoundMs)) return; bound = setTimeout(async () => { if (isDecided()) return; @@ -1238,7 +1241,7 @@ function certificationFailure(component, deploymentId, decision, { onOrigin }) { `again. ` : ''; const retry = `Deploy a fix, or activate deployment ${deploymentId} again once the cause is fixed.`; - const error = new ClientError( + const error: ClientError & { certification?: unknown } = new ClientError( `${component} was not deployed${onOrigin ? '' : ' on this node'}: release ${deploymentId} ${outcome}. ${remedy}` + `${onOrigin ? 'No other node received it. ' : ''}${retry}`, HTTP_STATUS_CODES.BAD_REQUEST @@ -1254,9 +1257,9 @@ function certificationFailure(component, deploymentId, decision, { onOrigin }) { } /** The fingerprint is read flat or from a wrapped body, as `unconfirmedStagingPeers` reads its marker. */ -function markInstallComparisons(ownFingerprint, replicated) { +function markInstallComparisons(ownFingerprint?: any, replicated?: any) { if (!ownFingerprint || !Array.isArray(replicated)) return undefined; - const peers = []; + const peers: any = []; for (const peer of replicated) { if (!peer || typeof peer !== 'object') continue; const comparison = compareInstallFingerprints( @@ -1273,13 +1276,13 @@ function markInstallComparisons(ownFingerprint, replicated) { // Ring buffer of install stdout/stderr lines, capped by both line count and bytes so // a chatty install can't unbounded-grow the error response. snapshot() reports whether // the head was dropped so callers can flag truncation. -function createInstallCapture(maxLines = 200, maxBytes = 16 * 1024) { - const lines = []; +function createInstallCapture(maxLines: any = 200, maxBytes: any = 16 * 1024) { + const lines: any = []; let bytes = 0; let dropped = 0; return { push(manager, stream, line) { - const entry = { manager, stream, line }; + const entry: any = { manager, stream, line }; const size = (line?.length ?? 0) + (stream?.length ?? 0) + (manager?.length ?? 0); lines.push(entry); bytes += size; @@ -1353,7 +1356,7 @@ async function getComponents() { continue; const itemPath = path.join(dir, itemName); if (item.isDirectory() || item.isSymbolicLink()) { - let res = { + let res: any = { name: itemName, entries: [], }; @@ -1361,7 +1364,7 @@ async function getComponents() { await walkDir(itemPath, res); } else { const stats = await fs.stat(itemPath); - const res = { + const res: any = { name: path.basename(itemName), mtime: stats.mtime, size: stats.size, @@ -1384,7 +1387,7 @@ async function getComponents() { name: componentsRoot.split(path.sep).slice(-1).pop(), entries: [], }); - const { getUnsatisfiedEnv } = require('./componentSecrets.ts'); + const { getUnsatisfiedEnv } = componentSecretsRuntimeModule; for (let entry of results.entries) { // Declared-but-unsatisfied `env:` expectations (#1550) — metadata only (name, description, // required, reason, tier), never values — so Studio/deploy output can render configure-me. @@ -1398,7 +1401,7 @@ async function getComponents() { if (componentConfig.loadComponent) entry.loadComponent = componentConfig.loadComponent; } - const { internal: statusInternal } = require('./status/index.ts'); + const { internal: statusInternal } = indexRuntimeModule; let consolidatedStatuses; try { @@ -1451,7 +1454,7 @@ function getPayloadRetentionMaxSize() { return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_PAYLOAD_RETENTION_MAX_SIZE; } -async function getComponentFile(req) { +async function getComponentFile(req?: any) { const validation = validator.getComponentFileValidator(req); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1505,7 +1508,7 @@ async function getComponentFile(req) { * @param req * @returns {Promise<{message:string}>} */ -async function setComponentFile(req) { +async function setComponentFile(req?: any) { const validation = validator.setComponentFileValidator(req); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1519,7 +1522,7 @@ async function setComponentFile(req) { } else { await fs.ensureDir(pathToComp); } - let response = await server.replication.replicateOperation(req); + let response: any = await server.replication.replicateOperation(req); response.message = `Successfully set component: ` + req.file; return response; } @@ -1529,7 +1532,7 @@ async function setComponentFile(req) { * @param req * @returns {{file:string, filePath:string}} */ -function resolveEnvFilePath(req) { +function resolveEnvFilePath(req?: any) { const file = req.file || '.env'; if (!isEnvFile(file)) { const msg = `'${file}' is not a .env file`; @@ -1544,7 +1547,7 @@ function resolveEnvFilePath(req) { * @param req * @returns {Promise<{file:string, keys:string[], size:number, mtime:Date}>} */ -async function getEnvKeys(req) { +async function getEnvKeys(req?: any) { const validation = validator.getEnvKeysValidator(req); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1568,7 +1571,7 @@ async function getEnvKeys(req) { * @param req * @returns {Promise<{message:string, keys:string[]}>} */ -async function setEnvValue(req) { +async function setEnvValue(req?: any) { const validation = validator.setEnvValueValidator(req); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1592,7 +1595,7 @@ async function setEnvValue(req) { } await fs.outputFile(filePath, updated, 'utf8'); - const response = await server.replication.replicateOperation(req); + const response: any = await server.replication.replicateOperation(req); response.message = `Successfully set env value(s) in ${file}`; response.keys = parseEnvKeys(updated); return response; @@ -1603,7 +1606,7 @@ async function setEnvValue(req) { * @param req * @returns {Promise<{message:string, keys:string[]}>} */ -async function deleteEnvValue(req) { +async function deleteEnvValue(req?: any) { const validation = validator.deleteEnvValueValidator(req); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1625,7 +1628,7 @@ async function deleteEnvValue(req) { const updated = removeEnvKeys(existing, keysToRemove); await fs.outputFile(filePath, updated, 'utf8'); - const response = await server.replication.replicateOperation(req); + const response: any = await server.replication.replicateOperation(req); response.message = `Successfully deleted env value(s) from ${file}`; response.keys = parseEnvKeys(updated); return response; @@ -1636,7 +1639,7 @@ async function deleteEnvValue(req) { * @param req * @returns {Promise<{message:string}>} */ -async function dropComponent(req) { +async function dropComponent(req?: any) { const validation = validator.dropComponentFileValidator(req); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1701,7 +1704,7 @@ async function dropComponent(req) { } response = await server.replication.replicateOperation(req); - const { applicationHasBranchStorage, removeBranchesForApplication } = require('../resources/branchDatabase.ts'); + const { applicationHasBranchStorage, removeBranchesForApplication } = branchDatabaseRuntimeModule; const branched = !file && applicationHasBranchStorage(project); if (req.restart !== true) { response.message = `Successfully dropped: ${projectPath}`; @@ -1724,7 +1727,7 @@ async function dropComponent(req) { } return; } - const { awaitRestart } = require('./awaitRestart.ts'); + const { awaitRestart } = awaitRestartRuntimeModule; const restart = await awaitRestart((onProgress) => manageThreads.restartWorkers('http', undefined, undefined, onProgress, restartScope) ); @@ -1750,25 +1753,25 @@ async function dropComponent(req) { return response; } -exports.customFunctionsStatus = customFunctionsStatus; -exports.getCustomFunctions = getCustomFunctions; -exports.getCustomFunction = getCustomFunction; -exports.setCustomFunction = setCustomFunction; -exports.dropCustomFunction = dropCustomFunction; -exports.addComponent = addComponent; -exports.dropCustomFunctionProject = dropCustomFunctionProject; -exports.packageComponent = packageComponent; -exports.deployComponent = deployComponent; -exports.unconfirmedStagingPeers = unconfirmedStagingPeers; -exports.confirmedStagingPeers = confirmedStagingPeers; -exports.certificationFailure = certificationFailure; -exports.drainWhileDeploying = drainWhileDeploying; -exports.markInstallComparisons = markInstallComparisons; -exports.peerDeployAnswerTimeoutMs = peerDeployAnswerTimeoutMs; -exports.getComponents = getComponents; -exports.getComponentFile = getComponentFile; -exports.setComponentFile = setComponentFile; -exports.getEnvKeys = getEnvKeys; -exports.setEnvValue = setEnvValue; -exports.deleteEnvValue = deleteEnvValue; -exports.dropComponent = dropComponent; +export { customFunctionsStatus }; +export { getCustomFunctions }; +export { getCustomFunction }; +export { setCustomFunction }; +export { dropCustomFunction }; +export { addComponent }; +export { dropCustomFunctionProject }; +export { packageComponent }; +export { deployComponent }; +export { unconfirmedStagingPeers }; +export { confirmedStagingPeers }; +export { certificationFailure }; +export { drainWhileDeploying }; +export { markInstallComparisons }; +export { peerDeployAnswerTimeoutMs }; +export { getComponents }; +export { getComponentFile }; +export { setComponentFile }; +export { getEnvKeys }; +export { setEnvValue }; +export { deleteEnvValue }; +export { dropComponent }; diff --git a/components/operationsValidation.js b/components/operationsValidation.ts similarity index 97% rename from components/operationsValidation.js rename to components/operationsValidation.ts index 8c5fd4db38..f278ee7b60 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.ts @@ -1,24 +1,22 @@ -'use strict'; - -const Joi = require('joi'); -const fs = require('fs-extra'); -const path = require('path'); -const validator = require('../validation/validationWrapper.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const hdbLogger = require('../utility/logging/harper_logger.ts'); -const configUtils = require('../config/configUtils.ts'); -const { hdbErrors } = require('../utility/errors/hdbError.ts'); +import Joi from 'joi'; +import fs from 'fs-extra'; +import * as path from 'node:path'; +import * as validator from '../validation/validationWrapper.ts'; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import hdbLogger from '../utility/logging/harper_logger.ts'; +import * as configUtils from '../config/configUtils.ts'; +import { hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS } = hdbErrors; -const { ENV_ENCRYPTED_PREFIX } = require('../utility/envFile.ts'); +import { ENV_ENCRYPTED_PREFIX } from '../utility/envFile.ts'; // File and project names can only be alphanumeric, dash and underscores. Both patterns are shared // with the CLI (utility/componentNames.ts): `harper deploy setup=true` resolves a project name and a // credential host client-side, and has to reject exactly what these schemas would. -const { - PROJECT_NAME_PATTERN: PROJECT_FILE_NAME_REGEX, +import { + PROJECT_NAME_PATTERN as PROJECT_FILE_NAME_REGEX, GIT_HOST_PATTERN, isReservedComponentName, -} = require('../utility/componentNames.ts'); -const { assertBranchedDatabases } = require('./Application.ts'); +} from '../utility/componentNames.ts'; +import { assertBranchedDatabases } from './Application.ts'; // dotenv's accepted key character set. Restricting keys to this prevents a crafted key (e.g. one // containing `=` or a newline) from injecting extra assignments into a .env file. @@ -35,7 +33,7 @@ const IPV6_SCHEMA = Joi.string().ip({ version: 'ipv6' }); // the retention pin would not recognise the artifact the request is about to activate, and would evict it. const DEPLOYMENT_ID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; -module.exports = { +export { getDropCustomFunctionValidator, setCustomFunctionValidator, addComponentValidator, diff --git a/components/requestRestart.ts b/components/requestRestart.ts index 50a318a82a..7cfa2d508b 100644 --- a/components/requestRestart.ts +++ b/components/requestRestart.ts @@ -1,3 +1,4 @@ +import { workerHooks } from '../server/threads/threadMessageState.ts'; import { Status } from '../server/status/index.ts'; interface NotifyingArrayBuffer extends ArrayBuffer { @@ -45,6 +46,8 @@ export function resetRestartNeeded() { restartNeededArray[0] = 0; } +workerHooks.resetRestartNeeded = resetRestartNeeded; + export function subscribeToRestartRequests(callback: () => void) { ensureInitialized(); if (onRestartRequestedCallback) throw new Error('A restart-request subscriber is already registered'); diff --git a/components/rootConfigPublication.ts b/components/rootConfigPublication.ts index 8cff6d83cd..ea62e5fa5b 100644 --- a/components/rootConfigPublication.ts +++ b/components/rootConfigPublication.ts @@ -13,7 +13,7 @@ import { composeReassertedEnvConfig, REASSERTING_CONFIG_ENV_VARS } from '../conf import logger, { errorForLog } from '../utility/logging/harper_logger.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import { ComponentPreparationLockTimeoutError, withComponentPreparationLock } from './componentPreparationLock.ts'; -import { isThreadRunning } from '../server/threads/manageThreads.js'; +import { isThreadRunning } from '../server/threads/manageThreads.ts'; /** * What an activation does to the component's entry in the root config, recorded in the activation journal diff --git a/components/secretOperations.ts b/components/secretOperations.ts index dcad57d249..2c98a5792d 100644 --- a/components/secretOperations.ts +++ b/components/secretOperations.ts @@ -19,7 +19,7 @@ import * as configUtils from '../config/configUtils.ts'; import * as terms from '../utility/hdbTerms.ts'; import { ClientError, handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; import logger from '../utility/logging/harper_logger.ts'; -import * as validator from './operationsValidation.js'; +import * as validator from './operationsValidation.ts'; import { getSecretCustody } from '../resources/secretDecryptor.ts'; import { encryptEnvelope, parseEnvelopeFields } from '../utility/secretEnvelope.ts'; import { ENV_ENCRYPTED_PREFIX } from '../utility/envFile.ts'; diff --git a/components/status/crossThread.ts b/components/status/crossThread.ts index 95715864ce..a8fa3d84b1 100644 --- a/components/status/crossThread.ts +++ b/components/status/crossThread.ts @@ -5,12 +5,12 @@ * and aggregating it into a unified view. */ -import { sendItcEvent } from '../../server/threads/itc.js'; +import { sendItcEvent } from '../../server/threads/itc.ts'; import { getWorkerIndex, onMessageByType, getEligibleBroadcastRecipientThreadIds, -} from '../../server/threads/manageThreads.js'; +} from '../../server/threads/manageThreads.ts'; import { ITC_EVENT_TYPES } from '../../utility/hdbTerms.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; import { ComponentStatusRegistry } from './ComponentStatusRegistry.ts'; diff --git a/config/configUtils.ts b/config/configUtils.ts index 9241c0e217..fea224eb02 100644 --- a/config/configUtils.ts +++ b/config/configUtils.ts @@ -1,15 +1,18 @@ import * as hdbTerms from '../utility/hdbTerms.ts'; import * as hdbUtils from '../utility/common_utils.ts'; -import logger from '../utility/logging/harper_logger.ts'; +import _logger from '../utility/logging/harper_logger.ts'; +const logger = _logger; import { configValidator, getDomainSocketPathLengthWarning, isLegacySqlApplicationEntry, } from '../validation/configValidator.ts'; import { isReservedComponentName } from '../utility/componentNames.ts'; -import fs from 'fs-extra'; -import YAML from 'yaml'; -import path from 'path'; +import _fs from 'fs-extra'; +const fs = _fs; +import _YAML from 'yaml'; +const YAML = _YAML; +import path from 'node:path'; import { constants as osConstants } from 'node:os'; import { isMainThread, threadId } from 'node:worker_threads'; import { randomBytes } from 'node:crypto'; diff --git a/config/harperConfigEnvVars.ts b/config/harperConfigEnvVars.ts index 6c5036f522..4f18b67a3e 100644 --- a/config/harperConfigEnvVars.ts +++ b/config/harperConfigEnvVars.ts @@ -23,7 +23,7 @@ */ import type { Logger } from '../utility/logging/logger.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'node:path'; import { isMainThread } from 'node:worker_threads'; import * as crypto from 'node:crypto'; @@ -31,6 +31,7 @@ import cloneDeep from 'lodash/cloneDeep.js'; import { getBackupDirPath } from './configHelpers.ts'; import { atomicWriteFile, renameWithRetry } from './configUtils.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; +import { loggerWithTag } from '../utility/logging/harper_logger.ts'; const STATE_FILE_NAME = '.harper-config-state.json'; // Staged beside the confirmed state while a config-file write is in flight, then renamed over it. @@ -46,12 +47,7 @@ const pendingStateFileName = () => `${PENDING_STATE_PREFIX}${process.pid}${PENDI // it, short enough that leaked wreckage does not suspend drift detection indefinitely. const PENDING_STATE_STALE_MS = 60 * 60 * 1000; -/** - * Get logger instance with tag - lazy loaded to avoid circular dependencies - * and ensure logger is initialized before use - */ function getLogger(): Logger { - const { loggerWithTag } = require('../utility/logging/harper_logger'); return loggerWithTag('env-config'); } diff --git a/dataLayer/SQLSearch.ts b/dataLayer/SQLSearch.ts index 1b74fc53a7..759ba9e0e1 100644 --- a/dataLayer/SQLSearch.ts +++ b/dataLayer/SQLSearch.ts @@ -9,14 +9,14 @@ import difference from 'lodash/difference.js'; import pullAt from 'lodash/pullAt.js'; import uniqBy from 'lodash/uniqBy.js'; -import * as alasql from 'alasql'; +import alasql from 'alasql'; alasql.options.cache = false; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; import clone from 'clone'; import RecursiveIterator from 'recursive-iterator'; import log from '../utility/logging/harper_logger.ts'; import * as commonUtils from '../utility/common_utils.ts'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import { hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; diff --git a/dataLayer/SearchObject.ts b/dataLayer/SearchObject.ts index cfdd1e8b88..d45bbaa5eb 100644 --- a/dataLayer/SearchObject.ts +++ b/dataLayer/SearchObject.ts @@ -25,7 +25,7 @@ class SearchObject { value, hash_attribute, get_attributes, - endValue, + endValue = undefined, reverse = false, limit = undefined, offset = undefined diff --git a/dataLayer/bulkLoad.ts b/dataLayer/bulkLoad.ts index b98d1e2878..92d0fc4235 100644 --- a/dataLayer/bulkLoad.ts +++ b/dataLayer/bulkLoad.ts @@ -1,32 +1,42 @@ import * as insert from './insert.ts'; -import * as validator from '../validation/fileLoadValidator.ts'; -import needle from 'needle'; +import * as _validator from '../validation/fileLoadValidator.ts'; +const validator = _validator; +import _needle from 'needle'; +const needle = _needle; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as hdbUtils from '../utility/common_utils.ts'; -import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +import { handleHDBError as _handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +const handleHDBError = _handleHDBError; import { HTTP_STATUS_CODES, HDB_ERROR_MSGS, CHECK_LOGS_WRAPPER } from '../utility/errors/commonErrors.ts'; import logger from '../utility/logging/harper_logger.ts'; -import * as papaParse from 'papaparse'; -import * as fs from 'fs-extra'; -import * as path from 'path'; -import { chain } from 'stream-chain'; -import StreamArray from 'stream-json/streamers/StreamArray'; -import Batch from 'stream-json/utils/Batch'; -import comp from 'stream-chain/utils/comp'; -import { finished } from 'stream'; +import papaParse from 'papaparse'; +import _fs from 'fs-extra'; +const fs = _fs; +import * as path from 'node:path'; +import streamChain from 'stream-chain'; +const chain = (streamChain as any).chain ?? streamChain; +import StreamArray from 'stream-json/streamers/StreamArray.js'; +import Batch from 'stream-json/utils/Batch.js'; +import comp from 'stream-chain/utils/comp.js'; +import { finished } from 'node:stream'; import * as env from '../utility/environment/environmentManager.ts'; import * as opFuncCaller from '../utility/OperationFunctionCaller.ts'; import * as AWSConnector from '../utility/AWS/AWSConnector.js'; import { MissingAwsSdkError } from '../utility/AWS/awsSdkLoader.ts'; import { BulkLoadFileObject, BulkLoadDataObject } from './dataObjects/BulkLoadObjects.js'; import PermissionResponseObject from '../security/data_objects/PermissionResponseObject.ts'; -import { verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts'; +import { verifyBulkLoadAttributePerms as _verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts'; +const verifyBulkLoadAttributePerms = _verifyBulkLoadAttributePerms; import { databases } from '../resources/databases.ts'; import { coerceType } from '../resources/Table.ts'; +import { onStartup } from '../utility/lifecycle.ts'; const CSV_NO_RECORDS_MSG = 'No records parsed from csv file.'; -const TEMP_DOWNLOAD_DIR = `${env.get('HDB_ROOT')}/tmp`; +let TEMP_DOWNLOAD_DIR = `${env.get('HDB_ROOT')}/tmp`; +onStartup(() => { + TEMP_DOWNLOAD_DIR = `${env.get('HDB_ROOT')}/tmp`; +}); import { schemaRegex } from '../validation/common_validators.ts'; const HIGHWATERMARK = 1024 * 1024 * 2; const MAX_JSON_ARRAY_SIZE = 5000; @@ -577,7 +587,7 @@ async function callPapaParse(jsonMessage) { } function createTransformMap(schema, table) { - const attributes = databases[schema][table].attributes; + const attributes = (databases[schema][table] as any).attributes; let mapOfTransforms = new Map(); // I don't know if this should be a Map, but this just makes a map of attributes with type coercions that we want for (let attribute of attributes) { if (attribute.type && !attribute.computed && !attribute.relationship) diff --git a/dataLayer/dataObjects/UpsertObject.js b/dataLayer/dataObjects/UpsertObject.ts similarity index 69% rename from dataLayer/dataObjects/UpsertObject.js rename to dataLayer/dataObjects/UpsertObject.ts index c318882c59..618be2415f 100644 --- a/dataLayer/dataObjects/UpsertObject.js +++ b/dataLayer/dataObjects/UpsertObject.ts @@ -1,10 +1,13 @@ -'use strict'; -const OPERATIONS_ENUM = require('../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import { OPERATIONS_ENUM } from '../../utility/hdbTerms.ts'; /** * object representing an UPSERT operation */ class UpsertObject { + operation: string; + schema: string; + table: string; + records: any[]; + __origin: any; /** * @param {String} schema * @param {string} table @@ -20,4 +23,4 @@ class UpsertObject { } } -module.exports = UpsertObject; +export default UpsertObject; diff --git a/dataLayer/delete.ts b/dataLayer/delete.ts index 2c52b084e6..bb52e182dd 100644 --- a/dataLayer/delete.ts +++ b/dataLayer/delete.ts @@ -1,14 +1,15 @@ -'use strict'; +import { loadNativePackage } from '../utility/packageUtils.js'; +('use strict'); import bulkDeleteValidator from '../validation/bulkDeleteValidator.ts'; import deleteValidator from '../validation/deleteValidator.ts'; import * as commonUtils from '../utility/common_utils.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; -import { promisify, callbackify } from 'util'; +import { promisify, callbackify } from 'node:util'; import * as terms from '../utility/hdbTerms.ts'; import * as globalSchema from '../utility/globalSchema.ts'; const pGlobalSchema = promisify(globalSchema.getTableSchema); -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import { DeleteResponseObject } from './DataLayerObjects.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; @@ -34,7 +35,7 @@ export async function deleteFilesBefore(deleteObj: any) { commonUtils.transformReq(deleteObj); - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); let parsedDate = moment(deleteObj.date, moment.ISO_8601); if (!parsedDate.isValid()) { throw handleHDBError( diff --git a/dataLayer/export.ts b/dataLayer/export.ts index 01440d8c25..bcdfdfa0ca 100644 --- a/dataLayer/export.ts +++ b/dataLayer/export.ts @@ -1,14 +1,14 @@ -'use strict'; +import { loadRuntimeModule } from '../utility/packageUtils.js'; import * as search from './search.ts'; import * as AWSConnector from '../utility/AWS/AWSConnector.js'; import * as awsSdkLoader from '../utility/AWS/awsSdkLoader.ts'; -import * as stream from 'stream'; +import * as stream from 'node:stream'; import * as hdbUtils from '../utility/common_utils.ts'; -import * as fs from 'fs-extra'; -import * as path from 'path'; +import fs from 'fs-extra'; +import * as path from 'node:path'; import hdbLogger from '../utility/logging/harper_logger.ts'; -import { promisify } from 'util'; +import { promisify } from 'node:util'; import * as hdbCommon from '../utility/common_utils.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; @@ -356,7 +356,7 @@ async function getRecords(exportObject: any) { break; case 'sql': { if (!pSql) { - const sql = require('../sqlTranslator/index'); + const sql = loadRuntimeModule('sqlTranslator/index'); pSql = promisify(sql.evaluateSQL); } operation = pSql; diff --git a/dataLayer/getBackup.ts b/dataLayer/getBackup.ts index ed8b339efb..a952135e7a 100644 --- a/dataLayer/getBackup.ts +++ b/dataLayer/getBackup.ts @@ -4,7 +4,7 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; // eslint-disable-next-line no-unused-vars import GetBackupObject from './GetBackupObject.ts'; import * as hdbUtils from '../utility/common_utils.ts'; diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index d60e80a338..d6437b0041 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -14,7 +14,7 @@ import { claimDatabaseDropPreparations, releaseDatabaseDropPreparations, } from '../../resources/databaseDropPreparation.ts'; -import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.js'; +import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.ts'; import SearchObject from '../SearchObject.ts'; import { OPERATIONS_ENUM, @@ -24,7 +24,7 @@ import { REPLICATED_FROM, } from '../../utility/hdbTerms.ts'; import * as signalling from '../../utility/signalling.ts'; -import { SchemaEventMsg } from '../../server/threads/itc.js'; +import { SchemaEventMsg } from '../../server/threads/itc.ts'; import { asyncSetTimeout } from '../../utility/common_utils.ts'; import { transaction } from '../../resources/transaction.ts'; import type { @@ -41,7 +41,7 @@ import { errorToString } from '../../utility/logging/harper_logger.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; import { boundedAuditPruneEnd, raiseAuditFloor } from '../../resources/auditStore.ts'; import { BridgeMethods } from './BridgeMethods.ts'; -import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.js'; +import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.ts'; import { createBackupStream, resolveSingleRootStore } from '../rocksdbBackup.ts'; import { DeleteTransactionLogsBeforeResults } from './DeleteTransactionLogsBeforeResults.ts'; import type { Readable } from 'node:stream'; @@ -356,7 +356,7 @@ export class ResourceBridge extends BridgeMethods { } async deleteRecords(deleteObj) { - const Table = getDatabases()[deleteObj.schema][deleteObj.table]; + const Table: any = getDatabases()[deleteObj.schema][deleteObj.table]; const context: Context = { user: deleteObj.hdb_user }; if (deleteObj.replicateTo) context.replicateTo = deleteObj.replicateTo; if (deleteObj.replicatedConfirmation) context.replicatedConfirmation = deleteObj.replicatedConfirmation; @@ -392,7 +392,7 @@ export class ResourceBridge extends BridgeMethods { * @returns {undefined} */ async deleteRecordsBefore(deleteObj) { - const Table = getDatabases()[deleteObj.schema][deleteObj.table]; + const Table: any = getDatabases()[deleteObj.schema][deleteObj.table]; if (!Table.createdTimeProperty) { throw new ClientError( `Table must have a '__createdtime__' attribute or @createdTime timestamp defined to perform this operation` diff --git a/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js b/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts similarity index 87% rename from dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js rename to dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts index 8fd159da10..ba8e693ddf 100644 --- a/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +++ b/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts @@ -1,11 +1,9 @@ -'use strict'; +import * as hdbUtils from '../../../utility/common_utils.ts'; +import log from '../../../utility/logging/harper_logger.ts'; +import { getDatabases } from '../../../resources/databases.ts'; +import { ClientError } from '../../../utility/errors/hdbError.ts'; -const hdbUtils = require('../../../utility/common_utils.ts'); -const log = require('../../../utility/logging/harper_logger.ts'); -const { getDatabases } = require('../../../resources/databases.ts'); -const { ClientError } = require('../../../utility/errors/hdbError.ts'); - -module.exports = insertUpdateValidate; +export default insertUpdateValidate; //IMPORTANT - This code is the same code as the async validation() function in dataLayer/insert - make sure any changes // below are also made there. This is to resolve a circular dependency. @@ -32,7 +30,7 @@ function insertUpdateValidate(writeObject) { throw new ClientError('records must be an array'); } - let schemaTable = getDatabases()[writeObject.schema]?.[writeObject.table]; + let schemaTable: any = getDatabases()[writeObject.schema]?.[writeObject.table]; if (hdbUtils.isEmpty(schemaTable)) { throw new ClientError(`could not retrieve schema:${writeObject.schema} and table ${writeObject.table}`); } diff --git a/dataLayer/harperBridge/harperBridge.ts b/dataLayer/harperBridge/harperBridge.ts index bb697a182d..051910d694 100644 --- a/dataLayer/harperBridge/harperBridge.ts +++ b/dataLayer/harperBridge/harperBridge.ts @@ -2,8 +2,11 @@ import { ResourceBridge } from './ResourceBridge.ts'; import * as envMngr from '../../utility/environment/environmentManager.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} let harperBridge; // ResourceBridge /** diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts index 513f6df3bb..474173508c 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts @@ -1,22 +1,17 @@ -'use strict'; - -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const writeUtility = require('../../../../utility/lmdb/writeUtility.ts'); -const { getSystemSchemaPath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const { validateBySchema } = require('../../../../validation/validationWrapper.ts'); -const Joi = require('joi'); -const LMDBCreateAttributeObject = - require('../lmdbUtility/LMDBCreateAttributeObject.js').default || - require('../lmdbUtility/LMDBCreateAttributeObject.js'); -const returnObject = require('../../bridgeUtility/insertUpdateReturnObj.js'); -const { handleHDBError, hdbErrors, ClientError } = require('../../../../utility/errors/hdbError.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as writeUtility from '../../../../utility/lmdb/writeUtility.ts'; +import { getSystemSchemaPath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import { validateBySchema } from '../../../../validation/validationWrapper.ts'; +import Joi from 'joi'; +import LMDBCreateAttributeObject from '../lmdbUtility/LMDBCreateAttributeObject.ts'; +import returnObject from '../../bridgeUtility/insertUpdateReturnObj.js'; +import { handleHDBError, hdbErrors, ClientError } from '../../../../utility/errors/hdbError.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - const ACTION = 'inserted'; -module.exports = lmdbCreateAttribute; +export default lmdbCreateAttribute; /** * First adds the attribute to the system attribute table, then creates the dbi. @@ -100,10 +95,10 @@ async function lmdbCreateAttribute(createAttributeObj) { hdbAttributeEnv, // I'm not sure what else to do with these for now, but I do want to eslint to check the rest of the codebase // for undefined vars. - WSM 2025-11-26 - // eslint-disable-next-line no-undef - HDB_TABLE_INFO.hash_attribute, - // eslint-disable-next-line no-undef - hdbAttributeAttributes, + // @ts-expect-error - HDB_TABLE_INFO is a legacy global injected at runtime + HDB_TABLE_INFO.hash_attribute, // eslint-disable-line no-undef + // @ts-expect-error - hdbAttributeAttributes is a legacy global injected at runtime + hdbAttributeAttributes, // eslint-disable-line no-undef [record] ); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts similarity index 62% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts index e9a4a085d3..7913cb513d 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts @@ -1,19 +1,17 @@ -'use strict'; - -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; // eslint-disable-next-line no-unused-vars -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbInsertRecords = require('../../../../utility/lmdb/writeUtility.ts').insertRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const logger = require('../../../../utility/logging/harper_logger.ts'); +import InsertObject from '../../../InsertObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +import { insertRecords as lmdbInsertRecords } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; -module.exports = lmdbCreateRecords; +export default lmdbCreateRecords; /** * Orchestrates the insertion of data into LMDB and the creation of new attributes/dbis @@ -23,7 +21,7 @@ module.exports = lmdbCreateRecords; */ async function lmdbCreateRecords(insertObj) { try { - let { schemaTable, attributes } = insertUpdateValidate(insertObj); + let { schema_table: schemaTable, attributes } = insertUpdateValidate(insertObj); lmdbProcessRows(insertObj, attributes, schemaTable.hash_attribute); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts similarity index 54% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts index 501a7200cf..dc3404367a 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts @@ -1,12 +1,11 @@ -'use strict'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import _lmdbCreateRecords from './lmdbCreateRecords.ts'; +const lmdbCreateRecords = _lmdbCreateRecords; +import InsertObject from '../../../InsertObject.ts'; +import fs from 'fs-extra'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbCreateRecords = require('./lmdbCreateRecords.js'); -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const fs = require('fs-extra'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); - -module.exports = lmdbCreateSchema; +export default lmdbCreateSchema; /** * creates the meta data for the schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts similarity index 70% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts index 1f95a69701..5ede0ab25b 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts @@ -1,17 +1,14 @@ -'use strict'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as _writeUtility from '../../../../utility/lmdb/writeUtility.ts'; +const writeUtility = _writeUtility; +import { getSystemSchemaPath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import lmdbCreateAttribute from './lmdbCreateAttribute.ts'; +import LMDBCreateAttributeObject from '../lmdbUtility/LMDBCreateAttributeObject.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; +import createTxnEnvironments from '../lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts'; -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const writeUtility = require('../../../../utility/lmdb/writeUtility.ts'); -const { getSystemSchemaPath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const lmdbCreateAttribute = require('./lmdbCreateAttribute.js'); -const LMDBCreateAttributeObject = - require('../lmdbUtility/LMDBCreateAttributeObject.js').default || - require('../lmdbUtility/LMDBCreateAttributeObject.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); -const createTxnEnvironments = require('../lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js'); - -module.exports = lmdbCreateTable; +export default lmdbCreateTable; /** * Writes new table data to the system tables creates the environment file and creates two datastores to track created and updated @@ -60,10 +57,10 @@ async function lmdbCreateTable(tableSystemData, tableCreateObj) { hdbTableEnv, // I'm not sure what else to do with these for now, but I do want to eslint to check the rest of the codebase // for undefined vars. - WSM 2025-11-26 - // eslint-disable-next-line no-undef - HDB_TABLE_INFO.hash_attribute, - // eslint-disable-next-line no-undef - hdbTableAttributes, + // @ts-expect-error - HDB_TABLE_INFO is a legacy global injected at runtime + HDB_TABLE_INFO.hash_attribute, // eslint-disable-line no-undef + // @ts-expect-error - hdbTableAttributes is a legacy global injected at runtime + hdbTableAttributes, // eslint-disable-line no-undef [tableSystemData] ); //create attributes for hash attribute created/updated time stamps diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts similarity index 81% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts index 0d2f14df30..22b9ae5ade 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts @@ -1,20 +1,17 @@ -'use strict'; - -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; // eslint-disable-next-line no-unused-vars -const DeleteBeforeObject = - require('../../../DeleteBeforeObject.ts').default || require('../../../DeleteBeforeObject.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const DeleteAuditLogsBeforeResults = require('./DeleteAuditLogsBeforeResults.js'); -const promisify = require('util').promisify; +import DeleteBeforeObject from '../../../DeleteBeforeObject.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import DeleteAuditLogsBeforeResults from './DeleteAuditLogsBeforeResults.js'; +import { promisify } from 'node:util'; const pSettimeout = promisify(setTimeout); const BATCH_SIZE = 10000; const SLEEP_TIME_MS = 100; -module.exports = deleteAuditLogsBefore; +export default deleteAuditLogsBefore; /** * diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts index fc9c872e35..1f771cb1a2 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts @@ -1,13 +1,11 @@ -'use strict'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as deleteUtility from '../../../../utility/lmdb/deleteUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; -const hdbUtils = require('../../../../utility/common_utils.ts'); -const deleteUtility = require('../../../../utility/lmdb/deleteUtility.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); -const logger = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbDeleteRecords; +export default lmdbDeleteRecords; /** * Deletes a full table row at a certain hash. @@ -76,7 +74,7 @@ async function lmdbDeleteRecords(deleteObj, writeToTxnLog = true) { * @param {number} txnTime - the transaction timestamp * @returns {{skipped_hashes: [], deleted_hashes: [], message: string}} */ -function createDeleteResponse(deleted, skipped, txnTime) { +function createDeleteResponse(deleted, skipped, txnTime?) { let total = deleted.length + skipped.length; let plural = total === 1 ? 'record' : 'records'; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts similarity index 81% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts index de7445992c..e58edbb6de 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts @@ -1,19 +1,16 @@ -'use strict'; - -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); +import SearchObject from '../../../SearchObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; // eslint-disable-next-line no-unused-vars -const DropAttributeObject = - require('../../../DropAttributeObject.ts').default || require('../../../DropAttributeObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const systemSchema = require('../../../../json/systemSchema.json'); -const searchByValue = require('./lmdbSearchByValue.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); +import DropAttributeObject from '../../../DropAttributeObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { systemSchema } from '../../../../utility/systemSchema.js'; +import searchByValue from './lmdbSearchByValue.ts'; +import deleteRecords from './lmdbDeleteRecords.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; -module.exports = lmdbDropAttribute; +export default lmdbDropAttribute; /** * First deletes the attribute/dbi from lmdb then removes its record from system table diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts similarity index 71% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts index 7191f91f1e..5e206c2cbb 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts @@ -1,20 +1,16 @@ -'use strict'; - -const fs = require('fs-extra'); -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const SearchByHashObject = - require('../../../SearchByHashObject.ts').default || require('../../../SearchByHashObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); -const dropTable = require('./lmdbDropTable.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const getDataByHash = require('./lmdbGetDataByHash.js'); -const searchDataByValue = require('./lmdbSearchByValue.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import fs from 'fs-extra'; +import SearchObject from '../../../SearchObject.ts'; +import SearchByHashObject from '../../../SearchByHashObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; +import dropTable from './lmdbDropTable.ts'; +import deleteRecords from './lmdbDeleteRecords.ts'; +import getDataByHash from './lmdbGetDataByHash.ts'; +import searchDataByValue from './lmdbSearchByValue.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - -module.exports = lmdbDropSchema; +export default lmdbDropSchema; /** * deletes all environment files under the schema folder, deletes all schema/table/attribute meta data from system @@ -36,7 +32,7 @@ async function lmdbDropSchema(dropSchemaObj) { [hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_NAME_KEY] ); - let tables = Array.from(await searchDataByValue(tableSearchObj)); + let tables: any[] = Array.from(await searchDataByValue(tableSearchObj)); for (let x = 0; x < tables.length; x++) { const deleteTableObj = { diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts similarity index 79% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts index 8573271d12..fca3ae35dd 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts @@ -1,16 +1,16 @@ -'use strict'; +import SearchObject from '../../../SearchObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; +import _searchByValue from './lmdbSearchByValue.ts'; +const searchByValue = _searchByValue; +import _deleteRecords from './lmdbDeleteRecords.ts'; +const deleteRecords = _deleteRecords; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); -const searchByValue = require('./lmdbSearchByValue.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbDropTable; +export default lmdbDropTable; /** * Calls drops the table, all of it's attribute & deletes the environment @@ -68,7 +68,7 @@ async function deleteAttributesFromSystem(dropTableObj) { [hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ID_KEY] ); - let searchResult = Array.from(await searchByValue(searchObj)); + let searchResult: any[] = Array.from(await searchByValue(searchObj)); let deleteIds = []; for (let x = 0; x < searchResult.length; x++) { @@ -109,7 +109,7 @@ async function dropTableFromSystem(dropTableObj) { let searchResult; let deleteTable; try { - searchResult = Array.from(await searchByValue(searchObj)); + searchResult = Array.from(await searchByValue(searchObj)) as any[]; } catch (err) { throw err; } diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts index 3ca34e7262..d0aa6713cf 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts @@ -1,13 +1,6 @@ -'use strict'; - -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); - -module.exports = { - flush, - resetReadTxn, -}; - +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +export { flush, resetReadTxn }; /** * This is wrapper for sync/flush to disk * @param schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts similarity index 74% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts index 941467fce2..6415b89a64 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts @@ -1,14 +1,13 @@ -'use strict'; +import { loadNativePackage } from '../../../../utility/packageUtils.js'; +import { Readable } from 'node:stream'; +import { getDatabases } from '../../../../resources/databases.ts'; +import { readSync, openSync, createReadStream } from 'node:fs'; +import { OpenDBIObject } from '../../../../utility/lmdb/OpenDBIObject.ts'; +import OpenEnvironmentObject from '../../../../utility/lmdb/OpenEnvironmentObject.ts'; +import { AUDIT_STORE_OPTIONS } from '../../../../resources/auditStore.ts'; +import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../../../../utility/lmdb/terms.ts'; -const { Readable } = require('stream'); -const { getDatabases } = require('../../../../resources/databases.ts'); -const { readSync, openSync, createReadStream } = require('fs'); -const { OpenDBIObject } = require('../../../../utility/lmdb/OpenDBIObject.ts'); -const OpenEnvironmentObject = require('../../../../utility/lmdb/OpenEnvironmentObject.ts'); -const { AUDIT_STORE_OPTIONS } = require('../../../../resources/auditStore.ts'); -const { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } = require('../../../../utility/lmdb/terms.ts'); - -module.exports = getBackup; +export default getBackup; const META_SIZE = 32768; const DELAY_ITERATIONS = 100; /** @@ -23,11 +22,11 @@ async function getBackup(getBackupObj) { let tables = getBackupObj.tables || (getBackupObj.table && [getBackupObj.table]); if (tables) { // if tables are specified, we have to copy the database with just the specified tables and then stream that - let tableClass = database[tables[0]]; + let tableClass: any = database[tables[0]]; if (!tableClass) throw new Error(`Can not find table ${tables[0]}`); // we use the attribute store to drive this process, finding the right stores to duplicate let attributeStore = tableClass.dbisDB; - let backupRoot = require('lmdb').open({ noSync: true, maxDbs: OpenEnvironmentObject.MAX_DBS }); // open a temporary database (this + let backupRoot = loadNativePackage('lmdb').open({ noSync: true, maxDbs: OpenEnvironmentObject.MAX_DBS }); // open a temporary database (this // will also cause it to // close on completion) let resolution; @@ -65,7 +64,7 @@ async function getBackup(getBackupObj) { await copyDatabase(AUDIT_STORE_NAME, { ...AUDIT_STORE_OPTIONS }); } await resolution; - let stream = createReadStream(backupRoot.path); + let stream: any = createReadStream((backupRoot as any).path); stream.headers = getHeaders(); stream.on('close', () => { readTxn.done(); @@ -73,21 +72,21 @@ async function getBackup(getBackupObj) { }); return stream; } - const firstTable = database[Object.keys(database)[0]]; - const store = firstTable.primaryStore; + const firstTable: any = database[Object.keys(database)[0]]; + const store: any = firstTable.primaryStore; - let fd = openSync(store.path); + let fd = openSync(store.path, 'r'); return store.transaction(() => { let metaBuffers = Buffer.alloc(META_SIZE); - readSync(fd, metaBuffers, 0, META_SIZE); // sync, need to do this as fast as possible since we are in a write txn + readSync(fd, metaBuffers, 0, META_SIZE, null); // sync, need to do this as fast as possible since we are in a write txn store.resetReadTxn(); // make sure we are not using a cached read transaction, force a fresh one let readTxn = store.useReadTransaction(); // this guarantees the current transaction is preserved in the backup // renew is necessary because normally renew is actually lazily called on the next db operation, but // we are not performing any db operations readTxn.renew(); // create a file stream that starts after the meta area - let fileStream = createReadStream(null, { fd, start: META_SIZE }); - let stream = new Readable.from( + let fileStream = createReadStream(null as any, { fd, start: META_SIZE }); + let stream: any = Readable.from( (async function* () { yield metaBuffers; // return the meta area that was frozen inside the write transaction for await (const chunk of fileStream) { diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts index aa6d05980d..cb07662299 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts @@ -1,9 +1,7 @@ -'use strict'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import hashSearchInit from '../lmdbUtility/initializeHashSearch.ts'; -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js'); - -module.exports = lmdbGetDataByHash; +export default lmdbGetDataByHash; /** * fetches records by their hash values and returns a map of the results diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts index e5b6abd264..03531c93fc 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts @@ -1,12 +1,8 @@ -'use strict'; - -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbSearch = require('../lmdbUtility/lmdbSearch.js'); - -module.exports = lmdbGetDataByValue; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as lmdbSearch from '../lmdbUtility/lmdbSearch.ts'; +export default lmdbGetDataByValue; /** * gets records by value returns a map of objects diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts similarity index 88% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts index 30f91654fb..b514326ce2 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts @@ -1,15 +1,13 @@ -'use strict'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import LMDBTransactionObject from '../lmdbUtility/LMDBTransactionObject.js'; +import log from '../../../../utility/logging/harper_logger.ts'; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const LMDBTransactionObject = require('../lmdbUtility/LMDBTransactionObject.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = readAuditLog; +export default readAuditLog; /** * function execute the readTransactionLog operation @@ -67,7 +65,7 @@ function searchTransactionsByTimestamp(env, timestamps = [0, Date.now()]) { return timestampDbi .getRange({ start: timestamps[0], end: nextValue }) - .map(({ value }) => Object.assign(new LMDBTransactionObject(), value)); + .map(({ value }) => Object.assign(Object.create(LMDBTransactionObject.prototype), value)); } /** @@ -193,7 +191,7 @@ function batchSearchTransactions(env, ids) { try { let value = timestampDbi.get(ids[x]); if (value) { - let txnRecord = Object.assign(new LMDBTransactionObject(), value); + let txnRecord = Object.assign(Object.create(LMDBTransactionObject.prototype), value); results.push(txnRecord); } } catch (e) { diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts index cd6d4df03b..c89212ba85 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts @@ -1,23 +1,17 @@ -'use strict'; - -// eslint-disable-next-line no-unused-vars -const { SearchByConditionsObject, SearchCondition } = - require('../../../SearchByConditionsObject.ts').default || require('../../../SearchByConditionsObject.ts'); -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const lmdb_search = require('../lmdbUtility/lmdbSearch.js'); -const cursorFunctions = require('../../../../utility/lmdb/searchCursorFunctions.ts'); -const sortBy = require('lodash/sortBy'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import SearchObject from '../../../SearchObject.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as lmdb_search from '../lmdbUtility/lmdbSearch.ts'; +import * as cursorFunctions from '../../../../utility/lmdb/searchCursorFunctions.ts'; +import sortBy from 'lodash/sortBy.js'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; const RANGE_ESTIMATE = 100000000; -module.exports = lmdbSearchByConditions; +export default lmdbSearchByConditions; /** * gets records by conditions - returns array of Objects @@ -81,7 +75,7 @@ async function lmdbSearchByConditions(searchObject) { // get the intersection of condition searches by using the indexed query for the first condition // and then filtering by all subsequent conditions let primaryDbi = env.dbis[tableInfo.hash_attribute]; - let filters = sortedConditions.slice(1).map(lmdb_search.filterByType); + let filters: any[] = sortedConditions.slice(1).map(lmdb_search.filterByType as any); let filtersLength = filters.length; let fetchAttributes = searchUtility.setGetWholeRowAttributes(env, searchObject.get_attributes); records = ids.map((id) => primaryDbi.get(id, { transaction, lazy: true })); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts similarity index 69% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts index 3be46a4256..f28aa8f41b 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts @@ -1,9 +1,7 @@ -'use strict'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import hashSearchInit from '../lmdbUtility/initializeHashSearch.ts'; -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js'); - -module.exports = lmdbSearchByHash; +export default lmdbSearchByHash; /** * fetches records by their hash values and returns an Array of the results diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts similarity index 54% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts index 45d0303b1c..a9e4f2260f 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts @@ -1,14 +1,10 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_search = require('../lmdbUtility/lmdbSearch.js'); - -module.exports = lmdbSearchByValue; +import SearchObject from '../../../SearchObject.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as lmdb_search from '../lmdbUtility/lmdbSearch.ts'; +export default lmdbSearchByValue; /** * gets records by value - returns array of Objects @@ -16,7 +12,7 @@ module.exports = lmdbSearchByValue; * @param {hdbTerms.VALUE_SEARCH_COMPARATORS} [comparator] * @returns {Promise<{}|{}[]>} */ -async function lmdbSearchByValue(searchObject, comparator) { +async function lmdbSearchByValue(searchObject, comparator?) { let comparatorSearch = !commonUtils.isEmpty(comparator); if (comparatorSearch && hdbTerms.VALUE_SEARCH_COMPARATORS_REVERSE_LOOKUP[comparator] === undefined) { throw new Error(`Value search comparator - ${comparator} - is not valid`); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts similarity index 74% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts index 9541156eac..d2c8855963 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts @@ -1,11 +1,6 @@ -'use strict'; - -const { database } = require('../../../../resources/databases.ts'); - -module.exports = { - writeTransaction, -}; +import { database } from '../../../../resources/databases.ts'; +export { writeTransaction }; /** * This is wrapper for write transactions, ensuring that all reads and writes within the callback occur atomically * @param schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts index 2b2dfa8c46..1e01412425 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts @@ -1,16 +1,14 @@ -'use strict'; - -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_update_records = require('../../../../utility/lmdb/writeUtility.ts').updateRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); -const logger = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbUpdateRecords; +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; +import lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { updateRecords as lmdb_update_records } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; + +export default lmdbUpdateRecords; /** * Orchestrates the update of data in LMDB and the creation of new attributes/dbis @@ -20,7 +18,7 @@ module.exports = lmdbUpdateRecords; */ async function lmdbUpdateRecords(updateObj) { try { - let { schemaTable, attributes } = insertUpdateValidate(updateObj); + let { schema_table: schemaTable, attributes } = insertUpdateValidate(updateObj); lmdbProcessRows(updateObj, attributes, schemaTable.hash_attribute); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts index f84db3a9fd..7756752844 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts @@ -1,21 +1,19 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const UpsertObject = - require('../../../dataObjects/UpsertObject.js').default || require('../../../dataObjects/UpsertObject.js'); -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_upsert_records = require('../../../../utility/lmdb/writeUtility.ts').upsertRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); +import UpsertObject from '../../../dataObjects/UpsertObject.ts'; +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; +import _lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +const lmdbProcessRows = _lmdbProcessRows; +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { upsertRecords as lmdb_upsert_records } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; -const logger = require('../../../../utility/logging/harper_logger.ts'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import logger from '../../../../utility/logging/harper_logger.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; -module.exports = lmdbUpsertRecords; +export default lmdbUpsertRecords; /** * Orchestrates the UPSERT of data in LMDB and the creation of new attributes/dbis diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts index e109a0845b..a27399c691 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts @@ -1,8 +1,4 @@ -'use strict'; - -const CreateAttributeObject = - require('../../../CreateAttributeObject.ts').default || require('../../../CreateAttributeObject.ts'); - +import CreateAttributeObject from '../../../CreateAttributeObject.ts'; class LMDBCreateAttributeObject extends CreateAttributeObject { /** * @@ -20,4 +16,4 @@ class LMDBCreateAttributeObject extends CreateAttributeObject { } } -module.exports = LMDBCreateAttributeObject; +export default LMDBCreateAttributeObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts index 5f08bbe765..b28282efdc 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts @@ -1,11 +1,10 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define a delete transaction */ class LMDBDeleteTransactionObject extends LMDBTransactionObject { + original_records: any; /** * @param {Array.} hash_values - hash values of deleted records * @param {Array.} originalRecords - original records prior to delete @@ -20,4 +19,4 @@ class LMDBDeleteTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBDeleteTransactionObject; +export default LMDBDeleteTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts similarity index 72% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts index 24d5c32710..287437b85f 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts @@ -1,11 +1,10 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an insert transaction */ class LMDBInsertTransactionObject extends LMDBTransactionObject { + records: any; /** * @param {Array.} records - inserted records * @param {string} userName - username that executed trasaction @@ -19,4 +18,4 @@ class LMDBInsertTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBInsertTransactionObject; +export default LMDBInsertTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts index d10d8e7360..290aed96fc 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts @@ -1,11 +1,11 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an update transaction */ class LMDBUpdateTransactionObject extends LMDBTransactionObject { + records: any; + original_records: any; /** * @param {Array.} records - records updated * @param {Array.} originalRecords - original state of records that were updated @@ -21,4 +21,4 @@ class LMDBUpdateTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBUpdateTransactionObject; +export default LMDBUpdateTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts index 5e39e3f5b4..1e42435ec1 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts @@ -1,11 +1,11 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an update transaction */ class LMDBUpsertTransactionObject extends LMDBTransactionObject { + records: any; + original_records: any; /** * @param {Array.} records - records updated * @param {Array.} originalRecords - original state of records that were updated @@ -21,4 +21,4 @@ class LMDBUpsertTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBUpsertTransactionObject; +export default LMDBUpsertTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts similarity index 52% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts index 9f1492513f..79f67e72be 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts @@ -1,11 +1,8 @@ -'use strict'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import { getSchemaPath } from './initializePaths.ts'; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const { getSchemaPath } = require('./initializePaths.js'); - -module.exports = initialize; +export default initialize; /** * diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts similarity index 90% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts index 4f89775d63..9b9a69e44b 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts @@ -1,14 +1,11 @@ -'use strict'; - -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const env = require('../../../../utility/environment/environmentManager.ts'); -const path = require('path'); -const minimist = require('minimist'); -const fs = require('fs-extra'); -const setPath = require('lodash/set'); -const { getConfigPath, updateConfigObject } = require('../../../../config/configUtils.ts'); -env.initSync(); +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as env from '../../../../utility/environment/environmentManager.ts'; +import * as path from 'node:path'; +import minimist from 'minimist'; +import fs from 'fs-extra'; +import setPath from 'lodash/set.js'; +import { getConfigPath, updateConfigObject } from '../../../../config/configUtils.ts'; const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms; let BASE_SCHEMA_PATH = undefined; @@ -68,7 +65,7 @@ function getTransactionAuditStorePath(schema, table) { ); } -function getSchemaPath(schema, table) { +function getSchemaPath(schema, table?: string | null) { schema = schema.toString(); table = table ? table.toString() : table; let schemaConfig = env.get(hdbTerms.CONFIG_PARAMS.DATABASES)?.[schema]; @@ -148,7 +145,7 @@ function resetPaths() { SYSTEM_SCHEMA_PATH = undefined; TRANSACTION_STORE_PATH = undefined; } -module.exports = { +export { getBaseSchemaPath, getSystemSchemaPath, getTransactionAuditStorePath, diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts similarity index 79% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts index e519e55e98..ecbd04dcc4 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts @@ -1,17 +1,14 @@ -'use strict'; - -const hUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const logger = require('../../../../utility/logging/harper_logger.ts'); -const lmdbCreateAttribute = require('../lmdbMethods/lmdbCreateAttribute.js'); -const LMDBCreateAttributeObject = - require('./LMDBCreateAttributeObject.js').default || require('./LMDBCreateAttributeObject.js'); -const signalling = require('../../../../utility/signalling.ts'); -const { SchemaEventMsg } = require('../../../../server/threads/itc.js'); +import * as hUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; +import lmdbCreateAttribute from '../lmdbMethods/lmdbCreateAttribute.ts'; +import LMDBCreateAttributeObject from './LMDBCreateAttributeObject.ts'; +import * as signalling from '../../../../utility/signalling.ts'; +import { SchemaEventMsg } from '../../../../server/threads/itc.ts'; const ATTRIBUTE_ALREADY_EXISTS = 'already exists in'; -module.exports = lmdbCheckForNewAttributes; +export default lmdbCheckForNewAttributes; /** * Uses a utility function to check if there are any new attributes that dont exist. Utility function diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts similarity index 70% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts index 41c4d8c2c0..8b723c3c74 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts @@ -1,16 +1,11 @@ -'use strict'; - -const fs = require('fs-extra'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); +import fs from 'fs-extra'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; // eslint-disable-next-line no-unused-vars -const CreateTableObject = - require('../../../CreateTableObject.ts').default || - require('../../../CreateTableObject.ts').default || - require('../../../CreateTableObject.ts'); +import CreateTableObject from '../../../CreateTableObject.ts'; -module.exports = createTransactionsAuditEnvironment; +export default createTransactionsAuditEnvironment; /** * Creates the environment to hold transactions diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts index 837a526e88..81994b5f0e 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts @@ -1,15 +1,12 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const log = require('../../../../utility/logging/harper_logger.ts'); -const uuid = require('uuid'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import InsertObject from '../../../InsertObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; +import { v4 as uuidv4 } from 'uuid'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - -module.exports = processRows; +export default processRows; /** * parses the records and validates the hash value for each row as well as adding updated/created time stamps @@ -70,7 +67,7 @@ function validateAttribute(attribute) { function validateHash(record, hash_attribute, operation) { if (!record.hasOwnProperty(hash_attribute) || hdbUtils.isEmptyOrZeroLength(record[hash_attribute])) { if (operation === hdbTerms.OPERATIONS_ENUM.INSERT || operation === hdbTerms.OPERATIONS_ENUM.UPSERT) { - record[hash_attribute] = uuid.v4(); + record[hash_attribute] = uuidv4(); //return here since the rest of the validations do not apply return; } diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts similarity index 93% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts index e5af231e85..3f4fa952bc 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts @@ -1,13 +1,11 @@ -'use strict'; - -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const systemSchema = require('../../../../json/systemSchema.json'); -const LMDB_ERRORS = require('../../../../utility/errors/commonErrors.ts').LMDB_ERRORS_ENUM; -const { getSchemaPath } = require('./initializePaths.js'); +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { systemSchema } from '../../../../utility/systemSchema.js'; +import { LMDB_ERRORS_ENUM as LMDB_ERRORS } from '../../../../utility/errors/commonErrors.ts'; +import { getSchemaPath } from './initializePaths.ts'; const WILDCARDS = hdbTerms.SEARCH_WILDCARDS; @@ -362,10 +360,5 @@ function createSearchTypeFromSearchObject(searchObject, hash_attribute, returnMa } } -module.exports = { - executeSearch, - createSearchTypeFromSearchObject, - prepSearch, - searchByType, - // filterByType, -}; +export { executeSearch, createSearchTypeFromSearchObject, prepSearch, searchByType }; +export const filterByType: any = undefined; // placeholder: filterByType is not yet implemented diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts similarity index 74% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts index 0889ca63f3..47d4240156 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts @@ -1,21 +1,18 @@ -'use strict'; +import * as environmentUtil from '../../../../utility/lmdb/environmentUtility.ts'; +import LMDBInsertTransactionObject from './LMDBInsertTransactionObject.ts'; +import LMDBUpdateTransactionObject from './LMDBUpdateTransactionObject.ts'; +import LMDBUpsertTransactionObject from './LMDBUpsertTransactionObject.ts'; +import LMDBDeleteTransactionObject from './LMDBDeleteTransactionObject.ts'; -const environmentUtil = require('../../../../utility/lmdb/environmentUtility.ts'); -const LMDBInsertTransactionObject = require('./LMDBInsertTransactionObject.js'); -const LMDBUpdateTransactionObject = require('./LMDBUpdateTransactionObject.js'); -const LMDBUpsertTransactionObject = require('./LMDBUpsertTransactionObject.js'); -const LMDBDeleteTransactionObject = require('./LMDBDeleteTransactionObject.js'); +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbUtil from '../../../../utility/common_utils.ts'; +import { CONFIG_PARAMS } from '../../../../utility/hdbTerms.ts'; +import * as envMngr from '../../../../utility/environment/environmentManager.ts'; -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbUtil = require('../../../../utility/common_utils.ts'); -const { CONFIG_PARAMS } = require('../../../../utility/hdbTerms.ts'); -const envMngr = require('../../../../utility/environment/environmentManager.ts'); -envMngr.initSync(); +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; +import { getTransactionAuditStorePath } from './initializePaths.ts'; -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; -const { getTransactionAuditStorePath } = require('./initializePaths.js'); - -module.exports = writeTransaction; +export default writeTransaction; /** * @@ -33,7 +30,7 @@ async function writeTransaction(hdbOperation, lmdbResponse) { let txnObject = createTransactionObject(hdbOperation, lmdbResponse); - if (txnObject === undefined || txnObject.hash_values.length === 0) { + if (txnObject === undefined || (txnObject as any).hash_values.length === 0) { return; } diff --git a/dataLayer/insert.ts b/dataLayer/insert.ts index f65f7e6de6..e53abcdd6e 100644 --- a/dataLayer/insert.ts +++ b/dataLayer/insert.ts @@ -7,10 +7,10 @@ * as the update module is meant to be used in more specific circumstances. */ import insertValidator from '../validation/insertValidator.ts'; -import * as hdbUtils from '../utility/common_utils.ts'; -import * as util from 'util'; -// Leave this unused signalling import here. Due to circular dependencies we bring it in early to load it before the bridge -const harperBridge = require('./harperBridge/harperBridge').default; +import * as _hdbUtils from '../utility/common_utils.ts'; +const hdbUtils = _hdbUtils; +import * as util from 'node:util'; +import harperBridge from './harperBridge/harperBridge.ts'; import * as globalSchema from '../utility/globalSchema.ts'; import log from '../utility/logging/harper_logger.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; diff --git a/dataLayer/readAuditLog.ts b/dataLayer/readAuditLog.ts index b49b4b089f..236a0f4724 100644 --- a/dataLayer/readAuditLog.ts +++ b/dataLayer/readAuditLog.ts @@ -1,6 +1,6 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; // eslint-disable-next-line no-unused-vars import ReadAuditLogObject from './ReadAuditLogObject.ts'; import * as hdbUtils from '../utility/common_utils.ts'; diff --git a/dataLayer/rocksdbBackup.ts b/dataLayer/rocksdbBackup.ts index ed81817ce8..21cfd65bd3 100644 --- a/dataLayer/rocksdbBackup.ts +++ b/dataLayer/rocksdbBackup.ts @@ -26,7 +26,7 @@ import { getBackupDirPath } from '../config/configHelpers.ts'; import { CONFIG_PARAMS, OPERATIONS_ENUM } from '../utility/hdbTerms.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; import { beginRestore, clearRestoringMarker, diff --git a/dataLayer/schema.ts b/dataLayer/schema.ts index 31ca3882e5..b042fe9636 100644 --- a/dataLayer/schema.ts +++ b/dataLayer/schema.ts @@ -1,6 +1,7 @@ 'use strict'; -import * as schemaMetadataValidator from '../validation/schemaMetadataValidator.ts'; +import * as _schemaMetadataValidator from '../validation/schemaMetadataValidator.ts'; +const schemaMetadataValidator = _schemaMetadataValidator; import { validateBySchema } from '../validation/validationWrapper.ts'; import { commonValidators, schemaRegex } from '../validation/common_validators.ts'; import Joi from 'joi'; @@ -8,20 +9,21 @@ import logger from '../utility/logging/harper_logger.ts'; import { v4 as uuidV4 } from 'uuid'; import * as signalling from '../utility/signalling.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; -import * as util from 'util'; -const harperBridge = require('./harperBridge/harperBridge').default; +import * as util from 'node:util'; +import harperBridge from './harperBridge/harperBridge.ts'; import { handleHDBError, ClientError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; import { databaseDropRecoveryPending, - getDatabases, + getDatabases as _getDatabases, dropTableMeta, isBranchIdentity, isNodeLocalTable, recordTableDrop, } from '../resources/databases.ts'; +const getDatabases = _getDatabases; import { transformReq } from '../utility/common_utils.ts'; import { server } from '../server/Server.ts'; import { cleanupOrphans } from '../resources/blob.ts'; @@ -369,7 +371,7 @@ function dropAttributeFromGlobal(dropAttributeObject) { export async function createAttribute(createAttributeObject: any) { transformReq(createAttributeObject); - const tableAttr = getDatabases()[createAttributeObject.schema][createAttributeObject.table].attributes; + const tableAttr = (getDatabases()[createAttributeObject.schema][createAttributeObject.table] as any).attributes; for (const { name } of tableAttr) { if (name === createAttributeObject.attribute) { throw handleHDBError( diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 35df438f1e..bbcccde0a9 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -8,12 +8,17 @@ import * as hdbUtils from '../utility/common_utils.ts'; import { handleHDBError, ClientError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; +import { getDatabases as _getDatabases } from '../resources/databases.ts'; +const getDatabases = _getDatabases; +import fs from 'fs-extra'; import * as envMngr from '../utility/environment/environmentManager.ts'; -envMngr.initSync(); -import { getDatabases } from '../resources/databases.ts'; -import * as fs from 'fs-extra'; import { isOperationAuthorizationBypassed } from '../server/serverHelpers/operationAuthorizationState.ts'; +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} /** * This method is exposed to the API and internally for system operations. If the op is being made internally, the `opObj` * argument is not passed and, therefore, no permissions are used to filter the final schema metadata results. @@ -141,7 +146,7 @@ async function descTable(describeTableObject: any, attrPerms?: any) { HTTP_STATUS_CODES.NOT_FOUND ); } - let tableObj = tables[table]; + let tableObj: any = tables[table]; if (!tableObj) throw handleHDBError( new Error(), diff --git a/dataLayer/search.ts b/dataLayer/search.ts index 2edd0f4eef..06d58f4082 100644 --- a/dataLayer/search.ts +++ b/dataLayer/search.ts @@ -1,6 +1,6 @@ -'use strict'; +import { loadRuntimeModule } from '../utility/packageUtils.js'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import { transformReq } from '../utility/common_utils.ts'; export async function searchByConditions(searchObject: any) { @@ -30,11 +30,12 @@ export async function searchByValue(searchObject: any) { return array; } +let SelectValidator; +let SQLSearch; export function search(statement: any, callback: any) { try { - const SelectValidator = - require('../sqlTranslator/SelectValidator').default || require('../sqlTranslator/SelectValidator'); - const SQLSearch = require('./SQLSearch').default || require('./SQLSearch'); + SelectValidator ??= loadRuntimeModule('sqlTranslator/SelectValidator').default; + SQLSearch ??= loadRuntimeModule('dataLayer/SQLSearch').default; let validator = new SelectValidator(statement); validator.validate(); diff --git a/dataLayer/transaction.ts b/dataLayer/transaction.ts deleted file mode 100644 index c2a23f91ed..0000000000 --- a/dataLayer/transaction.ts +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -const harperBridge = require('./harperBridge/harperBridge').default; - -/** - * This is wrapper for write transactions, ensuring that all reads and writes within the callback occur atomically - * @param schema - * @param table - * @param callback - * @returns {Promise} - */ -export function writeTransaction(schema: string, table: string, callback: any) { - return harperBridge.writeTransaction(schema, table, callback); -} diff --git a/dataLayer/update.ts b/dataLayer/update.ts index 974d5bfa6a..ba9e45cf4c 100644 --- a/dataLayer/update.ts +++ b/dataLayer/update.ts @@ -3,11 +3,12 @@ import * as search from './search.ts'; import * as globalSchema from '../utility/globalSchema.ts'; import logger from '../utility/logging/harper_logger.ts'; -import * as write from './insert.ts'; +import * as _write from './insert.ts'; +const write = _write; import clone from 'clone'; -import * as alasql from 'alasql'; +import alasql from 'alasql'; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; -import * as util from 'util'; +import * as util from 'node:util'; const pGetTableSchema = util.promisify(globalSchema.getTableSchema); const pSearch = util.promisify(search.search); diff --git a/index.ts b/index.ts index d9177585c0..e0b8fd2a67 100644 --- a/index.ts +++ b/index.ts @@ -135,18 +135,8 @@ export type { DirectoryEntryEvent, } from './components/EntryHandler.ts'; -// Globals and values -// This section is responsible for creating the CJS exports map (for static analysis) -// as well as defining the globals and values exports. -// The stuff exported here are actually functional pieces of code. -// Importantly, do not import any values directly. -// For example, `import { tables } from './resources/databases.ts';` is NOT OKAY! -// This breaks Harper's dynamic runtime assignment of exports -// You MUST import as a type and then use `export declare const` instead. -// This results in the types being written to dist/index.d.ts, but not dist/index.js - -// And for my sanity please keep these alphabetically sorted so we can ensure nothing is missing. - +import runtimeGlobals from './globals.js'; +import './server/threads/threadServer.ts'; import type { contentTypes as ContentTypesImport } from './server/serverHelpers/contentTypes.ts'; import type { createBlob as CreateBlobImport } from './resources/blob.ts'; import type { databases as DatabasesImport } from './resources/databases.ts'; @@ -157,12 +147,15 @@ import type { Resource as ResourceImport } from './resources/Resource.ts'; import type { SecretsView as SecretsImport } from './components/componentSecrets.ts'; // per-component secrets view (#1550) import type { server as ServerImport } from './server/Server.ts'; import type { tables as TablesImport } from './resources/databases.ts'; -type ThreadsImport = unknown[]; // TODO: figure out actual type for this +type ThreadsImport = unknown[] & { + sendToThread?: (threadId: number, message: any) => boolean; + onMessageByType?: (type: string, listener: (...args: any[]) => any) => void; +}; import type { transaction as TransactionImport } from './resources/transaction.ts'; // These names are exposed TWO ways that resolve to the SAME live, process-wide value: // 1. as ambient globals (the `declare global` block below), and -// 2. as named exports of the `harper` package (the `export declare const` block below). +// 2. as named exports of the `harper` package (the `export const` block below). // At runtime each is populated in place by `_assignPackageExport(name, value)` (see globals.js), // which assigns BOTH `global[name]` and `exports[name]` to the one shared instance. So // `tables`/`databases`/etc. are not per-module or per-compartment copies: the bare global `tables` @@ -185,37 +178,18 @@ declare global { const transaction: typeof TransactionImport; } -// Declare constant types so these are defined in `index.d.ts` -export declare const contentTypes: typeof ContentTypesImport; -export declare const createBlob: typeof CreateBlobImport; -export declare const databases: typeof DatabasesImport; -export declare const logger: Logger; -export declare const models: typeof ModelsImport; -export declare const operation: typeof OperationImport; -export declare const Resource: typeof ResourceImport; -export declare const secrets: SecretsImport; -export declare const server: typeof ServerImport; -export declare const tables: typeof TablesImport; -export declare const threads: ThreadsImport; -export declare const transaction: typeof TransactionImport; - -// Actual define the values on the `exports` for CJS static analysis -exports.contentTypes = null; -exports.createBlob = undefined; -exports.databases = {}; -exports.logger = {}; -exports.models = undefined; -exports.operation = undefined; -exports.Resource = undefined; -exports.secrets = undefined; -exports.server = {}; -exports.tables = {}; -exports.threads = []; -exports.transaction = undefined; - -// And finally assign globals to exports. -// These values are populated at runtime by `_assignPackageExport()` in their respective modules -// (e.g. Resource.ts, databases.ts, Server.ts, etc.) -import { globals } from './server/threads/threadServer.js'; +// The CommonJS default is the live object; its ESM namespace named properties are early snapshots. +export const contentTypes = runtimeGlobals.contentTypes as typeof ContentTypesImport; +export const createBlob = runtimeGlobals.createBlob as typeof CreateBlobImport; +export const databases = runtimeGlobals.databases as typeof DatabasesImport; +export const logger = runtimeGlobals.logger as Logger; +export const models = runtimeGlobals.models as typeof ModelsImport; +export const operation = runtimeGlobals.operation as typeof OperationImport; +export const Resource = runtimeGlobals.Resource as typeof ResourceImport; +export const secrets = runtimeGlobals.secrets as SecretsImport; +export const server = runtimeGlobals.server as typeof ServerImport; +export const tables = runtimeGlobals.tables as typeof TablesImport; +export const threads = runtimeGlobals.threads as ThreadsImport; +export const transaction = runtimeGlobals.transaction as typeof TransactionImport; -Object.assign(exports, globals); +if (typeof exports !== 'undefined') Object.assign(exports, runtimeGlobals); diff --git a/integrationTests/database/blob-commit-over-time.test.ts b/integrationTests/database/blob-commit-over-time.test.ts index 3a26a378fe..b471bd9738 100644 --- a/integrationTests/database/blob-commit-over-time.test.ts +++ b/integrationTests/database/blob-commit-over-time.test.ts @@ -28,66 +28,72 @@ const CHUNKS = 20; const CHUNK_SIZE = 65536; const CHUNK_DELAY_MS = 100; -suite('Blob save that outruns the open-transaction limit (#2062)', (ctx: ContextWithHarper) => { - let httpURL: string; - let auth: string; +for (const mode of ['compiled', 'typestrip']) { + suite(`Blob save that outruns the open-transaction limit (#2062, ${mode})`, (ctx: ContextWithHarper) => { + let httpURL: string; + let auth: string; - before(async () => { - await setupHarperWithFixture(ctx, FIXTURE_PATH, { - config: { storage: { maxTransactionOpenTime: MAX_TXN_OPEN_MS }, logging: { level: 'warn' } }, - env: {}, + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: { storage: { maxTransactionOpenTime: MAX_TXN_OPEN_MS }, logging: { level: 'warn' } }, + env: mode === 'typestrip' ? { NODE_OPTIONS: '--conditions=typestrip' } : {}, + harperBinPath: resolve( + import.meta.dirname, + `../../${mode === 'typestrip' ? 'bin/harper.ts' : 'dist/bin/harper.js'}` + ), + }); + httpURL = ctx.harper.httpURL; + auth = 'Basic ' + Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64'); + // Wait for the component's routes to register on the workers. + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const probe = await fetch(`${httpURL}/Doc/__seed__`, { headers: { Authorization: auth } }); + if (probe.status < 500) break; + await sleep(250); + } }); - httpURL = ctx.harper.httpURL; - auth = 'Basic ' + Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64'); - // Wait for the component's routes to register on the workers. - const deadline = Date.now() + 30_000; - while (Date.now() < deadline) { - const probe = await fetch(`${httpURL}/Doc/__seed__`, { headers: { Authorization: auth } }); - if (probe.status < 500) break; - await sleep(250); - } - }); - after(async () => { - await teardownHarper(ctx); - }); + after(async () => { + await teardownHarper(ctx); + }); - function harperLog(): string { - const logDir = (ctx.harper as any).logDir as string | undefined; - const path = logDir && join(logDir, 'hdb.log'); - return path && existsSync(path) ? readFileSync(path, 'utf8') : ''; - } + function harperLog(): string { + const logDir = (ctx.harper as any).logDir as string | undefined; + const path = logDir && join(logDir, 'hdb.log'); + return path && existsSync(path) ? readFileSync(path, 'utf8') : ''; + } - test('the write commits and its blob survives', async () => { - const response = await fetch(`${httpURL}/SlowBlobWrite/`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'Authorization': auth }, - body: JSON.stringify({ id: 'slow-blob', chunks: CHUNKS, chunkSize: CHUNK_SIZE, chunkDelay: CHUNK_DELAY_MS }), - }); - strictEqual( - response.status, - 200, - `slow blob write should commit, got ${response.status}: ${await response.text()}` - ); + test('the write commits and its blob survives', async () => { + const response = await fetch(`${httpURL}/SlowBlobWrite/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': auth }, + body: JSON.stringify({ id: 'slow-blob', chunks: CHUNKS, chunkSize: CHUNK_SIZE, chunkDelay: CHUNK_DELAY_MS }), + }); + strictEqual( + response.status, + 200, + `slow blob write should commit, got ${response.status}: ${await response.text()}` + ); - const stored = await fetch(`${httpURL}/Doc/slow-blob`, { headers: { Authorization: auth } }); - strictEqual(stored.status, 200, 'the record must have been committed'); - strictEqual((await stored.json()).size, CHUNKS * CHUNK_SIZE); + const stored = await fetch(`${httpURL}/Doc/slow-blob`, { headers: { Authorization: auth } }); + strictEqual(stored.status, 200, 'the record must have been committed'); + strictEqual((await stored.json()).size, CHUNKS * CHUNK_SIZE); - // The blob file itself — what the abort used to unlink out from under the committed record. - const blob = await fetch(`${httpURL}/ReadBlob/slow-blob`, { headers: { Authorization: auth } }); - strictEqual(blob.status, 200, 'the committed record must not reference a destroyed blob'); - strictEqual((await blob.json()).blobBytes, CHUNKS * CHUNK_SIZE); + // The blob file itself — what the abort used to unlink out from under the committed record. + const blob = await fetch(`${httpURL}/ReadBlob/slow-blob`, { headers: { Authorization: auth } }); + strictEqual(blob.status, 200, 'the committed record must not reference a destroyed blob'); + strictEqual((await blob.json()).blobBytes, CHUNKS * CHUNK_SIZE); - const log = harperLog(); - ok( - !/has been aborted after exceeding the open-transaction limit/.test(log), - 'a transaction in its commit phase must not be poisoned by the monitor' - ); - // Confirms the window was actually crossed rather than the test passing vacuously. - ok( - /in its commit phase past the open-transaction limit/.test(log), - 'expected the monitor to observe (and spare) the over-limit commit' - ); + const log = harperLog(); + ok( + !/has been aborted after exceeding the open-transaction limit/.test(log), + 'a transaction in its commit phase must not be poisoned by the monitor' + ); + // Confirms the window was actually crossed rather than the test passing vacuously. + ok( + /in its commit phase past the open-transaction limit/.test(log), + 'expected the monitor to observe (and spare) the over-limit commit' + ); + }); }); -}); +} diff --git a/integrationTests/security/operation-denial-log.test.ts b/integrationTests/security/operation-denial-log.test.ts index 596665107e..352277898e 100644 --- a/integrationTests/security/operation-denial-log.test.ts +++ b/integrationTests/security/operation-denial-log.test.ts @@ -8,7 +8,7 @@ */ import { suite, test, before, after } from 'node:test'; import { deepStrictEqual, ok, strictEqual } from 'node:assert'; -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import { existsSync, readFileSync } from 'node:fs'; import { setTimeout as sleep } from 'node:timers/promises'; @@ -25,145 +25,163 @@ const OP_AUTH_PERMS_ERROR = 'This operation is not authorized due to role restri const notInOperations = (operation: string) => `Operation '${operation}' is not permitted for this role's operations configuration`; -suite('an operation refused by the permission check', (ctx: ContextWithHarper) => { - let client: ReturnType; - let userHeaders: Record; - let logPath: string; +for (const mode of ['compiled', 'typestrip']) { + suite(`an operation refused by the permission check (${mode})`, (ctx: ContextWithHarper) => { + let client: ReturnType; + let userHeaders: Record; + let logPath: string; - function readLog(): string { - return existsSync(logPath) ? readFileSync(logPath, 'utf8') : ''; - } + function readLog(): string { + return existsSync(logPath) ? readFileSync(logPath, 'utf8') : ''; + } - before(async () => { - await startHarper(ctx, { config: {}, env: {} }); - client = createApiClient(ctx.harper); - userHeaders = createHeaders(USER.username, USER.password); - logPath = ctx.harper.logDir ? join(ctx.harper.logDir, 'hdb.log') : join(ctx.harper.dataRootDir, 'log', 'hdb.log'); + before(async () => { + await startHarper(ctx, { + config: {}, + env: mode === 'typestrip' ? { NODE_OPTIONS: '--conditions=typestrip' } : {}, + startupMaxMs: 60000, + harperBinPath: resolve( + import.meta.dirname, + `../../${mode === 'typestrip' ? 'bin/harper.ts' : 'dist/bin/harper.js'}` + ), + }); + client = createApiClient(ctx.harper); + userHeaders = createHeaders(USER.username, USER.password); + logPath = ctx.harper.logDir ? join(ctx.harper.logDir, 'hdb.log') : join(ctx.harper.dataRootDir, 'log', 'hdb.log'); - await client - .req() - .send({ operation: 'add_role', role: ROLE, permission: { super_user: false, operations: ['user_info'] } }) - .expect(200); - await client - .req() - .send({ operation: 'add_user', role: ROLE, username: USER.username, password: USER.password, active: true }) - .expect(200); + await client + .req() + .send({ operation: 'add_role', role: ROLE, permission: { super_user: false, operations: ['user_info'] } }) + .expect(200); + await client + .req() + .send({ operation: 'add_user', role: ROLE, username: USER.username, password: USER.password, active: true }) + .expect(200); - await client - .req() - .send({ operation: 'create_table', database: 'data', table: TABLE, primary_key: 'id' }) - .expect(200); - await client - .req() - .send({ operation: 'insert', database: 'data', table: TABLE, records: [{ id: 1, name: 'Harper' }] }) - .expect(200); - await client - .req() - .send({ - operation: 'add_role', - role: LOADER_ROLE, - permission: { - super_user: false, - data: { - tables: { - [TABLE]: { - read: true, - insert: true, - update: true, - delete: false, - attribute_permissions: [ - { attribute_name: 'id', read: true, insert: true, update: true }, - { attribute_name: 'name', read: true, insert: false, update: false }, - ], + await client + .req() + .send({ operation: 'create_table', database: 'data', table: TABLE, primary_key: 'id' }) + .expect(200); + await client + .req() + .send({ operation: 'insert', database: 'data', table: TABLE, records: [{ id: 1, name: 'Harper' }] }) + .expect(200); + await client + .req() + .send({ + operation: 'add_role', + role: LOADER_ROLE, + permission: { + super_user: false, + data: { + tables: { + [TABLE]: { + read: true, + insert: true, + update: true, + delete: false, + attribute_permissions: [ + { attribute_name: 'id', read: true, insert: true, update: true }, + { attribute_name: 'name', read: true, insert: false, update: false }, + ], + }, }, }, }, - }, - }) - .expect(200); - await client - .req() - .send({ - operation: 'add_user', - role: LOADER_ROLE, - username: LOADER.username, - password: LOADER.password, - active: true, - }) - .expect(200); - }); + }) + .expect(200); + await client + .req() + .send({ + operation: 'add_user', + role: LOADER_ROLE, + username: LOADER.username, + password: LOADER.password, + active: true, + }) + .expect(200); + }); - after(async () => { - await teardownHarper(ctx); - }); + after(async () => { + await teardownHarper(ctx); + }); - test('answers 403 with the permission report as the body', async () => { - const response = await client.reqAs(userHeaders).send({ operation: 'list_users' }); - strictEqual(response.status, 403, response.text); - strictEqual( - response.text, - JSON.stringify({ - error: OP_AUTH_PERMS_ERROR, - unauthorized_access: [notInOperations('list_users')], - invalid_schema_items: [], - }) - ); - }); + test('answers 403 with the permission report as the body', async () => { + const response = await client.reqAs(userHeaders).send({ operation: 'list_users' }); + strictEqual(response.status, 403, response.text); + strictEqual( + response.text, + JSON.stringify({ + error: OP_AUTH_PERMS_ERROR, + unauthorized_access: [notInOperations('list_users')], + invalid_schema_items: [], + }) + ); + }); - test('is logged once, with the reason it was refused', async () => { - const offset = readLog().length; - const refused = await client - .reqAs(userHeaders) - .send({ operation: 'add_user', role: ROLE, username: 'denial_log_other', password: 'Other-pw-1!', active: true }); - strictEqual(refused.status, 403, JSON.stringify(refused.body)); - // A refusal logs every line before its response is sent, and all of them go through the main - // thread's log in order, so a second refusal made after this one answered marks where the - // first one's lines end. - const marker = await client.reqAs(userHeaders).send({ operation: 'list_users' }); - strictEqual(marker.status, 403, JSON.stringify(marker.body)); + test('is logged once, with the reason it was refused', async () => { + const offset = readLog().length; + const refused = await client.reqAs(userHeaders).send({ + operation: 'add_user', + role: ROLE, + username: 'denial_log_other', + password: 'Other-pw-1!', + active: true, + }); + strictEqual(refused.status, 403, JSON.stringify(refused.body)); + // A refusal logs every line before its response is sent, and all of them go through the main + // thread's log in order, so a second refusal made after this one answered marks where the + // first one's lines end. + const marker = await client.reqAs(userHeaders).send({ operation: 'list_users' }); + strictEqual(marker.status, 403, JSON.stringify(marker.body)); - const markerLine = '403 from operation list_users'; - const deadline = Date.now() + 15_000; - let written = readLog().slice(offset); - while (!written.includes(markerLine) && Date.now() < deadline) { - await sleep(100); - written = readLog().slice(offset); - } - ok(written.includes(markerLine), `the marker refusal never reached ${logPath}:\n${written}`); + const markerLine = '403 from operation list_users'; + const deadline = Date.now() + 15_000; + let written = readLog().slice(offset); + while (!written.includes(markerLine) && Date.now() < deadline) { + await sleep(100); + written = readLog().slice(offset); + } + ok(written.includes(markerLine), `the marker refusal never reached ${logPath}:\n${written}`); - const firstRefusal = written.slice(0, written.indexOf(markerLine)); - const reasonLines = firstRefusal.split('\n').filter((line) => line.includes(notInOperations('add_user'))); - strictEqual(reasonLines.length, 1, `expected the refusal reason logged once:\n${firstRefusal}`); - ok(!written.includes('[object Object]'), `a refusal was logged as [object Object]:\n${written}`); - }); + const firstRefusal = written.slice(0, written.indexOf(markerLine)); + const reasonLines = firstRefusal.split('\n').filter((line) => line.includes(notInOperations('add_user'))); + strictEqual(reasonLines.length, 1, `expected the refusal reason logged once:\n${firstRefusal}`); + ok(!written.includes('[object Object]'), `a refusal was logged as [object Object]:\n${written}`); + }); - // Refused in the job worker, after the request itself was accepted. - test('a bulk load refused on an attribute permission keeps the report as its job message', async () => { - const started = await client - .reqAs(createHeaders(LOADER.username, LOADER.password)) - .send({ operation: 'csv_data_load', action: 'insert', database: 'data', table: TABLE, data: 'id,name\n2,Rex\n' }); - strictEqual(started.status, 200, JSON.stringify(started.body)); + // Refused in the job worker, after the request itself was accepted. + test('a bulk load refused on an attribute permission keeps the report as its job message', async () => { + const started = await client.reqAs(createHeaders(LOADER.username, LOADER.password)).send({ + operation: 'csv_data_load', + action: 'insert', + database: 'data', + table: TABLE, + data: 'id,name\n2,Rex\n', + }); + strictEqual(started.status, 200, JSON.stringify(started.body)); - const deadline = Date.now() + 30_000; - let job: Record | undefined; - while (Date.now() < deadline) { - const response = await client.req().send({ operation: 'get_job', id: started.body.job_id }).expect(200); - job = response.body[0]; - if (job?.status === 'COMPLETE' || job?.status === 'ERROR') break; - await sleep(250); - } - strictEqual(job?.status, 'ERROR', JSON.stringify(job)); - deepStrictEqual(job.message, { - error: OP_AUTH_PERMS_ERROR, - unauthorized_access: [ - { - schema: 'data', - table: TABLE, - required_table_permissions: [], - required_attribute_permissions: [{ attribute_name: 'name', required_permissions: ['insert'] }], - }, - ], - invalid_schema_items: [], + const deadline = Date.now() + 30_000; + let job: Record | undefined; + while (Date.now() < deadline) { + const response = await client.req().send({ operation: 'get_job', id: started.body.job_id }).expect(200); + job = response.body[0]; + if (job?.status === 'COMPLETE' || job?.status === 'ERROR') break; + await sleep(250); + } + strictEqual(job?.status, 'ERROR', JSON.stringify(job)); + deepStrictEqual(job.message, { + error: OP_AUTH_PERMS_ERROR, + unauthorized_access: [ + { + schema: 'data', + table: TABLE, + required_table_permissions: [], + required_attribute_permissions: [{ attribute_name: 'name', required_permissions: ['insert'] }], + }, + ], + invalid_schema_items: [], + }); }); }); -}); +} diff --git a/integrationTests/server/operations-server.test.ts b/integrationTests/server/operations-server.test.ts index 4c9099b372..48caf40314 100644 --- a/integrationTests/server/operations-server.test.ts +++ b/integrationTests/server/operations-server.test.ts @@ -11,7 +11,6 @@ import { suite, test, before, after } from 'node:test'; import { ok, strictEqual } from 'node:assert'; import { pack, unpack } from 'msgpackr'; import { encode, decode } from 'cbor-x'; - import { startHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; suite('Operations Server', (ctx: ContextWithHarper) => { diff --git a/integrationTests/server/thread-heap-memory-recovery.test.ts b/integrationTests/server/thread-heap-memory-recovery.test.ts index 9294e7bf14..93613e1f78 100644 --- a/integrationTests/server/thread-heap-memory-recovery.test.ts +++ b/integrationTests/server/thread-heap-memory-recovery.test.ts @@ -16,8 +16,9 @@ import { suite, test, before, after } from 'node:test'; import { ok, strictEqual } from 'node:assert'; import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; +import { waitFor } from '../../unitTests/waitFor.js'; import { startHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; import { MIN_THREAD_HEAP_MEMORY_MB } from '../../utility/hdbTerms.ts'; @@ -68,26 +69,53 @@ suite('Harper boots on a threads.maxHeapMemory no worker thread could start on', // The accepted boundary is the one number the guard hinges on, so prove a worker actually completes // startup there rather than only that an isolate initializes. Measured floor for a bare worker is // between 48 (fails) and 56 (serves). -suite('Harper boots at the accepted threads.maxHeapMemory minimum', (ctx: ContextWithHarper) => { - before(async () => { - await startHarper(ctx, { - config: { threads: { count: 1, maxHeapMemory: MIN_THREAD_HEAP_MEMORY_MB } }, - env: {}, +for (const mode of ['compiled', 'typestrip']) { + suite(`Harper boots at the accepted threads.maxHeapMemory minimum (${mode})`, (ctx: ContextWithHarper) => { + before(async () => { + await startHarper(ctx, { + config: { threads: { count: 1, maxHeapMemory: MIN_THREAD_HEAP_MEMORY_MB } }, + env: mode === 'typestrip' ? { NODE_OPTIONS: '--conditions=typestrip' } : {}, + startupMaxMs: 60000, + harperBinPath: resolve( + import.meta.dirname, + `../../${mode === 'typestrip' ? 'bin/harper.ts' : 'dist/bin/harper.js'}` + ), + }); }); - }); - after(async () => { - await teardownHarper(ctx); - }); + after(async () => { + await teardownHarper(ctx); + }); - test('the operations API serves traffic on the configured minimum', async () => { - const res = await fetch(`${ctx.harper.operationsAPIURL}/health`); - strictEqual(res.status, 200); - }); + test('the operations API serves traffic on the configured minimum', async () => { + const res = await fetch(`${ctx.harper.operationsAPIURL}/health`); + strictEqual(res.status, 200); + const threads = await waitFor( + async () => { + const response = await fetch(ctx.harper.operationsAPIURL, { + method: 'POST', + headers: { + 'Authorization': `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ operation: 'system_information', attributes: ['threads'] }), + signal: AbortSignal.timeout(10000), + }); + strictEqual(response.status, 200); + const { threads } = await response.json(); + return threads.some((thread) => thread.heapUsed > 0) && threads; + }, + { timeout: 10000, interval: 100 } + ); + console.log( + `[${mode}] ready worker heap: ${JSON.stringify(threads.map(({ heapUsed, heapTotal }) => ({ heapUsed, heapTotal })))}` + ); + }); - test('the configured minimum is honored, not displaced', async () => { - const logDir = ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'); - const contents = await waitForLogMatch(logDir, RECOVERY_LOG, 2000); - ok(!RECOVERY_LOG.test(contents), 'a value at the minimum must not trigger recovery'); + test('the configured minimum is honored, not displaced', async () => { + const logDir = ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'); + const contents = await waitForLogMatch(logDir, RECOVERY_LOG, 2000); + ok(!RECOVERY_LOG.test(contents), 'a value at the minimum must not trigger recovery'); + }); }); -}); +} diff --git a/integrationTests/server/typestrip.test.ts b/integrationTests/server/typestrip.test.ts new file mode 100644 index 0000000000..11f7422ca3 --- /dev/null +++ b/integrationTests/server/typestrip.test.ts @@ -0,0 +1,386 @@ +/** + * PR #562: native TypeStrip CLI and worker startup serve real REST reads and writes, + * with the same compiled CommonJS behavior and configured worker restrictions. + */ +import { suite, test, before, after } from 'node:test'; +import { deepStrictEqual, strictEqual, ok } from 'node:assert'; +import { resolve, join } from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { once } from 'node:events'; +import WebSocket from 'ws'; +import { waitFor } from '../../unitTests/waitFor.js'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; + +for (const mode of ['compiled', 'typestrip']) { + suite(`REST workers (${mode})`, (ctx: ContextWithHarper) => { + before(async () => { + await setupHarperWithFixture(ctx, resolve(import.meta.dirname, 'typestrip'), { + config: { + threads: { count: 2 }, + authentication: { operationTokenTimeout: '2h', refreshTokenTimeout: '3h' }, + applications: { allowedBuiltinModules: ['worker_threads'] }, + }, + env: { + HARPER_SQL_ENGINE: 'legacy', + ...(mode === 'typestrip' ? { NODE_OPTIONS: '--conditions=typestrip' } : {}), + }, + harperBinPath: resolve( + import.meta.dirname, + `../../${mode === 'typestrip' ? 'bin/harper.ts' : 'dist/bin/harper.js'}` + ), + }); + }); + after(async () => { + await teardownHarper(ctx); + }); + + test('loads cold SQL for date searches, direct queries and export jobs, preserving authorization', async () => { + const headers = { + 'Authorization': `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`, + 'Content-Type': 'application/json', + }; + async function operation(body: object, authorization = headers.Authorization) { + const response = await fetch(ctx.harper.operationsAPIURL, { + method: 'POST', + headers: { ...headers, Authorization: authorization }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30000), + }); + return { status: response.status, body: await response.json() }; + } + const jobs = await operation({ + operation: 'search_jobs_by_start_date', + from_date: '2000-01-01', + to_date: '2100-01-01', + }); + strictEqual(jobs.status, 200, JSON.stringify(jobs.body)); + ok(Array.isArray(jobs.body)); + const sql = `SELECT username FROM system.hdb_user WHERE username = '${ctx.harper.admin.username.replaceAll("'", "''")}'`; + const query = await operation({ operation: 'sql', sql }); + strictEqual(query.status, 200, JSON.stringify(query.body)); + deepStrictEqual(query.body, [{ username: ctx.harper.admin.username }]); + const exported = await operation({ + operation: 'export_local', + path: ctx.harper.dataRootDir, + filename: 'cold-sql', + format: 'json', + search_operation: { operation: 'sql', sql }, + }); + strictEqual(exported.status, 200, JSON.stringify(exported.body)); + await waitFor( + async () => { + const result = await operation({ operation: 'get_job', id: exported.body.job_id }); + strictEqual(result.status, 200); + const job = result.body[0]; + if (job?.status !== 'COMPLETE' && job?.status !== 'ERROR') return false; + strictEqual(job.status, 'COMPLETE', JSON.stringify(job)); + return true; + }, + { timeout: 30000, interval: 100 } + ); + deepStrictEqual(JSON.parse(await readFile(join(ctx.harper.dataRootDir, 'cold-sql.json'), 'utf8')), query.body); + strictEqual( + ( + await operation({ + operation: 'add_role', + role: 'cold_sql_denied', + permission: { super_user: false, operations: ['user_info'] }, + }) + ).status, + 200 + ); + strictEqual( + ( + await operation({ + operation: 'add_user', + role: 'cold_sql_denied', + username: 'cold_sql_denied', + password: 'Cold-sql-pw-1!', + active: true, + }) + ).status, + 200 + ); + const denied = await operation( + { operation: 'sql', sql }, + `Basic ${Buffer.from('cold_sql_denied:Cold-sql-pw-1!').toString('base64')}` + ); + strictEqual(denied.status, 403, JSON.stringify(denied.body)); + ok(JSON.stringify(denied.body).includes("Operation 'sql' is not permitted")); + + const row = { id: 'sql-permissions', value: 'protected' }; + const inserted = await operation({ operation: 'insert', database: 'data', table: 'Probe', records: [row] }); + strictEqual(inserted.status, 200, JSON.stringify(inserted.body)); + for (const [role, tables] of [ + ['cold_sql_table_denied', {}], + [ + 'cold_sql_attribute_reader', + { + Probe: { + read: true, + insert: false, + update: false, + delete: false, + attribute_permissions: [ + { attribute_name: 'id', read: true, insert: false, update: false }, + { attribute_name: 'value', read: false, insert: false, update: false }, + ], + }, + }, + ], + ] as const) { + const addedRole = await operation({ + operation: 'add_role', + role, + permission: { super_user: false, operations: ['sql'], data: { tables } }, + }); + strictEqual(addedRole.status, 200, JSON.stringify(addedRole.body)); + const addedUser = await operation({ + operation: 'add_user', + role, + username: role, + password: 'Cold-sql-pw-1!', + active: true, + }); + strictEqual(addedUser.status, 200, JSON.stringify(addedUser.body)); + } + const probeSql = "SELECT * FROM data.Probe WHERE id = 'sql-permissions'"; + const tableDenied = await operation( + { operation: 'sql', sql: probeSql }, + `Basic ${Buffer.from('cold_sql_table_denied:Cold-sql-pw-1!').toString('base64')}` + ); + strictEqual(tableDenied.status, 403, JSON.stringify(tableDenied.body)); + ok(!JSON.stringify(tableDenied.body).includes("Operation 'sql' is not permitted")); + const filtered = await operation( + { operation: 'sql', sql: probeSql }, + `Basic ${Buffer.from('cold_sql_attribute_reader:Cold-sql-pw-1!').toString('base64')}` + ); + strictEqual(filtered.status, 200, JSON.stringify(filtered.body)); + deepStrictEqual(filtered.body, [{ id: row.id }]); + }); + + test('serves requests in workers and persists a REST record', async () => { + const headers = { + 'Authorization': `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`, + 'Content-Type': 'application/json', + }; + const request = (path: string, options = {}) => + fetch(`${ctx.harper.httpURL}${path}`, { + ...options, + headers, + signal: AbortSignal.timeout(10000), + }); + const runtime = await request('/Runtime/'); + strictEqual(runtime.status, 200); + const info = await runtime.json(); + strictEqual(info.isMainThread, false); + ok(info.threadId > 0); + ok(info.workerIndex === 0 || info.workerIndex === 1); + strictEqual(info.noServerStart, false); + const record = { id: 'source-execution', value: mode }; + const write = await request('/Probe/source-execution', { method: 'PUT', body: JSON.stringify(record) }); + strictEqual(write.status, 204); + const read = await request('/Probe/source-execution'); + strictEqual(read.status, 200); + deepStrictEqual(await read.json(), record); + }); + + test('uses the configured token lifetimes', async () => { + const response = await fetch(ctx.harper.operationsAPIURL, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + operation: 'create_authentication_tokens', + username: ctx.harper.admin.username, + password: ctx.harper.admin.password, + }), + signal: AbortSignal.timeout(10000), + }); + strictEqual(response.status, 200); + const tokens = await response.json(); + for (const [name, seconds] of [ + ['operation_token', 7200], + ['refresh_token', 10800], + ] as const) { + const payload = JSON.parse(Buffer.from(tokens[name].split('.')[1], 'base64url').toString()); + strictEqual(payload.exp - payload.iat, seconds); + } + }); + + test('enforces the configured builtin allowlist in application code', async () => { + const response = await fetch(`${ctx.harper.httpURL}/BuiltinCheck/`, { + headers: { + Authorization: `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`, + }, + signal: AbortSignal.timeout(10000), + }); + strictEqual(response.status, 500); + ok((await response.text()).includes('Module node:fs is not allowed')); + }); + }); +} + +for (const mode of ['compiled', 'typestrip']) { + for (const bigInt of [false, true]) { + suite(`Configured JSON workers (${mode}, bigInt=${bigInt})`, (ctx: ContextWithHarper) => { + before(async () => { + await setupHarperWithFixture(ctx, resolve(import.meta.dirname, 'typestrip'), { + config: { + threads: { count: 1 }, + serialization: { bigInt }, + applications: { allowedBuiltinModules: ['worker_threads'] }, + }, + env: mode === 'typestrip' ? { NODE_OPTIONS: '--conditions=typestrip' } : {}, + harperBinPath: resolve( + import.meta.dirname, + `../../${mode === 'typestrip' ? 'bin/harper.ts' : 'dist/bin/harper.js'}` + ), + }); + }); + after(() => teardownHarper(ctx)); + + test('serves the configured BigInt behavior and serializes a live subscription', async () => { + const headers = { + 'Authorization': `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`, + 'Content-Type': 'application/json', + }; + const response = await fetch(`${ctx.harper.httpURL}/BigNumber/`, { + headers, + signal: AbortSignal.timeout(10000), + }); + strictEqual(response.status, bigInt ? 200 : 500); + if (bigInt) strictEqual(await response.text(), '{"value":9007199254740993}'); + else ok((await response.text()).includes('Cannot serialize BigInt')); + const id = 'configured-codec'; + const record = { id, value: `${mode}:${bigInt}` }; + const initial = await fetch(`${ctx.harper.httpURL}/Probe/${id}`, { + method: 'PUT', + headers, + body: JSON.stringify({ id, value: 'initial' }), + signal: AbortSignal.timeout(10000), + }); + strictEqual(initial.status, 204); + const ws = new WebSocket(`${ctx.harper.httpURL.replace(/^http/, 'ws')}/Probe/${id}`, { headers }); + const messages: any[] = []; + ws.on('message', (data) => messages.push(JSON.parse(data.toString()))); + try { + await once(ws, 'open', { signal: AbortSignal.timeout(10000) }); + await waitFor(() => messages.some((message) => message.value?.value === 'initial'), { timeout: 10000 }); + const write = await fetch(`${ctx.harper.httpURL}/Probe/${id}`, { + method: 'PUT', + headers, + body: JSON.stringify(record), + signal: AbortSignal.timeout(10000), + }); + strictEqual(write.status, 204); + await waitFor(() => messages.some((message) => message.value?.value === record.value), { timeout: 10000 }); + const read = await fetch(`${ctx.harper.httpURL}/Probe/${id}`, { + headers, + signal: AbortSignal.timeout(10000), + }); + strictEqual(read.status, 200); + deepStrictEqual(await read.json(), record); + } finally { + ws.terminate(); + } + }); + }); + } +} + +for (const mode of ['compiled', 'typestrip']) { + suite(`fresh-install main-thread configuration (${mode})`, (ctx: ContextWithHarper) => { + const username = `fresh_cache_${mode}`; + const sentinel = `fresh_cache_sentinel_${mode}`; + let logPath: string; + const authorization = (user: string, password: string) => + `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; + const operation = async (body: object, credentials?: string) => + fetch(ctx.harper.operationsAPIURL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': credentials ?? authorization(ctx.harper.admin.username, ctx.harper.admin.password), + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(10000), + }); + const logEntries = async () => + (await readFile(logPath, 'utf8')) + .split(/(?=^\d{4}-\d{2}-\d{2}T)/m) + .filter((entry) => entry.includes('[auth-event]')); + + before(async () => { + await setupHarperWithFixture(ctx, resolve(import.meta.dirname, 'typestrip'), { + config: { + threads: { count: 0 }, + authentication: { cacheTTL: 31337, authorizeLocal: false }, + applications: { allowedBuiltinModules: ['worker_threads'] }, + logging: { auditAuthEvents: { logSuccessful: true, logFailed: true } }, + }, + env: { + DEV_MODE: '', + HARPER_STORAGE_ENGINE: 'rocksdb', + NODE_OPTIONS: + mode === 'typestrip' ? '--experimental-vm-modules --conditions=typestrip' : '--experimental-vm-modules', + }, + harperBinPath: resolve( + import.meta.dirname, + `../../${mode === 'typestrip' ? 'bin/harper.ts' : 'dist/bin/harper.js'}` + ), + startupMaxMs: 60000, + }); + logPath = join(ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'), 'hdb.log'); + const created = await operation({ + operation: 'add_user', + role: 'super_user', + username, + password: 'Fresh-cache-pw-1!', + active: true, + }); + strictEqual(created.status, 200, await created.text()); + }); + after(() => teardownHarper(ctx)); + + test('serves application traffic on the main thread', async () => { + const response = await fetch(`${ctx.harper.httpURL}/Runtime/`, { + headers: { Authorization: authorization(ctx.harper.admin.username, ctx.harper.admin.password) }, + signal: AbortSignal.timeout(10000), + }); + strictEqual(response.status, 200); + const runtime = await response.json(); + strictEqual(runtime.isMainThread, true); + strictEqual(runtime.threadId, 0); + }); + + test('enforces the configured builtin list after an in-process first install', async () => { + const response = await fetch(`${ctx.harper.httpURL}/BuiltinCheck/`, { + headers: { Authorization: authorization(ctx.harper.admin.username, ctx.harper.admin.password) }, + signal: AbortSignal.timeout(10000), + }); + strictEqual(response.status, 500); + ok((await response.text()).includes('Module node:fs is not allowed')); + }); + + test('reuses Basic credentials across requests within the configured cache window', async () => { + for (let index = 0; index < 4; index++) { + const response = await operation({ operation: 'user_info' }, authorization(username, 'Fresh-cache-pw-1!')); + strictEqual(response.status, 200, await response.text()); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + const failed = await operation({ operation: 'user_info' }, authorization(sentinel, 'wrong')); + strictEqual(failed.status, 401); + await waitFor( + async () => + (await logEntries()).some( + (entry) => entry.includes(`username: '${sentinel}'`) && entry.includes("status: 'failure'") + ), + { timeout: 10000, interval: 50 } + ); + const successes = (await logEntries()).filter( + (entry) => entry.includes(`username: '${username}'`) && entry.includes("status: 'success'") + ); + strictEqual(successes.length, 1, successes.join('\n')); + }); + }); +} diff --git a/integrationTests/server/typestrip/config.yaml b/integrationTests/server/typestrip/config.yaml new file mode 100644 index 0000000000..cb418f6fc0 --- /dev/null +++ b/integrationTests/server/typestrip/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.ts +rest: true diff --git a/integrationTests/server/typestrip/resources.ts b/integrationTests/server/typestrip/resources.ts new file mode 100644 index 0000000000..86ebecded6 --- /dev/null +++ b/integrationTests/server/typestrip/resources.ts @@ -0,0 +1,20 @@ +import { isMainThread, threadId, workerData } from 'node:worker_threads'; + +export class Runtime extends Resource { + get() { + return { isMainThread, threadId, workerIndex: workerData?.workerIndex, noServerStart: !!workerData?.noServerStart }; + } +} + +export class BigNumber extends Resource { + get() { + return { value: 9007199254740993n }; + } +} + +export class BuiltinCheck extends Resource { + async get() { + const fs = await import('node:fs'); + return typeof fs.readFile; + } +} diff --git a/integrationTests/server/typestrip/schema.graphql b/integrationTests/server/typestrip/schema.graphql new file mode 100644 index 0000000000..2c6044c026 --- /dev/null +++ b/integrationTests/server/typestrip/schema.graphql @@ -0,0 +1,4 @@ +type Probe @table @export { + id: ID @primaryKey + value: String +} diff --git a/launchServiceScripts/launchHarperDB.js b/launchServiceScripts/launchHarperDB.js index 7bc8392550..d5d1e75e20 100644 --- a/launchServiceScripts/launchHarperDB.js +++ b/launchServiceScripts/launchHarperDB.js @@ -1,3 +1,3 @@ 'use strict'; -require('../server/operationsServer.ts').hdbServer(); +require('../utility/packageUtils.js').loadRuntimeModule('launchServiceScripts/startHarperDB'); diff --git a/launchServiceScripts/startHarperDB.ts b/launchServiceScripts/startHarperDB.ts new file mode 100644 index 0000000000..f019521d6c --- /dev/null +++ b/launchServiceScripts/startHarperDB.ts @@ -0,0 +1,16 @@ +import { hdbServer } from '../server/operationsServer.ts'; +import * as env from '../utility/environment/environmentManager.ts'; +import { runStartup } from '../utility/lifecycle.ts'; +import logger from '../utility/logging/harper_logger.ts'; +import { realExit } from '../server/threads/workerProcessGuard.ts'; + +async function launch() { + env.initSync(); + await runStartup(); + await hdbServer(); +} + +launch().catch((error) => { + logger.fatal('Failed to start the operations server', error); + realExit(1); +}); diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index b0712a827c..c1eb09c45d 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -13,6 +13,7 @@ import * as harperLogger from '../utility/logging/harper_logger.ts'; import type { Context, Id } from './ResourceInterface.ts'; import * as envMngr from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { settleBeforeDeadline, when } from '../utility/when.ts'; import { setTimeout as delay } from 'node:timers/promises'; @@ -44,7 +45,7 @@ const ONCE = Object.freeze({ once: true }); // chain root — so a chain child can never become its own timeout root (issue #2231). const supervisedWriteRoots = new Set(); let MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them -const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); +let DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); export const TRANSACTION_STATE = { CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads) OPEN: 1, // the transaction is open and can be used for reads and writes @@ -3363,6 +3364,14 @@ function startMonitoringTxns() { startMonitoringTxns(); +// The ESM graph can load this module before configuration. Refresh the cached +// limits and re-arm the monitor before startup begins serving requests. +onStartup(() => { + MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; + DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); + setTxnExpiration(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONOPENTIME) ?? 30000); +}); + /** * Test seam: re-arms the once-per-process replay warning. The whole unit suite shares one process, * so whichever test first drives a commit under open iterators consumes the warning for every test diff --git a/resources/ErrorResource.ts b/resources/ErrorResource.ts index 07b86001fe..fe488780ba 100644 --- a/resources/ErrorResource.ts +++ b/resources/ErrorResource.ts @@ -1,58 +1,82 @@ import { Resource } from './Resource.ts'; import type { Context } from './ResourceInterface.ts'; + /** * ErrorResource is a Resource that throws an error on any request, communicating to the client when attempts are made * to access endpoints/resources that had an internal error in their configuration or setup. This helps ensure that * if there is a problem with a resource, it is immediately apparent and can be fixed. + * + * Construct lazily because Resource is uninitialized while its ESM cycle evaluates. */ -export class ErrorResource extends Resource { - error: Error; - constructor(error: Error) { - super(null as any, null); - this.error = error; - } - isError = true; - allowRead(): never { - throw this.error; - } - allowUpdate(): never { - throw this.error; - } - allowCreate(): never { - throw this.error; - } - allowDelete(): never { - throw this.error; - } - getId(): never { - throw this.error; - } - getContext(): Context { - throw this.error; - } - get(): never { - throw this.error; - } - post(): never { - throw this.error; - } - put(): never { - throw this.error; - } - delete(): never { - throw this.error; - } - connect(): never { - throw this.error; - } - getResource() { - // all child paths resolve back to reporting this error - return this; - } - publish(): never { - throw this.error; - } - subscribe(): never { - throw this.error; +let _ErrorResource: any; +function getErrorResource(): any { + if (!_ErrorResource) { + _ErrorResource = class ErrorResource extends Resource { + error: Error; + constructor(error: Error) { + super(null as any, null); + this.error = error; + } + isError = true; + allowRead(): never { + throw this.error; + } + allowUpdate(): never { + throw this.error; + } + allowCreate(): never { + throw this.error; + } + allowDelete(): never { + throw this.error; + } + getId(): never { + throw this.error; + } + getContext(): Context { + throw this.error; + } + get(): never { + throw this.error; + } + post(): never { + throw this.error; + } + put(): never { + throw this.error; + } + delete(): never { + throw this.error; + } + connect(): never { + throw this.error; + } + getResource() { + // all child paths resolve back to reporting this error + return this; + } + publish(): never { + throw this.error; + } + subscribe(): never { + throw this.error; + } + }; } + return _ErrorResource; } + +export const ErrorResource: any = new Proxy(function () {} as any, { + construct(_target, args, newTarget) { + return Reflect.construct(getErrorResource(), args, newTarget); + }, + get(_target, prop) { + return getErrorResource()[prop]; + }, + has(_target, prop) { + return prop in getErrorResource(); + }, + getPrototypeOf() { + return Object.getPrototypeOf(getErrorResource()); + }, +}); diff --git a/resources/LMDBTransaction.ts b/resources/LMDBTransaction.ts index ef800cc98d..73eef7a5ec 100644 --- a/resources/LMDBTransaction.ts +++ b/resources/LMDBTransaction.ts @@ -10,7 +10,7 @@ import { type CommitOptions, type TransactionWrite, type CommitResolution, -} from './DatabaseTransaction'; +} from './DatabaseTransaction.ts'; import { cleanupUnusedBlobs, collectRetainedFileIds } from './blob.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index e261b5f343..970739459b 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -33,7 +33,7 @@ import { recordAction } from './analytics/write.ts'; import { constants, RocksDatabase } from '@harperfast/rocksdb-js'; import { when } from '../utility/when.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; -import * as envMngr from '../utility/environment/environmentManager.js'; +import * as envMngr from '../utility/environment/environmentManager.ts'; const StructonEncoder = createStructon(Encoder) as typeof Encoder; diff --git a/resources/Resource.ts b/resources/Resource.ts index 720d3028c4..eeb38d2ee5 100644 --- a/resources/Resource.ts +++ b/resources/Resource.ts @@ -1,15 +1,16 @@ +import * as userRuntimeModule from '../security/user.ts'; import type { User } from '../security/user.ts'; import type { RecordObject } from './RecordEncoder.ts'; import { - ResourceInterface, - SubscriptionRequest, - Id, - Context, - Query, - SourceContext, - RequestTargetOrId, + type ResourceInterface, + type SubscriptionRequest, + type Id, + type Context, + type Query, + type SourceContext, + type RequestTargetOrId, } from './ResourceInterface.ts'; -import { randomUUID } from 'crypto'; +import { randomUUID } from 'node:crypto'; import { DatabaseTransaction, isJoinableScope, @@ -1062,7 +1063,7 @@ function registerLiveSubscriptionForContext( if (!username) return subscription; // no subscription, or one already closed (a refused resume): nothing will be delivered to revoke if (!subscription || typeof subscription !== 'object' || subscription.closed) return subscription; - securityUser ??= require('../security/user'); + securityUser ??= userRuntimeModule; const { cloneUserView, hasRecordProvenance, isCurrentUser } = securityUser!; // A select written by authorization names no relationship, so the default allowRead's result still // reads only the role's table permission; a caller's select can reach related tables' allowRead. diff --git a/resources/Resources.ts b/resources/Resources.ts index 8ffcbd6032..4bcdd737db 100644 --- a/resources/Resources.ts +++ b/resources/Resources.ts @@ -1,7 +1,9 @@ +import * as ErrorResourceRuntimeModule from './ErrorResource.ts'; import { transaction } from './transaction.ts'; import logger from '../utility/logging/harper_logger.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import { server } from '../server/Server.ts'; +import { ErrorResource } from './ErrorResource.ts'; export interface ResourceEntry { Resource: any; @@ -160,7 +162,6 @@ export class Resources extends Map { // don't provide anything more descriptive. const error = new ServerError(`Conflicting paths for ${path}`); logger.error(error); - const { ErrorResource } = require('./ErrorResource'); entry.Resource = new ErrorResource(error); } super.set(path, entry); @@ -221,7 +222,7 @@ export class Resources extends Map { // conflicting registrations for the same parameterised path; surface it like the static-path conflict const error = new ServerError(`Conflicting paths for ${path}`); logger.error(error); - const { ErrorResource } = require('./ErrorResource'); + const { ErrorResource } = ErrorResourceRuntimeModule; compiled.entry.Resource = new ErrorResource(error); } this.paramRoutes[existingIndex] = compiled; diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index 6bcc403df3..a5424340ee 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -9,9 +9,8 @@ import { type StoreRemoveOptions, RocksDatabase, } from '@harperfast/rocksdb-js'; -import { Id } from './ResourceInterface.ts'; +import { type Id } from './ResourceInterface.ts'; import { MAXIMUM_KEY } from 'ordered-binary'; - declare module '@harperfast/rocksdb-js' { interface DBI { getValuesCount(indexedValue: any): number; diff --git a/resources/RocksTransactionLogStore.ts b/resources/RocksTransactionLogStore.ts index 48321732e7..5f57e43394 100644 --- a/resources/RocksTransactionLogStore.ts +++ b/resources/RocksTransactionLogStore.ts @@ -2,7 +2,7 @@ import { TransactionLog, RocksDatabase, shutdown, type TransactionEntry } from ' import { isReservedForLocalAppend } from './originClosedFloor.ts'; import { ExtendedIterable } from '@harperfast/extended-iterable'; import { exportIdMapping, getIdOfRemoteNode, getNodeNameForId } from './nodeIdMapping.ts'; -import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry } from './auditStore.ts'; +import { Decoder, readAuditEntry, ENTRY_DATAVIEW, type AuditRecord, createAuditEntry } from './auditStore.ts'; import { HAS_STRUCTURE_UPDATE } from './RecordEncoder.ts'; import { createCorruptFrameReporter, diff --git a/resources/Table.ts b/resources/Table.ts index 12731ccc09..20c598987c 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -29,7 +29,7 @@ import type { User } from '../security/user.ts'; import type { AssertNoDrift, AssertTrue, ExactlyEqual, MemberDrift, ParitySentinel } from './typeParity.ts'; import type { IterableEventQueue } from './IterableEventQueue.ts'; import type { Contract, SchemaClass } from './defineResource.ts'; -import lmdbProcessRows from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js'; +import lmdbProcessRows from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts'; import { Resource, SEARCH_AUTHORIZATION, transformForSelect, defaultAllowReads } from './Resource.ts'; import { settleBeforeDeadline, when, promiseNormalize } from '../utility/when.ts'; import { @@ -80,7 +80,7 @@ import { type ValidationIssue, } from '../utility/errors/hdbError.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; import { isDedicatedPoolWorker } from '../server/threads/workerPools.ts'; import { databases, @@ -132,7 +132,7 @@ import { ownsStoreExpiration, runsApplicationCodeSingletons, isDedicatedWorker, -} from '../server/threads/manageThreads.js'; +} from '../server/threads/manageThreads.ts'; import { HAS_BLOBS, LOCAL_ONLY, @@ -213,8 +213,8 @@ import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { throttle } from '../server/throttle.ts'; import { RocksDatabase, Transaction as RocksTransaction } from '@harperfast/rocksdb-js'; -import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction'; -import { contentTypes } from '../server/serverHelpers/contentTypes'; +import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction.ts'; +import { contentTypes } from '../server/serverHelpers/contentTypes.ts'; import { type JsonSchemaFragment, projectAttributesToProperties } from './jsonSchemaTypes.ts'; import { persistedFullTextIndexNames, @@ -223,7 +223,6 @@ import { } from './fullTextSchema.ts'; const { validateAttribute } = lmdbProcessRows; - export type Attribute = { name: string; type: 'ID' | 'Int' | 'Float' | 'Long' | 'String' | 'Boolean' | 'Date' | 'Bytes' | 'Any' | 'BigInt' | 'Blob' | string; @@ -306,7 +305,11 @@ function usableCount(estimate: any): number { return 0; return count; } -envMngr.initSync(); +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; export const UPDATE_ATTRIBUTES_LOCK_TIMEOUT = 10000; diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 3eb03b4967..eb4d2aa82d 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -1,14 +1,15 @@ -import { parentPort, threadId } from 'worker_threads'; -import { onMessageByType } from '../../server/threads/manageThreads.js'; +import { parentPort, threadId } from 'node:worker_threads'; +import { onStartup } from '../../utility/lifecycle.ts'; +import { onMessageByType } from '../../server/threads/manageThreads.ts'; import { getDatabases, table, isReadOnlyMode } from '../databases.ts'; import type { Databases, Table, Tables } from '../databases.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; import { stat, opendir } from 'node:fs/promises'; const { getLogFilePath, forComponent } = harperLogger; -import { dirname, join } from 'path'; -import { open } from 'fs/promises'; +import { dirname, join } from 'node:path'; +import { open } from 'node:fs/promises'; import { getNextMonotonicTime } from '../../utility/lmdb/commonUtility.ts'; -import { get as envGet, getHdbBasePath, initSync } from '../../utility/environment/environmentManager.ts'; +import { get as envGet, getHdbBasePath } from '../../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS, MAX_SET_TIMEOUT_MS } from '../../utility/hdbTerms.ts'; import { server } from '../../server/Server.ts'; import * as fs from 'node:fs'; @@ -21,8 +22,6 @@ import { RocksDatabase, type TransactionLogStats } from '@harperfast/rocksdb-js' const log = forComponent('analytics').conditional; const isBun = typeof globalThis.Bun !== 'undefined'; -initSync(); - type ActionCallback = (action: Action) => void; export type Value = number | boolean | ActionCallback; interface Action { @@ -43,6 +42,11 @@ let analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1; let analyticsReadOnlyChecked = false; let sendAnalyticsTimeout: NodeJS.Timeout; +onStartup(() => { + analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1; + analyticsReadOnlyChecked = false; +}); + // Check read-only mode lazily to avoid circular dependency at module load time function checkAnalyticsEnabled(): boolean { if (!analyticsReadOnlyChecked) { @@ -130,8 +134,6 @@ export function recordAction(value: Value, metric: string, path?: string, method if (!sendAnalyticsTimeout) contextStorage.exit(sendAnalytics); } -server.recordAnalytics = recordAction; - // Let the storage layer emit write-commit latency without statically depending on this module. setCommitLatencyRecorder((durationMs) => recordAction(durationMs, METRIC.TRANSACTION_COMMIT_TIME)); @@ -1404,3 +1406,5 @@ function rebalance({ counts, values, totalCount }, resetCounts: boolean) { else counts.set(targetCounts); } */ + +server.recordAnalytics = recordAction; diff --git a/resources/auditStore.ts b/resources/auditStore.ts index 1f4cb748d5..f71a1bebad 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -1,9 +1,9 @@ import { randomBytes } from 'node:crypto'; import { readKey, writeKey } from 'ordered-binary'; -import { initSync, get as envGet } from '../utility/environment/environmentManager.ts'; +import { get as envGet } from '../utility/environment/environmentManager.ts'; import { AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; -import { getWorkerIndex, ownsStoreMaintenance } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, ownsStoreMaintenance } from '../server/threads/manageThreads.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { LAST_TIMESTAMP_PLACEHOLDER, HAS_STRUCTURE_UPDATE, PENDING_LOCAL_TIME } from './RecordEncoder.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; @@ -40,7 +40,6 @@ import { isReadOnlyMode, openRocksDatabase } from './databases.ts'; * username * remaining bytes (optional, not included for deletes/invalidation): the record itself, using the same encoding as its primary store */ -initSync(); export type AuditRecord = { version: number; // the record's own version: LWW ordering, @updatedTime, ETag diff --git a/resources/blob.ts b/resources/blob.ts index a6aa275ba5..80063e722c 100644 --- a/resources/blob.ts +++ b/resources/blob.ts @@ -48,10 +48,11 @@ import { import type { ReadStream, StatsFs } from 'node:fs'; import { createDeflate, createInflate, inflate } from 'node:zlib'; import { Readable, Transform, pipeline } from 'node:stream'; -import { ensureDirSync } from 'fs-extra'; +import _typestrip_fs_extra from 'fs-extra'; +const { ensureDirSync } = _typestrip_fs_extra; import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS, MAX_SET_TIMEOUT_MS } from '../utility/hdbTerms.ts'; -import { join, dirname } from 'path'; +import { join, dirname } from 'node:path'; import { logger } from '../utility/logging/logger.ts'; import { resolveWatchTarget } from '../utility/watchPath.ts'; import type { RootDatabase } from 'lmdb'; diff --git a/resources/dataLoader.ts b/resources/dataLoader.ts index 61d9fc339f..476f09e8e0 100644 --- a/resources/dataLoader.ts +++ b/resources/dataLoader.ts @@ -1,15 +1,31 @@ import { basename, extname } from 'node:path'; import { createHash } from 'node:crypto'; import { parseDocument } from 'yaml'; -import { Databases, databases, table, Tables, tables } from './databases.ts'; -import { isApplicationPrimaryWorker } from '../server/threads/manageThreads.js'; +import { type Databases, databases, table, type Tables, tables } from './databases.ts'; +import { isApplicationPrimaryWorker } from '../server/threads/manageThreads.ts'; import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; -import { Attribute } from './Table.ts'; -import { FileEntry } from '../components/EntryHandler.ts'; - -const dataLoaderLogger = harperLogger.forComponent('dataLoader'); +import { type Attribute } from './Table.ts'; +import { type FileEntry } from '../components/EntryHandler.ts'; + +// Resolve the logger lazily so unit tests can stub `forComponent` between +// module load and first use (otherwise transitive importers — componentLoader +// pulled in via http.ts / threadServer.ts / operations.ts — capture the real +// logger before dataLoader.test.js installs its stub). +let _dataLoaderLogger: any; +function getDataLoaderLogger() { + if (!_dataLoaderLogger) _dataLoaderLogger = harperLogger.forComponent('dataLoader'); + return _dataLoaderLogger; +} +const dataLoaderLogger = new Proxy( + {}, + { + get(_t, prop) { + return getDataLoaderLogger()[prop]; + }, + } +) as any; /** System table name for storing data loader hashes */ const DATA_LOADER_HASH_TABLE = 'hdb_dataloader_hash'; diff --git a/resources/databases.ts b/resources/databases.ts index b53f086248..f375fb4648 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1,5 +1,7 @@ +import { loadNativePackage } from '../utility/packageUtils.js'; import { EventEmitter } from 'node:events'; import { randomBytes, randomUUID } from 'node:crypto'; +import { setDatabasesGetter } from '../utility/databasesRef.ts'; import { initSync, getHdbBasePath, get as envGet } from '../utility/environment/environmentManager.ts'; import { INTERNAL_DBIS_NAME } from '../utility/lmdb/terms.ts'; import type { Database, RootDatabase } from 'lmdb'; @@ -19,7 +21,7 @@ import { rm, unlink } from 'node:fs/promises'; import { getBaseSchemaPath, getTransactionAuditStoreBasePath, -} from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +} from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import { makeTable, ignoreAlreadyDropped, @@ -42,11 +44,11 @@ import { ClientError, DatabaseClosingError, TableGenerationDroppedError } from ' import { _assignPackageExport } from '../globals.js'; import { getIndexedValues, getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; -import { workerData } from 'worker_threads'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; +import { workerData } from 'node:worker_threads'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent } = harperLogger; -import * as manageThreads from '../server/threads/manageThreads.js'; +import * as manageThreads from '../server/threads/manageThreads.ts'; import { establishAuditFloor, openAuditStore, @@ -94,7 +96,7 @@ import { import { totalmem } from 'node:os'; import { RocksIndexStore } from './RocksIndexStore.ts'; import { resolveRocksMemoryConfig } from '../utility/rocksMemoryConfig.ts'; -import { isProcessRunning } from '../utility/processManagement/processManagement.js'; +import { isProcessRunning } from '../utility/processManagement/processManagement.ts'; import { compileFullTextDefinitions, compileFullTextFields, @@ -175,7 +177,7 @@ function createOpenDBIObject(dupSort = false, isPrimary = false) { } // lmdb allocates a 16 MB native buffer per thread on load, so it is loaded only when an LMDB store is opened function openLmdb(options: Parameters[0]): RootDatabase { - return (require('lmdb') as typeof import('lmdb')).open(options); + return (loadNativePackage('lmdb') as typeof import('lmdb')).open(options); } // The __dbis__ metadata DBI is non-versioned (OpenDBIObject useVersions=false); only versioned // primary stores carry the per-record metadata prefix. lmdb/rocksdb don't forward `useVersions` to @@ -191,7 +193,11 @@ const DEFAULT_DATABASE_NAME = 'data'; const DEFINED_TABLES = Symbol('defined-tables'); const CATALOG_RELATIONSHIP = Symbol('catalog-relationship'); const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages -initSync(); +try { + initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} type RelationshipTarget = { database: string; table: string }; type PersistedRelationship = { @@ -1027,6 +1033,9 @@ export function getTables(): Tables { * but in newer multi-table databases, there is one consistent, integrated audit table for the database since transactions * can span any tables in the database. */ +// Register getter in the shared registry so common_utils.ts can access databases +// without importing this module directly (which would create a circular dep). +setDatabasesGetter(() => getDatabases()); export function getDatabases(): Databases { if (loadedDatabases) { return databases; diff --git a/resources/derivedIndexes.ts b/resources/derivedIndexes.ts index bf8d7bcc7b..de75c5eb48 100644 --- a/resources/derivedIndexes.ts +++ b/resources/derivedIndexes.ts @@ -3,7 +3,7 @@ import { ClientError } from '../utility/errors/hdbError.ts'; import { settleBeforeDeadline } from '../utility/when.ts'; import { ITC_EVENT_TYPES } from '../utility/hdbTerms.ts'; import { loggerWithTag } from '../utility/logging/logger.ts'; -import { sendItcEventStrict } from '../server/threads/itc.js'; +import { sendItcEventStrict } from '../server/threads/itc.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; import type { RocksTransactionLogStore } from './RocksTransactionLogStore.ts'; import { diff --git a/resources/graphql.ts b/resources/graphql.ts index 668b61e42a..7ae383b273 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -1,7 +1,7 @@ -import { dirname } from 'path'; +import { dirname } from 'node:path'; import { Script } from 'node:vm'; import { scopedTableFactory, table } from './databases.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads.js'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; import { thisThreadOwnsApplication } from '../server/threads/isolatedApplications.ts'; import { Resources } from './Resources.ts'; import type { DirectiveNode, NamedTypeNode, StringValueNode, ValueNode } from 'graphql'; diff --git a/resources/indexes/hnswPlaneBinding.ts b/resources/indexes/hnswPlaneBinding.ts index 34401a5dfa..143d76e8e4 100644 --- a/resources/indexes/hnswPlaneBinding.ts +++ b/resources/indexes/hnswPlaneBinding.ts @@ -1,3 +1,4 @@ +import { loadNativePackage } from '../../utility/packageUtils.js'; import { closeSync, fsyncSync, openSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { loggerWithTag } from '../../utility/logging/logger.ts'; @@ -123,7 +124,7 @@ let bindingWarningLogged = false; function getHnswPackage(warn = true): HnswPlanePackage | null { if (binding === undefined) { try { - binding = require('@harperfast/hnsw') as HnswPlanePackage; + binding = loadNativePackage('@harperfast/hnsw') as HnswPlanePackage; } catch (error) { binding = null; bindingLoadError = error as Error; diff --git a/resources/login.ts b/resources/login.ts index 321b4aeb6f..1e9629b94b 100644 --- a/resources/login.ts +++ b/resources/login.ts @@ -1,22 +1,30 @@ import { Resource } from './Resource.ts'; -import { Scope } from '../components/Scope.ts'; +import type { Scope } from '../components/Scope.ts'; + +// Login class is created lazily because login.ts is loaded transitively from +// Resource.ts's own static graph; declaring `class Login extends Resource` at +// module-load would hit Resource's TDZ under that ESM cycle. +let LoginClass: any; +function getLoginClass() { + if (LoginClass) return LoginClass; + // @ts-ignore + LoginClass = class Login extends Resource { + static async get(_id, _body, _request) { + // TODO: Return a login page + } + static async post(_id, body, request) { + const { username, password, token } = (await body) ?? {}; + return { + data: await request.login(username, password, token), + }; + } + }; + return LoginClass; +} + export function handleApplication(scope: Scope) { - scope.resources.set('login', Login); + scope.resources.set('login', getLoginClass()); scope.resources.loginPath = (request) => { return '/login?redirect=' + encodeURIComponent(request.url); }; } -// @ts-ignore -class Login extends Resource { - static async get(_id, _body, _request) { - // TODO: Return a login page - } - static async post(_id, body, request) { - // a static override replaces the transactional() wrapper that would otherwise resolve this, - // so the body arrives as a pending promise over REST (see resources/DESIGN.md) - const { username, password, token } = (await body) ?? {}; - return { - data: await request.login(username, password, token), - }; - } -} diff --git a/resources/models/calibrationStore.ts b/resources/models/calibrationStore.ts index 90dc303906..9d9947faa3 100644 --- a/resources/models/calibrationStore.ts +++ b/resources/models/calibrationStore.ts @@ -2,7 +2,7 @@ import { getDatabases, isReadOnlyMode, table } from '../databases.ts'; import { contextStorage, transaction } from '../transaction.ts'; import type { Context } from '../ResourceInterface.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; -import { isApplicationPrimaryWorker } from '../../server/threads/manageThreads.js'; +import { isApplicationPrimaryWorker } from '../../server/threads/manageThreads.ts'; import { recordAction } from '../analytics/write.ts'; import { internalJobOwner, diff --git a/resources/models/decideHook.ts b/resources/models/decideHook.ts index 5a2402c10d..3e654c1879 100644 --- a/resources/models/decideHook.ts +++ b/resources/models/decideHook.ts @@ -1,3 +1,4 @@ +import * as ModelsRuntimeModule from './Models.ts'; import { ClientError } from '../../utility/errors/hdbError.ts'; import { isAllowedValue } from './decision.ts'; import { @@ -41,7 +42,7 @@ type DecideFn = (state: DecideInput, schema: DecisionLeaf, opts: DecideOpts) => let _decideFn: DecideFn | undefined; function resolveDecideFn(): DecideFn { if (_decideFn) return _decideFn; - const { Models } = require('#src/resources/models/Models'); // eslint-disable-line @typescript-eslint/no-var-requires + const { Models } = ModelsRuntimeModule; // eslint-disable-line @typescript-eslint/no-var-requires const models = new Models(); _decideFn = (state, schema, opts) => models.decide(state, schema, opts); return _decideFn; diff --git a/resources/models/embedHook.ts b/resources/models/embedHook.ts index 61af9d2663..bf0dccbafc 100644 --- a/resources/models/embedHook.ts +++ b/resources/models/embedHook.ts @@ -1,3 +1,5 @@ +import * as loggerRuntimeModule from '../../utility/logging/harper_logger.ts'; +import * as ModelsRuntimeModule from './Models.ts'; import { ClientError } from '../../utility/errors/hdbError.ts'; import { validateDecisionSchema } from './decision.ts'; @@ -5,7 +7,7 @@ import { validateDecisionSchema } from './decision.ts'; function getLogger(): { error?: (...args: any[]) => void } { try { // eslint-disable-next-line @typescript-eslint/no-var-requires - return require('#src/utility/logging/logger').logger ?? {}; + return { error: loggerRuntimeModule.default.logsAtLevel('error') ? loggerRuntimeModule.default.error : undefined }; } catch { return {}; } @@ -39,7 +41,7 @@ type EmbedFn = ( let _embedFn: EmbedFn | undefined; function resolveEmbedFn(): EmbedFn { if (_embedFn) return _embedFn; - const { Models } = require('#src/resources/models/Models'); // eslint-disable-line @typescript-eslint/no-var-requires + const { Models } = ModelsRuntimeModule; // eslint-disable-line @typescript-eslint/no-var-requires const models = new Models(); _embedFn = (input, opts) => models.embed(input, opts); return _embedFn; diff --git a/resources/originClosedFloor.ts b/resources/originClosedFloor.ts index 6b39328305..4feb795ce2 100644 --- a/resources/originClosedFloor.ts +++ b/resources/originClosedFloor.ts @@ -1,7 +1,7 @@ import { threadId } from 'node:worker_threads'; import type { Transaction as RocksTransaction, RocksDatabase } from '@harperfast/rocksdb-js'; import * as harperLogger from '../utility/logging/harper_logger.ts'; -import { onMessageByType } from '../server/threads/manageThreads.js'; +import { onMessageByType } from '../server/threads/manageThreads.ts'; /** * Origin-closed timestamp floor (harper-pro#922, item 1). diff --git a/resources/replayLogs.ts b/resources/replayLogs.ts index 465d555358..91593b0db8 100644 --- a/resources/replayLogs.ts +++ b/resources/replayLogs.ts @@ -1,7 +1,7 @@ import { RocksDatabase } from '@harperfast/rocksdb-js'; import { Resource } from './Resource.ts'; import type { Context } from './ResourceInterface.ts'; -import * as logger from '../utility/logging/harper_logger.js'; +import * as logger from '../utility/logging/harper_logger.ts'; import { DatabaseTransaction } from './DatabaseTransaction.ts'; import { RocksTransactionLogStore } from './RocksTransactionLogStore.ts'; import { isMainThread } from 'node:worker_threads'; diff --git a/resources/scheduler/scheduler.ts b/resources/scheduler/scheduler.ts index 5f5e5cda3d..5766f91eb9 100644 --- a/resources/scheduler/scheduler.ts +++ b/resources/scheduler/scheduler.ts @@ -1,5 +1,5 @@ import { isAbsolute, join } from 'node:path'; -import { isApplicationPrimaryWorker } from '../../server/threads/manageThreads.js'; +import { isApplicationPrimaryWorker } from '../../server/threads/manageThreads.ts'; import { ClientError } from '../../utility/errors/hdbError.ts'; import { convertToMS } from '../../utility/common_utils.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; diff --git a/resources/search.ts b/resources/search.ts index 5d52091a15..4b155d7c7e 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -7,7 +7,7 @@ import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; import { lastMetadata } from './RecordEncoder.ts'; import { writeKeyId, getReadTransactionGuard } from './DatabaseTransaction.ts'; -import { recordAction } from './analytics/write'; +import { recordAction } from './analytics/write.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; import { appendHeader } from '../server/serverHelpers/Headers.ts'; import type { FullTextCondition } from './indexes/fullTextQueryIndex.ts'; diff --git a/resources/transactionBroadcast.ts b/resources/transactionBroadcast.ts index 034ed1a6a1..84bb6b44d5 100644 --- a/resources/transactionBroadcast.ts +++ b/resources/transactionBroadcast.ts @@ -1,5 +1,5 @@ import { basename } from 'node:path'; -import { warn } from '../utility/logging/harper_logger.js'; +import { warn } from '../utility/logging/harper_logger.ts'; import { DatabaseClosingError, DatabaseGenerationChangedError } from '../utility/errors/hdbError.ts'; import { IterableEventQueue } from './IterableEventQueue.ts'; import { keyArrayToString } from './Resources.ts'; diff --git a/security/auth.ts b/security/auth.ts index 559598b6bb..54f279b794 100644 --- a/security/auth.ts +++ b/security/auth.ts @@ -21,46 +21,68 @@ import { import { serializeMessage } from '../server/serverHelpers/contentTypes.ts'; import { hdbErrors } from '../utility/errors/hdbError.ts'; const { AUTHENTICATION_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const authLogger = forComponent('authentication'); -const { debug } = authLogger; -const authEventLog = authLogger.withTag('auth-event'); -env.initSync(); - -const appsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST); -const appsCors = env.get(CONFIG_PARAMS.HTTP_CORS); -const operationsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST); -const operationsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS); +import { onStartup } from '../utility/lifecycle.ts'; +let authLogger = forComponent('authentication'); +let { debug } = authLogger; +let authEventLog = authLogger.withTag('auth-event'); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} +// Config-derived state is populated during startup, after the environment is +// initialized and the module graph is fully linked. Reading config here at +// module-load would either TDZ inside an ESM cycle or pick up stale defaults. +let appsCorsAccesslist: any; +let appsCors: any; +let operationsCorsAccesslist: any; +let operationsCors: any; +let _sessionTable: Table | undefined; +let ENABLE_SESSIONS: boolean = true; +let AUTHORIZE_LOCAL: any = envFlag(process.env.AUTHENTICATION_AUTHORIZELOCAL) ?? envFlag(process.env.DEV_MODE); +let LOG_AUTH_SUCCESSFUL: boolean = false; +let LOG_AUTH_FAILED: boolean = false; -const _sessionTable = table({ - table: 'hdb_session', - database: 'system', - attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'user' }], -}); function getSessionTable() { return _sessionTable; } -const ENABLE_SESSIONS = env.get(CONFIG_PARAMS.AUTHENTICATION_ENABLESESSIONS) ?? true; -// env-var strings need boolean parsing: a raw 'false'/'0' string is truthy, which would turn -// AUTHENTICATION_AUTHORIZELOCAL=false into an *enabled* auth bypass function envFlag(value: string | undefined): boolean | undefined { if (value === undefined) return undefined; const normalized = value.trim().toLowerCase(); return normalized !== 'false' && normalized !== '0' && normalized !== ''; } -// check the environment for a flag to bypass authentication (for testing) since it doesn't necessarily get set on child threads -let AUTHORIZE_LOCAL = - envFlag(process.env.AUTHENTICATION_AUTHORIZELOCAL) ?? - env.get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? - envFlag(process.env.DEV_MODE); -const LOG_AUTH_SUCCESSFUL = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGSUCCESSFUL) ?? false; -const LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED) ?? false; const DEFAULT_COOKIE_EXPIRES = 'Tue, 01 Oct 8307 19:33:20 GMT'; let authorizationCache = new Map(); -server.onInvalidatedUser(() => { - // TODO: Eventually we probably want to be able to invalidate individual users - authorizationCache = new Map(); + +onStartup(() => { + authLogger = forComponent('authentication'); + ({ debug } = authLogger); + authEventLog = authLogger.withTag('auth-event'); + appsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST); + appsCors = env.get(CONFIG_PARAMS.HTTP_CORS); + operationsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST); + operationsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS); + ENABLE_SESSIONS = env.get(CONFIG_PARAMS.AUTHENTICATION_ENABLESESSIONS) ?? true; + AUTHORIZE_LOCAL = + envFlag(process.env.AUTHENTICATION_AUTHORIZELOCAL) ?? + env.get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? + envFlag(process.env.DEV_MODE); + LOG_AUTH_SUCCESSFUL = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGSUCCESSFUL) ?? false; + LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED) ?? false; + _sessionTable = table
({ + table: 'hdb_session', + database: 'system', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'user' }], + }); + server.onInvalidatedUser(() => { + // TODO: Eventually we probably want to be able to invalidate individual users + authorizationCache = new Map(); + }); + setInterval(() => { + authorizationCache = new Map(); + }, env.get(CONFIG_PARAMS.AUTHENTICATION_CACHETTL)).unref(); }); let bypassUser: any; export function bypassAuth() { @@ -484,9 +506,6 @@ export async function authentication(request, nextHandler) { return response; } } -setInterval(() => { - authorizationCache = new Map(); -}, env.get(CONFIG_PARAMS.AUTHENTICATION_CACHETTL)).unref(); let started = false; export function handleApplication(scope: import('../components/Scope.ts').Scope) { if (started) return; diff --git a/security/certificateVerification/certificateVerificationSource.ts b/security/certificateVerification/certificateVerificationSource.ts index c1bdbe9f70..61fc20676b 100644 --- a/security/certificateVerification/certificateVerificationSource.ts +++ b/security/certificateVerification/certificateVerificationSource.ts @@ -13,11 +13,11 @@ let performOCSPCheck: any; // Lazy load to avoid circular dependencies async function loadVerificationFunctions() { if (!performCRLCheck) { - const crlModule = await import('./crlVerification.js'); + const crlModule = await import('./crlVerification.ts'); performCRLCheck = (crlModule as any).performCRLCheck; } if (!performOCSPCheck) { - const ocspModule = await import('./ocspVerification.js'); + const ocspModule = await import('./ocspVerification.ts'); performOCSPCheck = (ocspModule as any).performOCSPCheck; } } @@ -25,59 +25,79 @@ async function loadVerificationFunctions() { /** * Certificate Verification Source that can handle both CRL and OCSP */ -export class CertificateVerificationSource extends Resource { - async get(query: Query) { - const id = query.id as string; +let _CertificateVerificationSource: any; +function getCertificateVerificationSource(): any { + if (!_CertificateVerificationSource) { + _CertificateVerificationSource = class CertificateVerificationSource extends Resource { + async get(query: Query) { + const id = query.id as string; - // Get the certificate data from requestContext - const context = this.getContext() as SourceContext; - const requestContext = context?.requestContext; + // Get the certificate data from requestContext + const context = this.getContext() as SourceContext; + const requestContext = context?.requestContext; - if (!requestContext || !requestContext.certPem || !requestContext.issuerPem) { - // Likely a source request for an expired entry - we can't verify without cert and issuer data - return null; - } + if (!requestContext || !requestContext.certPem || !requestContext.issuerPem) { + // Likely a source request for an expired entry - we can't verify without cert and issuer data + return null; + } - const { certPem: certPemStr, issuerPem: issuerPemStr, ocspUrls, config } = requestContext; + const { certPem: certPemStr, issuerPem: issuerPemStr, ocspUrls, config } = requestContext; - // Determine method from cache key - let method: string; - if (id.startsWith('crl:')) { - method = 'crl'; - } else if (id.startsWith('ocsp:')) { - method = 'ocsp'; - } else { - method = 'unknown'; - } + // Determine method from cache key + let method: string; + if (id.startsWith('crl:')) { + method = 'crl'; + } else if (id.startsWith('ocsp:')) { + method = 'ocsp'; + } else { + method = 'unknown'; + } - // Load verification functions - await loadVerificationFunctions(); + // Load verification functions + await loadVerificationFunctions(); - // Perform verification based on method - let result; - let methodConfig; + // Perform verification based on method + let result; + let methodConfig; - if (method === 'crl') { - methodConfig = config.crl; - // Pass distributionPoint as an array if available (for CRL fetch) - const crlUrls = requestContext.distributionPoint ? [requestContext.distributionPoint] : undefined; - result = await performCRLCheck(certPemStr, issuerPemStr, methodConfig, crlUrls); - } else if (method === 'ocsp') { - methodConfig = config.ocsp; - result = await performOCSPCheck(certPemStr, issuerPemStr, methodConfig, ocspUrls); - } else { - throw new Error(`Unsupported verification method: ${method} for ID: ${id}`); - } + if (method === 'crl') { + methodConfig = config.crl; + // Pass distributionPoint as an array if available (for CRL fetch) + const crlUrls = requestContext.distributionPoint ? [requestContext.distributionPoint] : undefined; + result = await performCRLCheck(certPemStr, issuerPemStr, methodConfig, crlUrls); + } else if (method === 'ocsp') { + methodConfig = config.ocsp; + result = await performOCSPCheck(certPemStr, issuerPemStr, methodConfig, ocspUrls); + } else { + throw new Error(`Unsupported verification method: ${method} for ID: ${id}`); + } - // without a cacheTtl, the table's expiration applies rather than none - if (Number.isFinite(methodConfig.cacheTtl)) context.expiresAt = Date.now() + methodConfig.cacheTtl; + // without a cacheTtl, the table's expiration applies rather than none + if (Number.isFinite(methodConfig.cacheTtl)) context.expiresAt = Date.now() + methodConfig.cacheTtl; - return { - certificate_id: id, - status: result.status, - reason: result.reason, - checked_at: Date.now(), - method, + return { + certificate_id: id, + status: result.status, + reason: result.reason, + checked_at: Date.now(), + method, + }; + } }; } + return _CertificateVerificationSource; } +export const CertificateVerificationSource: any = new Proxy(function () {} as any, { + construct(_target, args, newTarget) { + return Reflect.construct(getCertificateVerificationSource(), args, newTarget); + }, + get(_target, prop) { + return getCertificateVerificationSource()[prop]; + }, + has(_target, prop) { + return prop in getCertificateVerificationSource(); + }, + getPrototypeOf() { + return Object.getPrototypeOf(getCertificateVerificationSource()); + }, +}); diff --git a/security/certificateVerification/crlVerification.ts b/security/certificateVerification/crlVerification.ts index 6d7ecb93d2..e242c3342b 100644 --- a/security/certificateVerification/crlVerification.ts +++ b/security/certificateVerification/crlVerification.ts @@ -58,54 +58,75 @@ function getCertificateCacheTable() { /** * CRL fetching and validation source */ -class CertificateRevocationListSource extends Resource { - async get(id: string) { - const context = this.getContext() as SourceContext; - const requestContext = context?.requestContext; +let _CertificateRevocationListSource: any; +function getCertificateRevocationListSource(): any { + if (!_CertificateRevocationListSource) { + _CertificateRevocationListSource = class CertificateRevocationListSource extends Resource { + async get(id: string) { + const context = this.getContext() as SourceContext; + const requestContext = context?.requestContext; + + if (!requestContext?.distributionPoint || !requestContext?.issuerPem) { + throw new Error(`No CRL data provided for cache key: ${id}`); + } - if (!requestContext?.distributionPoint || !requestContext?.issuerPem) { - throw new Error(`No CRL data provided for cache key: ${id}`); - } + const { distributionPoint, issuerPem: issuerPemStr, config } = requestContext; - const { distributionPoint, issuerPem: issuerPemStr, config } = requestContext; + try { + const { entry } = await downloadAndParseCRLOnce(distributionPoint, issuerPemStr, config); - try { - const { entry } = await downloadAndParseCRLOnce(distributionPoint, issuerPemStr, config); + // Set expiration - use the CRL's nextUpdate time or configured TTL, whichever is sooner + context.expiresAt = Math.min(entry.next_update, Date.now() + config.cacheTtl); - // Set expiration - use the CRL's nextUpdate time or configured TTL, whichever is sooner - context.expiresAt = Math.min(entry.next_update, Date.now() + config.cacheTtl); - - return entry; - } catch (error) { - logger.error?.(`CRL fetch error for: ${distributionPoint} - ${error}`); + return entry; + } catch (error) { + logger.error?.(`CRL fetch error for: ${distributionPoint} - ${error}`); - if (error instanceof CRLSignatureVerificationError) { - throw error; - } + if (error instanceof CRLSignatureVerificationError) { + throw error; + } - // Check failure mode - if (config.failureMode === 'fail-closed') { - // Cache the error for faster recovery - context.expiresAt = Date.now() + ERROR_CACHE_TTL; + // Check failure mode + if (config.failureMode === 'fail-closed') { + // Cache the error for faster recovery + context.expiresAt = Date.now() + ERROR_CACHE_TTL; + + return { + crl_id: id, + distribution_point: distributionPoint, + issuer_dn: 'unknown', + crl_blob: Buffer.alloc(0), + this_update: Date.now(), + next_update: context.expiresAt, + signature_valid: false, + }; + } - return { - crl_id: id, - distribution_point: distributionPoint, - issuer_dn: 'unknown', - crl_blob: Buffer.alloc(0), - this_update: Date.now(), - next_update: context.expiresAt, - signature_valid: false, - }; + // Fail open - return null to not cache + logger.warn?.('CRL fetch failed, not caching (fail-open mode)'); + return null; + } } - - // Fail open - return null to not cache - logger.warn?.('CRL fetch failed, not caching (fail-open mode)'); - return null; - } + }; } + return _CertificateRevocationListSource; } +const CertificateRevocationListSource: any = new Proxy(function () {} as any, { + construct(_target, args, newTarget) { + return Reflect.construct(getCertificateRevocationListSource(), args, newTarget); + }, + get(_target, prop) { + return getCertificateRevocationListSource()[prop]; + }, + has(_target, prop) { + return prop in getCertificateRevocationListSource(); + }, + getPrototypeOf() { + return Object.getPrototypeOf(getCertificateRevocationListSource()); + }, +}); + // Lazy-load Harper tables let crlCacheTable: ReturnType; let revokedCertificateTable: ReturnType; diff --git a/security/certificateVerification/ocspVerification.ts b/security/certificateVerification/ocspVerification.ts index 6800bbd704..9a5c3ef57a 100644 --- a/security/certificateVerification/ocspVerification.ts +++ b/security/certificateVerification/ocspVerification.ts @@ -1,3 +1,4 @@ +import { loadNativePackage } from '../../utility/packageUtils.js'; /** * OCSP (Online Certificate Status Protocol) verification */ @@ -122,7 +123,7 @@ export async function performOCSPCheck( ): Promise { try { loadPkijs(); - const { getCertStatus } = require('easy-ocsp') as typeof import('easy-ocsp'); + const { getCertStatus } = loadNativePackage('easy-ocsp') as typeof import('easy-ocsp'); const response = await getCertStatus(certPem, { ca: issuerPem, timeout: config.timeout, diff --git a/security/certificateVerification/pkijs-ed25519-patch.ts b/security/certificateVerification/pkijs-ed25519-patch.ts index 7b8977bedd..714cb83309 100644 --- a/security/certificateVerification/pkijs-ed25519-patch.ts +++ b/security/certificateVerification/pkijs-ed25519-patch.ts @@ -1,3 +1,4 @@ +import { loadNativePackage } from '../../utility/packageUtils.js'; /** * PKI.js Ed25519/Ed448 Support Patch * @@ -42,7 +43,7 @@ function getEdDSAAlgorithmName(oid: string): EdDSAAlgorithmName { } export function loadPkijs(): typeof Pkijs { - const pkijs = require('pkijs') as typeof Pkijs; + const pkijs = loadNativePackage('pkijs') as typeof Pkijs; if (!patchesApplied) { applyEd25519Patch(pkijs); patchesApplied = true; diff --git a/security/certificateVerification/verificationUtils.ts b/security/certificateVerification/verificationUtils.ts index cbf9debd06..703696bfe2 100644 --- a/security/certificateVerification/verificationUtils.ts +++ b/security/certificateVerification/verificationUtils.ts @@ -1,3 +1,4 @@ +import { loadNativePackage } from '../../utility/packageUtils.js'; /** * Shared utilities for certificate verification */ @@ -70,7 +71,7 @@ export function extractCRLDistributionPoints(certPem: string): string[] { // Parse the certificate using PKI.js const certBuffer = pemToBuffer(certPem); const pkijs = loadPkijs(); - const asn1js = require('asn1js') as typeof Asn1js; + const asn1js = loadNativePackage('asn1js') as typeof Asn1js; const cert = pkijs.Certificate.fromBER(certBuffer); // Look for CRL Distribution Points extension (OID: 2.5.29.31) @@ -130,7 +131,7 @@ export function extractCRLDistributionPoints(certPem: string): string[] { export function extractRevocationUrls(certPem: string): { crlUrls: string[]; ocspUrls: string[] } { try { const pkijs = loadPkijs(); - const asn1js = require('asn1js') as typeof Asn1js; + const asn1js = loadNativePackage('asn1js') as typeof Asn1js; // Parse the certificate using PKI.js (single parse for both URL types) const certBuffer = pemToBuffer(certPem); const cert = pkijs.Certificate.fromBER(certBuffer); @@ -225,7 +226,7 @@ export function extractOCSPUrls(certPem: string): string[] { // Parse the certificate using PKI.js const certBuffer = pemToBuffer(certPem); const pkijs = loadPkijs(); - const asn1js = require('asn1js') as typeof Asn1js; + const asn1js = loadNativePackage('asn1js') as typeof Asn1js; const cert = pkijs.Certificate.fromBER(certBuffer); // Look for Authority Information Access extension (OID: 1.3.6.1.5.5.7.1.1) @@ -383,7 +384,7 @@ export function extractIssuerKeyId(certPem: string): string { try { const certBuffer = pemToBuffer(certPem); const pkijs = loadPkijs(); - const asn1js = require('asn1js') as typeof Asn1js; + const asn1js = loadNativePackage('asn1js') as typeof Asn1js; const cert = pkijs.Certificate.fromBER(certBuffer); // Look for Authority Key Identifier extension (OID: 2.5.29.35) diff --git a/security/fastifyAuth.ts b/security/fastifyAuth.ts index 004d353ee7..2f5873bfea 100644 --- a/security/fastifyAuth.ts +++ b/security/fastifyAuth.ts @@ -4,11 +4,12 @@ import * as validation from '../validation/check_permissions.ts'; import passport from 'passport'; import { Strategy as LocalStrategy } from 'passport-local'; import { BasicStrategy } from 'passport-http'; -import * as util from 'util'; +import * as util from 'node:util'; import * as userFunctions from './user.ts'; const cbFindValidateUsers = util.callbackify(userFunctions.findAndValidateUser); import * as hdbTerms from '../utility/hdbTerms.ts'; -import * as tokenAuthentication from './tokenAuthentication.ts'; +import * as _tokenAuthentication from './tokenAuthentication.ts'; +const tokenAuthentication = _tokenAuthentication; import { AccessViolation } from '../utility/errors/hdbError.ts'; import { authentication } from './auth.ts'; diff --git a/security/jsLoader.ts b/security/jsLoader.ts index 97dc75498b..7b32db5b3e 100644 --- a/security/jsLoader.ts +++ b/security/jsLoader.ts @@ -8,12 +8,21 @@ import { defineResource, t, schemaOf, projectTableFragment } from '../resources/ import { readFile } from 'node:fs/promises'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL, fileURLToPath } from 'node:url'; -import { SourceTextModule, SyntheticModule, createContext, runInContext, runInThisContext } from 'node:vm'; +import * as _vm from 'node:vm'; +import { createContext, runInContext, runInThisContext } from 'node:vm'; +// SourceTextModule and SyntheticModule require `--experimental-vm-modules`. Pull +// them off the vm namespace at runtime so the named ESM import doesn't fail when +// the flag isn't passed (e.g. CLI paths that never touch the JS loader). +const { SourceTextModule, SyntheticModule } = _vm as any; +type SourceTextModule = any; +type SyntheticModule = any; import { ApplicationScope } from '../components/ApplicationScope.ts'; import { getSecretsForComponent, runWithComponentBinding } from '../components/componentSecrets.ts'; import logger from '../utility/logging/harper_logger.ts'; import { createRequire } from 'node:module'; -import * as env from '../utility/environment/environmentManager'; +import { PACKAGE_ROOT } from '../utility/packageUtils.js'; +const nativeRequire = createRequire(join(PACKAGE_ROOT, 'package.json')); +import * as env from '../utility/environment/environmentManager.ts'; import * as child_process from 'node:child_process'; import { CONFIG_PARAMS, DEFAULT_DATABASE_NAME } from '../utility/hdbTerms.ts'; @@ -33,11 +42,12 @@ import { withFileRetry, writePidRecord, } from './spawnPidFile.ts'; -import { whenComponentsLoaded } from '../server/threads/threadServer.js'; +import { whenComponentsLoaded } from '../server/threads/threadServer.ts'; import { thisThreadOwnsApplication } from '../server/threads/isolatedApplications.ts'; +import { onStartup } from '../utility/lifecycle.ts'; type Lockdown = 'none' | 'freeze' | 'ses' | 'freeze-after-load'; -const APPLICATIONS_LOCKDOWN: Lockdown = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN); +let APPLICATIONS_LOCKDOWN: Lockdown = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN); const HARPER_MODULE_IDS = new Set([ 'harper', 'harperdb', @@ -135,7 +145,7 @@ let amaro: typeof import('amaro') | undefined; function stripTypeScriptTypes(source: string): string { // Use amaro - the library that Node.js uses internally for type stripping if (!amaro) { - amaro = require('amaro'); + amaro = nativeRequire('amaro'); } return amaro.transformSync(source, { mode: 'strip-only' }).code; } @@ -432,7 +442,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC return cjsModule; } function loadCJSModule(url: string, source: string, usePrivateGlobal: boolean): SyntheticModule { - const cjsModule = usePrivateGlobal ? loadCJS(url, source) : { exports: require(url) }; + const cjsModule = usePrivateGlobal ? loadCJS(url, source) : { exports: nativeRequire(url) }; let exports = cjsModule.exports; if (exports.default === undefined) { // provide the default export for compatibility @@ -713,10 +723,10 @@ async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope, useC // The ESM shims ses/index.js composes, in its order: ses's CommonJS bundle fails its own strict-mode check on Bun, // which drops function-level 'use strict' in CommonJS function loadSES(): void { - require('ses/lockdown-shim.js'); - require('ses/compartment-shim.js'); - require('ses/assert-shim.js'); - require('ses/console-shim.js'); + nativeRequire('ses/lockdown-shim.js'); + nativeRequire('ses/compartment-shim.js'); + nativeRequire('ses/assert-shim.js'); + nativeRequire('ses/console-shim.js'); } async function getCompartment(scope: ApplicationScope, globals) { @@ -960,14 +970,21 @@ function getHarperExports(scope: ApplicationScope) { User: undefined, }; } -const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES) - ? new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)) - : { - // if we don't have a list of allowed modules, allow everything - has() { - return true; - }, - }; +function getAllowedNodeBuiltinModules() { + return env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES) + ? new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDBUILTINMODULES)) + : { + // if we don't have a list of allowed modules, allow everything + has() { + return true; + }, + }; +} +let ALLOWED_NODE_BUILTIN_MODULES = getAllowedNodeBuiltinModules(); +onStartup(() => { + APPLICATIONS_LOCKDOWN = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN); + ALLOWED_NODE_BUILTIN_MODULES = getAllowedNodeBuiltinModules(); +}); const child_processConstrained: any = { exec: createSpawn(child_process.exec), execFile: createSpawn(child_process.execFile), diff --git a/security/keys.ts b/security/keys.ts index f152f9fe55..9ebde604c6 100644 --- a/security/keys.ts +++ b/security/keys.ts @@ -1,26 +1,28 @@ -'use strict'; +import { loadNativePackage } from '../utility/packageUtils.js'; +('use strict'); -import * as path from 'path'; -import * as fs from 'fs-extra'; +import * as path from 'node:path'; +import fs from 'fs-extra'; import type * as Forge from 'node-forge'; -import * as net from 'net'; +import * as net from 'node:net'; import { generateKeyPair as generateKeyPairOrig, X509Certificate, createPrivateKey, randomBytes } from 'node:crypto'; -import * as util from 'util'; +import * as util from 'node:util'; const generateKeyPair = util.promisify(generateKeyPairOrig); -const loadForge = () => require('node-forge') as typeof Forge; +const loadForge = () => loadNativePackage('node-forge') as typeof Forge; import { v4 as uuidv4 } from 'uuid'; import { forComponent } from '../utility/logging/harper_logger.ts'; import * as envManager from '../utility/environment/environmentManager.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as certificatesTerms from '../utility/terms/certificates.js'; -const tls = require('node:tls'); +import tls from 'node:tls'; import { relative, join, dirname, resolve } from 'node:path'; import assignCmdenvVars from '../utility/assignCmdEnvVariables.ts'; import * as configUtils from '../config/configUtils.ts'; +import { filterArgsAgainstRuntimeConfig } from '../config/harperConfigEnvVars.ts'; import { table, getDatabases, databases } from '../resources/databases.ts'; const logger = forComponent('tls').conditional; const { CONFIG_PARAMS } = hdbTerms; @@ -39,8 +41,8 @@ export const getPrivateKeys = () => { }; import { readFileSync, statSync, watchFile } from 'node:fs'; -import { getTicketKeys, onMessageFromWorkers } from '../server/threads/manageThreads.js'; -import { isMainThread } from 'worker_threads'; +import { getTicketKeys, onMessageFromWorkers } from '../server/threads/manageThreads.ts'; +import { isMainThread } from 'node:worker_threads'; import { POLLING_FALLBACK_OPTIONS, claimLostNativeWatchError, @@ -892,7 +894,6 @@ export function updateConfigCert() { // Filter out any cert config keys already set by HARPER_SET_CONFIG so we don't overwrite them // with defaults. On first boot, HARPER_SET_CONFIG values are written to the config file during // createConfigFile(), but updateConfigCert() runs afterward without re-applying HARPER_SET_CONFIG. - const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars'); const filteredCerts = filterArgsAgainstRuntimeConfig(newCerts); configUtils.updateConfigValue(undefined, undefined, filteredCerts, false, true); diff --git a/security/permissionsTranslator.js b/security/permissionsTranslator.ts similarity index 98% rename from security/permissionsTranslator.js rename to security/permissionsTranslator.ts index de4dc250e2..834b677961 100644 --- a/security/permissionsTranslator.js +++ b/security/permissionsTranslator.ts @@ -1,14 +1,9 @@ -'use strict'; - -const terms = require('../utility/hdbTerms.ts'); -const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.ts'); +import * as terms from '../utility/hdbTerms.ts'; +import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const logger = require('../utility/logging/harper_logger.ts'); - -module.exports = { - getRolePermissions, -}; +import logger from '../utility/logging/harper_logger.ts'; +export { getRolePermissions }; const rolePermsMap = Object.create(null); const permsTemplateObj = (permsKey) => ({ key: permsKey, perms: {} }); diff --git a/security/tokenAuthentication.ts b/security/tokenAuthentication.ts index 41fb5921ef..95dd7cd7e7 100644 --- a/security/tokenAuthentication.ts +++ b/security/tokenAuthentication.ts @@ -30,11 +30,19 @@ import { update } from '../dataLayer/insert.ts'; import UpdateObject from '../dataLayer/UpdateObject.ts'; import { isOperationAuthorizationBypassed } from '../server/serverHelpers/operationAuthorizationState.ts'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); - +import { onStartup } from '../utility/lifecycle.ts'; +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} type StringValue = SignOptions['expiresIn']; -const OPERATION_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_OPERATIONTOKENTIMEOUT) || '1d'; -const REFRESH_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_REFRESHTOKENTIMEOUT) || '30d'; +let OPERATION_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_OPERATIONTOKENTIMEOUT) || '1d'; +let REFRESH_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_REFRESHTOKENTIMEOUT) || '30d'; +onStartup(() => { + OPERATION_TOKEN_TIMEOUT = env.get(CONFIG_PARAMS.AUTHENTICATION_OPERATIONTOKENTIMEOUT) || '1d'; + REFRESH_TOKEN_TIMEOUT = env.get(CONFIG_PARAMS.AUTHENTICATION_REFRESHTOKENTIMEOUT) || '30d'; +}); // Default lifetime of a login-purpose token (see TOKEN_TYPE.LOGIN below). It only exists to be // exchanged for a session cookie, so it defaults far shorter than an operation token; callers can // still override via expires_in. diff --git a/security/user.ts b/security/user.ts index 8894fc4526..fa0f60ad5f 100644 --- a/security/user.ts +++ b/security/user.ts @@ -94,11 +94,13 @@ import * as delete_ from '../dataLayer/delete.ts'; import * as validation from '../validation/user_validation.ts'; import * as search from '../dataLayer/search.ts'; import * as hdbUtility from '../utility/common_utils.ts'; -import * as validate from 'validate.js'; +import _validate from 'validate.js'; +// validate.js is a CJS module; in ESM (typestrip) the default export IS the library object. +const validate: any = (_validate as any).default ?? _validate; import * as logger from '../utility/logging/harper_logger.ts'; -import { promisify } from 'util'; +import { promisify } from 'node:util'; import * as env from '../utility/environment/environmentManager.ts'; -import systemSchema from '../json/systemSchema.json'; +import { systemSchema } from '../utility/systemSchema.js'; import { hdbErrors, ClientError } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES, AUTHENTICATION_ERROR_MSGS, HDB_ERROR_MSGS } = hdbErrors; import cloneDeep from 'lodash/cloneDeep.js'; @@ -119,13 +121,13 @@ import { contextStorage } from '../resources/transaction.ts'; import { deliveryCertified } from '../resources/transactionBroadcast.ts'; import { writeKey } from 'ordered-binary'; -server.getUser = (username: string, password?: string | null): Promise => { +function getUserImpl(username: string, password?: string | null): Promise { return findAndValidateUser(username, password, password != null); -}; +} -server.authenticateUser = (username: string, password?: string | null): Promise => { +function authenticateUserImpl(username: string, password?: string | null): Promise { return findAndValidateUser(username, password); -}; +} const USER_ATTRIBUTE_ALLOWLIST = { username: true, @@ -813,7 +815,7 @@ export function _userChangeSubscription(tableName: string): any { } let invalidateCallbacks = []; -(server as any).invalidateUser = function (user: User | any) { +function invalidateUserImpl(user: User | any) { for (let callback of invalidateCallbacks) { try { callback(user); @@ -821,8 +823,13 @@ let invalidateCallbacks = []; harperLogger.error('Error invalidating user', error); } } -}; +} -server.onInvalidatedUser = function (callback) { +function onInvalidatedUserImpl(callback) { invalidateCallbacks.push(callback); -}; +} + +server.getUser = getUserImpl; +server.authenticateUser = authenticateUserImpl; +server.onInvalidatedUser = onInvalidatedUserImpl; +(server as any).invalidateUser = invalidateUserImpl; diff --git a/server/DESIGN.md b/server/DESIGN.md index a7f72fff64..62ae407a63 100644 --- a/server/DESIGN.md +++ b/server/DESIGN.md @@ -82,14 +82,14 @@ A request entering `http.ts` does **not** go through Fastify unless no Harper ha | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `threads/socketRouter.ts` | Starts the HTTP worker pool (`startHTTPThreads`), dedicated pools such as `replication` (`admittedWorkerPools`) and isolated applications' dedicated workers (`reconcileIsolatedWorkers`); workers bind their own ports, so no socket is routed. | | `threads/workerPools.ts` | Dedicated worker pools: which pools run, which worker type owns a listener (`claimListener`), and whether this thread binds it (`shouldBindListenerHere`). | -| `threads/manageThreads.js` | Thread pool lifecycle. | -| `threads/threadServer.js` | Worker entry point — loads components (`startServers`) and binds each registered server itself (`listenOnPorts`, `reusePort` where the OS has it). | -| `threads/itc.js` | Inter-thread comms primitives. | +| `threads/manageThreads.ts` | Thread pool lifecycle. | +| `threads/threadServer.ts` | Worker entry point — loads components (`startServers`) and binds each registered server itself (`listenOnPorts`, `reusePort` where the OS has it). | +| `threads/itc.ts` | Inter-thread comms primitives. | | `transactionLogCooling.ts` | Main-thread timer that cools transaction-log mmaps. | -Process-wide shutdown begins by calling `beginProcessShutdown()` in `threads/manageThreads.js`. +Process-wide shutdown begins by calling `beginProcessShutdown()` in `threads/manageThreads.ts`. Once set, this terminal state prevents every worker replacement path and makes new `startWorker()` -calls fail with `ERR_HARPER_PROCESS_SHUTTING_DOWN`; scoped worker-type restarts do not set it. +calls fail with `ERR_HARPER_PROCESS_SHUTTING_DOWN`; scoped worker-type restarts do not set it on main. In an outgoing worker, any SHUTDOWN makes that worker terminal. HTTP startup checks the flag after startup/component awaits and before binding or reporting ready. Its idempotent shutdown chain waits for component loading, registered drains, scope disposal and branch-database closure before exit; the existing termination backstops still bound cleanup. Main startup and rolling reloads also check resumed awaits before starting replacements. `shutdownWorkersNow()` remains an immediate teardown: its worker shutdown messages are best-effort, and it force-terminates the remaining worker set rather than waiting for application drain hooks. @@ -102,7 +102,7 @@ many workers it left on the old code because a replacement never came up, versus replacements never started after their predecessor was already gone. Other thread types start their replacement without being awaited. Each wait is bounded by a per-worker startup backstop, so resolution means "the restart finished", not "every worker is new". A caller that treats its own success as -"the component is live" must await it (see `deployComponent` in `components/operations.js`). +"the component is live" must await it (see `deployComponent` in `components/operations.ts`). A restart can hold its replacements, for release certification (components/DESIGN.md, "A restarting deploy is certified by its canary worker"). A held worker runs its boot load, reports `CHILD_COMPONENT_VERDICT`, and binds @@ -112,7 +112,7 @@ a port, and the predecessor is retired at admission. Restarts that start replace (`replacementRestarts`), none starts one while a release is armed (`untilNoCertificationArmed`), and the worker that asked for a certifying restart is replaced last. -> `index.ts` sets `workerData.noServerStart = true` when Harper is imported from a thread it did not spawn, so `threadServer.js` skips `startServers()` there; Harper's own HTTP workers start their servers. +> HTTP worker startup requires `threadServer` to be the actual entry file. Imports from user/job workers do not start listeners; `index.ts` also sets `workerData.noServerStart = true` for unmanaged workers. > > `threadServer.listenOnDomainSocket()` skips a listener only when its path exceeds the platform's > `sockaddr_un.sun_path` byte limit (some Node versions reject it; others silently truncate it). @@ -123,11 +123,27 @@ that asked for a certifying restart is replaced last. Single-instance background tasks pick their thread by what state they touch: -- **Last worker** (`ownsStoreMaintenance(storePath)` in `threads/manageThreads.js`: the last pool worker for shared stores; a dedicated isolated-application worker maintains only its own branch stores) — for tasks that operate on **worker-resident JS state**: audit cleanup (`resources/auditStore.ts`), TTL scans (`resources/Table.ts`) and disk reclamation (`storageReclamation.ts`) walk per-store objects that only exist in a worker. +- **Last worker** (`ownsStoreMaintenance(storePath)` in `threads/manageThreads.ts`: the last pool worker for shared stores; a dedicated isolated-application worker maintains only its own branch stores) — for tasks that operate on **worker-resident JS state**: audit cleanup (`resources/auditStore.ts`), TTL scans (`resources/Table.ts`) and disk reclamation (`storageReclamation.ts`) walk per-store objects that only exist in a worker. - **Main thread** (`isMainThread`) — for tasks that drive a **process-global native singleton** and need no JS state. `transactionLogCooling.ts` is the example: rocksdb-js's transaction-log registry is one C++ static shared across all worker threads, so any thread cools every log. The main thread is chosen because it is the only thread that lives for the whole process — a worker-driven timer would stall whenever that worker is recycled. --- +## Native and compiled runtime startup + +HTTP startup separates worker-owned loading and verdict publication from main-owned admission. Shutdown waits for loading to settle and then runs the existing disposal chain; it never waits for CHILD_ADMITTED. The optional heldStart reporting callback closes the load barrier after the verdict is posted, while whenComponentsLoaded retains its post-admission timing. A rejected bind on a terminal worker follows cleanup instead of forcing an early exit. Scopes registered after shutdown remain outside the existing disposal boundary; this does not cancel initialization. + +Native TypeStrip execution loads the runtime as ESM; compiled distribution files remain CommonJS. Worker paths use `RUNTIME_SRC_ROOT` and `RUNTIME_FILE_EXT` from `utility/packageUtils.js`, which stays CommonJS to resolve its own directory in both modes. `threadMessageState.ts` and `processIncarnation.ts` hold dependency-free worker state: cyclic imports may register callbacks before `manageThreads.ts` evaluates, so that registration must remain synchronous and must not be overwritten by a later initializer. Configuration-dependent wiring and built-in plugin preloads run through `utility/lifecycle.ts` after configuration initialization, before worker listeners bind. Environment-selected built-in overrides survive those preloads. The conditional logger also initializes eagerly so pre-startup warnings remain available, then refreshes in source mode after configuration loads. HTTP workers hold their parent-port ref throughout startup hooks as well as component loading, so an unreferenced asynchronous completion cannot end startup early. The compiled and source routes are exercised by `integrationTests/server/typestrip.test.ts`; native unit probes run through ESM stdin, where no global `module` exists. + +Transaction monitoring and token lifetimes can be imported before configuration is linked. Their startup hooks refresh cached settings, including re-arming the transaction monitor at the configured interval. Job workers also initialize configuration and run startup hooks before executing operations. Configured token lifetimes and builtin restrictions are covered by the two-mode REST test; `integrationTests/database/blob-commit-over-time.test.ts` proves the configured transaction monitor actually ticks in both modes. + +Runtime paths use the canonical parent of the CommonJS helper; the adjacent `hdbTerms.ts` marks source execution. Embedded compiled core keeps its own `dist/core` paths while package metadata and the compiled product daemon entry retain the outer package root. Public relative `startWorker()` paths retain that root and their `.js` default; internal job callers pass complete absolute runtime filenames. `threadServer.ts` automatically starts HTTP only when its own file is the worker entry; loading it from a job or evaluated worker must remain an import. Evaluated workers have a pseudo entry name rather than a real file. These boundaries are pinned by `unitTests/server/threads/typestrip-runtime.test.mjs`. + +Cold SQL dependencies retain synchronous first-use loading through `loadRuntimeModule()` in the intentional CommonJS `packageUtils.js` helper. Its own `__dirname` preserves the caller's path family under `--preserve-symlinks`, avoiding a second module instance. Cached call-site bindings avoid repeated resolution. Hoisting these loads pulls the legacy SQL math library into every worker's boot and exceeds the accepted 64 MB heap minimum. The legacy Fastify plugin uses the existing lazy trusted-plugin loader. Default tagged loggers follow the current main sink when an in-process install replaces it; explicit logger arguments retain their original receiver and method. Authentication refreshes its component logger and arms its cache-reset interval during startup; serving entries must drain startup to bound that cache. Application lockdown/builtin policy, MQTT local authentication, operation forwarding timeouts and CSV download paths also refresh after first-install configuration becomes available. Inspector setup runs once: configured workers initialize at import to preserve module-body breakpoints; the main thread waits for startup after CLI/environment overrides, and unconfigured workers also defer. Both-mode heap, cold SQL/export, denial-log and main-thread first-install integration tests enforce these boundaries. + +Startup drains callbacks in registration order, including callbacks appended by an initializer. A nested initializer runs after previously queued callbacks; failure rejects the cached startup promise and skips remaining initialization. `hasStarted()` attests only that initialization began. Hooks registered after success or failure run independently on the next microtask, preserving the late-import contract. Test resets detach the old queue so an in-flight generation cannot consume a new generation's callbacks. Entry points must initialize configuration and await `runStartup()` before handling requests; the legacy operations launcher follows the same order. Its existing `.js` entry delegates through `#src` to the TypeScript implementation, preserving both source and distribution paths without changing main's existing listener behavior. Managed source workers explicitly carry `--conditions=typestrip` even when the parent supplied it as a command-line flag rather than `NODE_OPTIONS`. + +The twelve declared package globals are also runtime named exports from `index.ts` in source mode. Their owners assign each identity during graph evaluation and subsequently mutate the objects in place. The facade reads the default CommonJS export from `globals.js` after the graph loads: CommonJS synthetic ESM named exports capture early values and cannot supply these identities. Compiled CommonJS retains its existing extra export properties. The direct facade and real user-worker checks cover this boundary; the package's bare `harper` specifier continues to resolve to the compiled distribution. + ## `http.ts` — symbol map Every entry is a top-level function or named const. Jump via go-to-symbol or `grep -n 'function ' server/http.ts`. @@ -395,7 +411,7 @@ Within a pass, `Resource.ts` resolves each username once (`RecheckPass.memo`) an | How are content types (de)serialized? | `serverHelpers/contentTypes.ts` | | Why doesn't every MQTT subscriber re-serialize the message it receives? | `serverHelpers/sharedMessageEncoding.ts` (memoized on the message object the fan-out shares); consumed by the outbound listener in `mqtt.ts` | | Where do durable subscriptions live? | `DurableSubscriptionsSession.ts` | -| How do worker threads get their connections? | Each binds its ports itself after the main thread's first bind (`threads/threadServer.js → listenOnPorts`, `reusePort` where the OS has it); `threads/socketRouter.ts → startHTTPThreads` starts them | +| How do worker threads get their connections? | Each binds its ports itself after the main thread's first bind (`threads/threadServer.ts → listenOnPorts`, `reusePort` where the OS has it); `threads/socketRouter.ts → startHTTPThreads` starts them | | Where is the Operations API wired into Fastify? | `operationsServer.ts → buildServer` | --- @@ -403,7 +419,7 @@ Within a pass, `Resource.ts` resolves each username once (`RecheckPass.memo`) an ## Conventions - Don't add new code to `fastifyRoutes.ts` — it's the legacy custom-functions path. -- New protocol plugins register through the `Server` interface (`Server.ts`): `server.http()`/`request()`/`ws()`/`upgrade()` (bound to `http.ts`'s `httpServer`/`onRequest`/`onWebSocket`/`onUpgrade`) and `server.socket()` (`threadServer.js → onSocket`). +- New protocol plugins register through the `Server` interface (`Server.ts`): `server.http()`/`request()`/`ws()`/`upgrade()` (bound to `http.ts`'s `httpServer`/`onRequest`/`onWebSocket`/`onUpgrade`) and `server.socket()` (`threadServer.ts → onSocket`). - Always pass `name` when registering a listener with `before`/`after` — anonymous entries can't be ordered against. - Tests live in `../unitTests/server/`. @@ -603,7 +619,7 @@ matching Node's upgrade-then-authorize order). No core component registers custo middleware; `onUpgrade()`/`installUwsWsHandler()` warn when one is registered for a uWS-served port so the gap is visible instead of silent. -## A per-thread UDS mirror is bound at a temp name and renamed over its published path (`server/threads/threadServer.js`) +## A per-thread UDS mirror is bound at a temp name and renamed over its published path (`server/threads/threadServer.ts`) libuv unlinks a pipe server's bound path when the handle closes (`uv__pipe_close` → `unlink`), with no check of who owns the path now. On Linux `restartWorkers()` pre-starts the replacement worker @@ -632,13 +648,13 @@ no longer exists, and the published path is never absent between an unlink and a `app.close()` never unlinks a `listen_unix` path, so it is not exposed. Bun restarts are non-overlapping, so the Bun mirror is not exposed either. -## A worker that misses an ITC ack gets its OS thread state logged (`server/threads/manageThreads.js`) +## A worker that misses an ITC ack gets its OS thread state logged (`server/threads/manageThreads.ts`) `broadcastWithAcknowledgement` already times out (30 s) on a worker whose port stays open but never acks, and that shape is almost always a blocked event loop — a native lock, a runaway synchronous call — which nothing inside the worker can report (harper-pro#788: a restarted node's single http worker went byte-silent while main kept serving `cluster_status`, and the app log only said "not acknowledged by worker thread(s) 2"). So each worker posts its Linux thread id (`readlink /proc/thread-self`) to main once at startup, before anything else runs on it, and the timeout branch reads that thread's kernel state from `/proc/self/task/`: state, `wchan`, the syscall number (the first token only — the rest of that file is argument registers and stack/instruction pointers), CPU ticks, and context-switch counts, plus two cross-platform signals main already has, `worker.performance.eventLoopUtilization()` and the age of the last 1 s resource report. It samples again a second later and logs the deltas: no CPU ticks, no context switches and `event loop active +1000ms` is "parked on a lock"; ticks climbing with state `R` is "spinning". It is deliberately main-thread-only and best-effort: `workers` and the tid live on the main thread's `Worker` objects, every `/proc` field is reported individually (a hardened container may deny `wchan`/`syscall` while `stat` stays readable), a follow-up sample whose `starttime` differs from the first is discarded (the tid may have been recycled), one diagnostic runs per worker with a 30 s cooldown so concurrent timeouts on the same worker don't multiply reads, and nothing here runs when acks arrive on time. It does not name the lock owner; that still needs a native stack from the next occurrence. That diagnostic is reactive — it only fires once something else (an ITC ack) has already timed out. `sampleWorkerELU`, run from the same 1 s monitoring tick that already computes `recentELU`, is the proactive counterpart: it warns as soon as a worker's `eventLoopUtilization()` stays `>= 0.99` for `PINNED_ELU_SUSTAINED_TICKS` (30) consecutive ticks, and warns once more on recovery. State (`pinnedELUTicks`, `pinnedELUWarned`) lives on the `Worker` object, so a replaced worker starts a clean streak, and a single tick below threshold resets it — no partial credit across a dip. The very first sample after a worker (re)starts is skipped: `worker.performance.eventLoopUtilization()` called with no prior baseline returns the worker's whole-lifetime total, not a 1 s delta, so counting it could misattribute pre-tick startup work as a pinned tick. That baseline check requires a nonzero `idle` or `active` on the prior sample, not just a prior sample existing: before a worker's loop is online, `eventLoopUtilization()` returns a truthy-but-empty `{ idle: 0, active: 0 }`, and treating that as a real baseline would let the following tick's delta span the same whole-lifetime window. Bun has no `eventLoopUtilization()` and reports a placeholder `{ utilization: 0 }` instead (see `sampleWorkerELU`'s Bun branch), so this warning is Node-only. A worker tearing down can briefly report `idle < 0`, putting `utilization` outside `[0, 1]`; that tick is treated as unmeasured — neither counted nor a streak reset — rather than logging an absurd percentage or ending an episode early. Both log lines name the worker by thread id plus `name`/`application` (e.g. a job or isolated-application worker), since a legitimately saturated worker looks identical to a wedged one from ELU alone; this warns on saturation regardless of cause; distinguishing "busy" from "stuck" is left to the operator. -## A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.js`) +## A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.ts`) `REMOVE_PORT {threadId}` and the `onThreadExit` listeners (database-drop owner release, deploy lifecycle, registered operations, log rotation) act on a thread's death. A sibling port's `close` is taken as the far thread's exit. A worker's `parentPort` carries `threadId = 0` so lookups can route to main, but its `close` means this worker is leaving: main's exit ends the process. `addPort`'s close listener therefore passes no dead thread id for `parentPort`. Announcing one would make every sibling fail its pending acks to main, drop its own `parentPort`, and fire exit listeners for main. Regression cover: `unitTests/server/threads/stuckWorkerDiagnostics.test.js`, "does not report the main thread as exited to a sibling of a worker that closes its parentPort". @@ -668,7 +684,7 @@ The SQL and job paths are additive rather than exclusive: `verifyPermsAST` valid - **Every piped chunk reaches `write()` by property lookup.** Node 26.11's `Readable.pipe` fast path (nodejs/node#66182) calls `_write()` directly when pipe's listener is a byte-mode source's only `'data'` listener, so `compression`'s instance patch of `res.write` was skipped for most chunks of `send(...).pipe(res)`: raw bytes inside the gzip stream, or a stall once `compression` had routed pipe's `'drain'` to its idle zlib stream. On runtimes that have the fast path (`'writeKnownBuffer' in Writable`, a static it added) the response adds an inert `'data'` listener to each byte-mode core `Readable` on `'pipe'`, which fails the fast path's single-listener check, and removes it on `'unpipe'` through a listener prepended in the constructor: pipe's own cleanup resumes a source awaiting `'drain'` while any `'data'` listener remains, which would leave an unpiped source flowing into nothing. Object-mode and decoded sources, and legacy `Stream.prototype.pipe` (which never emits `'unpipe'`), are skipped. Remove it once every supported Node 26 release dispatches pipe through an overridden `write()`. - **Express is not a target.** `express`'s `app.handle()` replaces the response's prototype with one rooted at `http.ServerResponse.prototype`, which no Writable-derived response survives, and would do the same to the request `Proxy`'s target, Harper's real `IncomingMessage`. Middleware that duck-types the response (Next.js, `compression`, `send`, `serve-static`, `finalhandler`, h3, fastify) is the supported surface. -## `manageThreads` has two different `workerCount`s (`server/threads/manageThreads.js`) +## `manageThreads` has two different `workerCount`s (`server/threads/manageThreads.ts`) The module-global `let workerCount` and the per-worker `workerData.workerCount` share a name and nothing else. `getWorkerCount()` (and therefore the `server.workerCount` a component reads) resolves @@ -785,3 +801,9 @@ and last-worker duties never land on one. The invariants: `releaseUnreadUpload`, an `onResponse` hook that `registerContentHandlers` installs beside the parser, discards the rest of such a request: it unpipes and destroys busboy and resumes the parser's input, so Node reads the remaining bytes without parsing them and the connection stays reusable. Response completion is the point because a route may return a lazy stream that reads the payload while the response is sent; operation settlement is too early. A route keeps the part by reading it before its response finishes (piping, iterating, a `data` or `readable` listener); one it took but never started is discarded, and so is one it destroys before its end, even after the response (the part's `'close'` re-enters the discard). A consumer that stops reading without destroying the part, such as a `.pipe()` whose destination failed, still holds it, and the request timeout bounds it. A busboy error after the hand-off, such as a malformed trailer, discards the rest too, since Node unpipes an errored destination, and so does a file part the parser refuses for its name, which never reaches the route: its 400 goes out and the rest is not parsed. A request still arriving a second after its response is destroyed, unless its body has fully arrived by then (the connection may already carry the next request): a bounded grace period for the client to read the answer, chosen over closing at once (a reset can lose the response, RFC 9112 §9.6) and over draining until the request timeout, which `0` disables. The discarded rest still passes through any `preParsing` transform, such as `@fastify/compress`'s decompressor, for at most that second: its `pump` would destroy the request if the transform were torn down instead. The state is keyed by the Fastify request rather than `request.body.payload`, and the grace targets `request.raw`, which a `preParsing` transform can stand in front of. Every part busboy emits gets an `'error'` listener before anything touches it, so an abort after the hand-off is never an uncaught exception. Node's `http` client is a misleading probe here: after a complete response arrives while its request waits for `'drain'`, it never resumes writing, although the server has read every byte it sent. The tests use raw sockets: `unitTests/server/serverHelpers/multipartParser.test.js`, the socket cases in `unitTests/server/serverHelpers/contentTypes.test.js`, and `integrationTests/deploy/deploy-multipart-stream.test.ts`. + +### Configured JSON codec identity + +`serverHelpers/contentTypes.ts` starts with the direct default BigInt codec functions, selects from available configuration at module end, and refreshes once through `onStartup` before listeners bind. The same initializer selects the response-compression threshold, preserving main's direct value lookup on each response rather than a lazy getter. This preserves direct serializer identity for shared-message encoding and removes codec-selection wrappers from each REST response and subscription message. The refresh only replaces media entries that still use the prior default; component-provided serializers remain intact. Native cycles can postpone configuration availability until startup. JSONStream's existing streaming and BigInt behavior is preserved. + +Log rotation's deferred reconfiguration callback loads its runtime module synchronously through `loadRuntimeModule`, so teardown and installation cannot interleave across a cold import. Disabling rotation cannot be undone by an older callback resuming later. diff --git a/server/DurableSubscriptionsSession.ts b/server/DurableSubscriptionsSession.ts index df989de066..0c609b6a57 100644 --- a/server/DurableSubscriptionsSession.ts +++ b/server/DurableSubscriptionsSession.ts @@ -1,12 +1,13 @@ import { table } from '../resources/databases.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { keyArrayToString, resources } from '../resources/Resources.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import { warn, trace } from '../utility/logging/harper_logger.ts'; import { transaction } from '../resources/transaction.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads.js'; -import { whenComponentsLoaded } from '../server/threads/threadServer.js'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; +import { whenComponentsLoaded } from '../server/threads/threadServer.ts'; import { server } from '../server/Server.ts'; -import { RequestTarget } from '../resources/RequestTarget'; +import { RequestTarget } from '../resources/RequestTarget.ts'; import { randomBytes } from 'node:crypto'; import { auditRetention, getDatabaseGeneration, isResumablePosition } from '../resources/auditStore.ts'; @@ -69,7 +70,10 @@ function getLastWill() { } return _LastWill; } -if (getWorkerIndex() === 0) { +// Defer to startup so `whenComponentsLoaded` (an `export const` in threadServer.ts) +// is past TDZ — this module is loaded inside threadServer's static-graph SCC. +onStartup(() => { + if (getWorkerIndex() !== 0) return; (async () => { await whenComponentsLoaded; await new Promise((resolve) => setTimeout(resolve, 2000)); @@ -95,7 +99,7 @@ if (getWorkerIndex() === 0) { } } })(); -} +}); /** * This is used for durable sessions, that is sessions in MQTT that are not "clean" sessions (and with QoS >= 1 diff --git a/server/REST.ts b/server/REST.ts index a79395420c..96450bf907 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -28,7 +28,7 @@ import { import { toCloseReason } from './serverHelpers/webSocketCloseReason.ts'; import { Request } from '../server/serverHelpers/Request.ts'; -import { RequestTarget } from '../resources/RequestTarget'; +import { RequestTarget } from '../resources/RequestTarget.ts'; import { entryMap } from '../resources/RecordEncoder.ts'; const { errorToString, errorForLog } = harperLogger; diff --git a/server/Server.ts b/server/Server.ts index 29851775d5..eceff7611e 100644 --- a/server/Server.ts +++ b/server/Server.ts @@ -1,11 +1,11 @@ -import { Socket } from 'net'; +import type { Socket } from 'node:net'; import { _assignPackageExport } from '../globals.js'; import type { Value } from '../resources/analytics/write.ts'; import type { Resources } from '../resources/Resources.ts'; import type { McpQuotaHandler } from '../components/mcp/quota.ts'; -import { OperationDefinition } from './serverHelpers/serverUtilities.ts'; -import { Duplex } from 'stream'; -import { Request } from './serverHelpers/Request.ts'; +import type { OperationDefinition } from './serverHelpers/serverUtilities.ts'; +import type { Duplex } from 'node:stream'; +import type { Request } from './serverHelpers/Request.ts'; export type HttpListener = (request: Request, nextLayer: (request: Request) => Response) => void; // based on `upgrade` event in Node.js http server: https://nodejs.org/docs/latest/api/http.html#event-upgrade-1 diff --git a/server/fastifyRoutes.ts b/server/fastifyRoutes.ts index 8f8d5053f8..65f4c52f4b 100644 --- a/server/fastifyRoutes.ts +++ b/server/fastifyRoutes.ts @@ -1,22 +1,22 @@ -import { dirname, basename } from 'path'; -import { existsSync } from 'fs'; +import { dirname, basename } from 'node:path'; +import { existsSync } from 'node:fs'; import { deriveRoutePrefix } from './fastifyRoutes/helpers/deriveRoutePrefix.ts'; import { resolveBaseURLPath } from '../components/resolveBaseURLPath.ts'; import fastify from 'fastify'; import fastifyCors from '@fastify/cors'; -import requestTimePlugin from './serverHelpers/requestTimePlugin.js'; +import requestTimePlugin from './serverHelpers/requestTimePlugin.ts'; import autoload from '@fastify/autoload'; import * as env from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; import { realExit } from './threads/workerProcessGuard.ts'; -import * as hdbCore from './fastifyRoutes/plugins/hdbCore.js'; +import * as hdbCore from './fastifyRoutes/plugins/hdbCore.ts'; // installs server.getUser/authenticateUser import '../security/user.ts'; -import getServerOptions from './fastifyRoutes/helpers/getServerOptions.js'; -import getCORSOptions from './fastifyRoutes/helpers/getCORSOptions.js'; -import getHeaderTimeoutConfig from './fastifyRoutes/helpers/getHeaderTimeoutConfig.js'; -import { serverErrorHandler } from '../server/serverHelpers/serverHandlers.js'; +import getServerOptions from './fastifyRoutes/helpers/getServerOptions.ts'; +import getCORSOptions from './fastifyRoutes/helpers/getCORSOptions.ts'; +import getHeaderTimeoutConfig from './fastifyRoutes/helpers/getHeaderTimeoutConfig.ts'; +import { serverErrorHandler } from '../server/serverHelpers/serverHandlers.ts'; import { registerContentHandlers } from '../server/serverHelpers/contentTypes.ts'; import { server } from './Server.ts'; import { registerFastifyInstance } from './http.ts'; diff --git a/server/fastifyRoutes/helpers/getCORSOptions.js b/server/fastifyRoutes/helpers/getCORSOptions.ts similarity index 79% rename from server/fastifyRoutes/helpers/getCORSOptions.js rename to server/fastifyRoutes/helpers/getCORSOptions.ts index de4a205b0a..222afd8b7d 100644 --- a/server/fastifyRoutes/helpers/getCORSOptions.js +++ b/server/fastifyRoutes/helpers/getCORSOptions.ts @@ -1,8 +1,5 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts'); +import * as env from '../../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../../utility/hdbTerms.ts'; /** * Builds CORS options object to pass to cors plugin when/if it needs to be registered with Fastify @@ -33,4 +30,4 @@ function getCORSOptions() { return corsOptions; } -module.exports = getCORSOptions; +export default getCORSOptions; diff --git a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js deleted file mode 100644 index ed4e4400c7..0000000000 --- a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const terms = require('../../../utility/hdbTerms.ts'); - -/** - * Returns header timeout value from config file - * @returns {*} - */ -function getHeaderTimeoutConfig() { - return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000; -} - -module.exports = getHeaderTimeoutConfig; diff --git a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts new file mode 100644 index 0000000000..55bd21cdbc --- /dev/null +++ b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts @@ -0,0 +1,12 @@ +import * as env from '../../../utility/environment/environmentManager.ts'; +import * as terms from '../../../utility/hdbTerms.ts'; + +/** + * Returns header timeout value from config file + * @returns {*} + */ +function getHeaderTimeoutConfig() { + return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000; +} + +export default getHeaderTimeoutConfig; diff --git a/server/fastifyRoutes/helpers/getServerOptions.js b/server/fastifyRoutes/helpers/getServerOptions.ts similarity index 84% rename from server/fastifyRoutes/helpers/getServerOptions.js rename to server/fastifyRoutes/helpers/getServerOptions.ts index 55fb537a52..b7537731fc 100644 --- a/server/fastifyRoutes/helpers/getServerOptions.js +++ b/server/fastifyRoutes/helpers/getServerOptions.ts @@ -1,8 +1,5 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts'); +import * as env from '../../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../../utility/hdbTerms.ts'; // eslint-disable-next-line no-magic-numbers const REQ_MAX_BODY_SIZE = 1024 * 1024 * 1024; //this is 1GB in bytes @@ -34,4 +31,4 @@ function getServerOptions(isHttps) { }; } -module.exports = getServerOptions; +export default getServerOptions; diff --git a/server/fastifyRoutes/plugins/hdbCore.js b/server/fastifyRoutes/plugins/hdbCore.ts similarity index 86% rename from server/fastifyRoutes/plugins/hdbCore.js rename to server/fastifyRoutes/plugins/hdbCore.ts index 8b5e7965d8..4ea45e51fb 100644 --- a/server/fastifyRoutes/plugins/hdbCore.js +++ b/server/fastifyRoutes/plugins/hdbCore.ts @@ -1,12 +1,10 @@ -'use strict'; +import fp from 'fastify-plugin'; -const fp = require('fastify-plugin'); - -const { +import { handlePostRequest, authHandler, reqBodyValidationHandler, -} = require('../../../server/serverHelpers/serverHandlers.js'); +} from '../../../server/serverHelpers/serverHandlers.ts'; /** * Generates a fastify plugin containing three core methods @@ -36,4 +34,4 @@ async function convertAsyncIterators(response) { return response; } -module.exports = fp(hdbCore); +export default fp(hdbCore); diff --git a/server/http.ts b/server/http.ts index a2728c2d21..d3181c8c43 100644 --- a/server/http.ts +++ b/server/http.ts @@ -12,7 +12,7 @@ import { parentPort } from 'node:worker_threads'; import * as env from '../utility/environment/environmentManager.ts'; import * as terms from '../utility/hdbTerms.ts'; import { atomicWriteFile, getConfigPath } from '../config/configUtils.ts'; -import { getTicketKeys, getWorkerIndex } from './threads/manageThreads.js'; +import { getTicketKeys, getWorkerIndex } from './threads/manageThreads.ts'; import { applicationSocketName, isolatedApplicationRoute, @@ -43,7 +43,7 @@ import { recordAction, recordActionBinary } from '../resources/analytics/write.t import { Readable, Writable, pipeline } from 'node:stream'; import { mkdirSync, unlinkSync, readdirSync, statSync, chmodSync } from 'node:fs'; import { join } from 'node:path'; -import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, UpgradeListener } from './Server.ts'; +import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, type UpgradeListener } from './Server.ts'; import { setPortServerMap, SERVERS, socketOptionDefaults } from './serverRegistry.ts'; import { getComponentName } from '../components/componentLoader.ts'; import { throttle } from './throttle.ts'; @@ -51,10 +51,6 @@ import { makeCallbackChain as buildCallbackChain, describeChains, type HttpEntry import { WebSocketServer } from 'ws'; const { errorToString, errorForLog } = harperLogger; -server.http = httpServer; -server.request = onRequest; -server.ws = onWebSocket; -server.upgrade = onUpgrade; const websocketServers = {}; const httpServers = {}, httpChain = {}, @@ -2103,3 +2099,8 @@ export function getRequestId() { } return Number(Atomics.add(nextRequestId, 0, 1n)); } + +server.http = httpServer; +server.request = onRequest; +server.ws = onWebSocket; +server.upgrade = onUpgrade; diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.ts similarity index 82% rename from server/itc/serverHandlers.js rename to server/itc/serverHandlers.ts index 1cbfc8afeb..675bc9bf3c 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.ts @@ -1,42 +1,52 @@ -'use strict'; - +import * as registeredOperationsRuntimeModule from '../serverHelpers/registeredOperations.ts'; +import * as fullTextQueryIndexRuntimeModule from '../../resources/indexes/fullTextQueryIndex.ts'; +import * as indexRuntimeModule from '../../components/status/index.ts'; +import * as manageThreadsRuntimeModule from '../threads/manageThreads.ts'; +import * as ResourcesRuntimeModule from '../../resources/Resources.ts'; +import * as openApiRuntimeModule from '../../resources/openApi.ts'; +import * as httpRuntimeModule from '../http.ts'; /* global threads */ -const hdbLogger = require('../../utility/logging/harper_logger.ts'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const cleanLmdbMap = - require('../../utility/lmdb/cleanLMDBMap.ts').default || require('../../utility/lmdb/cleanLMDBMap.ts'); -const { validateEvent } = require('../threads/itc.js'); -const { +import hdbLogger from '../../utility/logging/harper_logger.ts'; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import _cleanLmdbMap from '../../utility/lmdb/cleanLMDBMap.ts'; +import { validateEvent } from '../threads/itc.ts'; +import { FULL_TEXT_QUERY_PAUSE_OPERATION, FULL_TEXT_QUERY_RESUME_OPERATION, -} = require('../../resources/indexes/fullTextQueryProtocol.ts'); -const { isMainThread, threadId, workerData } = require('node:worker_threads'); -const { +} from '../../resources/indexes/fullTextQueryProtocol.ts'; +import { isMainThread, threadId, workerData } from 'node:worker_threads'; +import { databases, resetDatabases, - closeDatabase, + closeDatabase as _closeDatabase, prepareDatabaseDrop, completeDatabaseDropPreparation, reloadBranchAt, markDropInProgress, -} = require('../../resources/databases.ts'); -const { blockBlobSavesForRestore, resumeBlobSavesAfterRestore } = require('../../resources/blob.ts'); +} from '../../resources/databases.ts'; +import { + blockBlobSavesForRestore as _blockBlobSavesForRestore, + resumeBlobSavesAfterRestore as _resumeBlobSavesAfterRestore, +} from '../../resources/blob.ts'; +const cleanLmdbMap = _cleanLmdbMap; +const closeDatabase = _closeDatabase; +const blockBlobSavesForRestore = _blockBlobSavesForRestore; +const resumeBlobSavesAfterRestore = _resumeBlobSavesAfterRestore; /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. * @type {{schema: ((function(*): Promise)|*), job: ((function(*): Promise)|*)}} */ -const serverItcHandlers = { +const serverItcHandlers: any = { [hdbTerms.ITC_EVENT_TYPES.SCHEMA]: schemaHandler, [hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST]: componentStatusRequestHandler, [hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_REQUEST]: resourceOpenApiRequestHandler, [hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_REQUEST]: middlewareChainsRequestHandler, - // #1736 cross-thread registered-operation bridge. Lazy require: this module loads on every - // thread via itc.js, and registeredOperations pulls in the serverUtilities module graph. + // Read the dispatch binding at delivery because the operation bridge is cyclic. [hdbTerms.ITC_EVENT_TYPES.OPERATION_REGISTERED]: (event) => - require('../serverHelpers/registeredOperations.ts').operationRegisteredHandler(event), + registeredOperationsRuntimeModule.operationRegisteredHandler(event), [hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_REQUEST]: (event) => - require('../serverHelpers/registeredOperations.ts').operationExecuteRequestHandler(event), + registeredOperationsRuntimeModule.operationExecuteRequestHandler(event), }; /** @@ -44,8 +54,8 @@ const serverItcHandlers = { * @param event * @returns {Promise} */ -const schemaListeners = []; -async function schemaHandler(event) { +const schemaListeners: any = []; +async function schemaHandler(event?: any) { const validate = validateEvent(event); if (validate) { hdbLogger.error(validate); @@ -66,7 +76,7 @@ async function schemaHandler(event) { typeof event.message.allowUnregisteredReadiness !== 'boolean' ) throw new Error('Full-text query reader coordination requires a boolean unregistered-readiness flag'); - const fullTextQueries = require('../../resources/indexes/fullTextQueryIndex.ts'); + const fullTextQueries = fullTextQueryIndexRuntimeModule; const ownerEpoch = BigInt(event.message.ownerEpoch); if (event.message.operation === FULL_TEXT_QUERY_PAUSE_OPERATION) await fullTextQueries.pauseNativeFullTextQueryReaders( @@ -144,7 +154,7 @@ async function schemaHandler(event) { } } -schemaHandler.addListener = function (listener) { +schemaHandler.addListener = function (listener?: any) { schemaListeners.push(listener); return () => { const index = schemaListeners.indexOf(listener); @@ -159,7 +169,7 @@ schemaHandler.addListener = function (listener) { * @param msg * @returns {Promise} */ -async function syncSchemaMetadata(msg) { +async function syncSchemaMetadata(msg?: any) { try { // A change to a scope-private branch is not a change to any database in the global map, so the // rescan below has nothing to find; a thread holding that branch open reloads it instead. @@ -197,13 +207,13 @@ async function syncSchemaMetadata(msg) { const rescanned = resetDatabases(); if (msg.table && msg.database) // wait for a write to finish to ensure all writes have been written - await rescanned[msg.database][msg.table].put(Symbol.for('write-verify'), null); + await rescanned[msg.database][msg.table].put(Symbol.for('write-verify') as any, null); } catch (e) { hdbLogger.error(e); } } -const resourceListeners = []; +const resourceListeners: any = []; /** * Local-only fan-out for "JS resources just registered" (resources.js loaded via the jsResource * plugin). Unlike schema/user changes this is NOT an ITC event: each worker loads and registers @@ -220,7 +230,7 @@ function resourceHandler() { } } } -resourceHandler.addListener = function (listener) { +resourceHandler.addListener = function (listener?: any) { resourceListeners.push(listener); }; // Test seam: drop registered listeners so a unit suite doesn't leak fakes into later suites. @@ -241,7 +251,7 @@ resourceHandler._resetListenersForTest = function () { * @param {string} event.message.requestId - The unique identifier for the request. * @returns {Promise} Sends a response back to the originator thread or logs an error if validation fails. */ -async function componentStatusRequestHandler(event) { +async function componentStatusRequestHandler(event?: any) { try { const validate = validateEvent(event); if (validate) { @@ -252,8 +262,8 @@ async function componentStatusRequestHandler(event) { hdbLogger.trace(`ITC componentStatusRequestHandler received request:`, event); // Get current thread's component status - const { internal } = require('../../components/status/index.ts'); - const { getWorkerIndex } = require('../threads/manageThreads.js'); + const { internal } = indexRuntimeModule; + const { getWorkerIndex } = manageThreadsRuntimeModule; const componentStatuses = internal.componentStatusRegistry.getAllStatuses(); // Convert Map to array for serialization @@ -266,7 +276,7 @@ async function componentStatusRequestHandler(event) { // Send response directly back to the originating thread. validateEvent already // ensures originator is present. const originatorThreadId = event.message.originator; - const responseMessage = { + const responseMessage: any = { type: hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_RESPONSE, message: { requestId: event.message.requestId, @@ -297,7 +307,7 @@ async function componentStatusRequestHandler(event) { * Generates the spec from the local resources (which are only registered on worker threads) * and sends it back to the requesting thread. */ -async function resourceOpenApiRequestHandler(event) { +async function resourceOpenApiRequestHandler(event?: any) { try { const validate = validateEvent(event); if (validate) { @@ -307,17 +317,17 @@ async function resourceOpenApiRequestHandler(event) { hdbLogger.trace(`ITC resourceOpenApiRequestHandler received request:`, event); - const { resources } = require('../../resources/Resources.ts'); + const { resources } = ResourcesRuntimeModule; // Only respond if this thread has registered resources. Job-type workers with an empty // resources map must stay silent so that an app worker with real resources replies first. // If no worker has resources the main thread gets a 503 after the timeout, which is a // more honest response than silently returning an empty spec. if (!resources || resources.size === 0) return; - const { generateJsonApi } = require('../../resources/openApi.ts'); + const { generateJsonApi } = openApiRuntimeModule; const openapi = generateJsonApi(resources, event.message.serverHttpURL); const originatorThreadId = event.message.originator; - const responseMessage = { + const responseMessage: any = { type: hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_RESPONSE, message: { requestId: event.message.requestId, @@ -342,7 +352,7 @@ async function resourceOpenApiRequestHandler(event) { * they have no middleware (empty chains) so get_status doesn't wait out the request timeout. First * worker to answer wins — all HTTP workers register identically, so any one is representative. */ -async function middlewareChainsRequestHandler(event) { +async function middlewareChainsRequestHandler(event?: any) { try { const validate = validateEvent(event); if (validate) { @@ -351,9 +361,9 @@ async function middlewareChainsRequestHandler(event) { } if (isMainThread || workerData?.name !== hdbTerms.THREAD_TYPES.HTTP) return; - const { describeMiddlewareChains } = require('../http.ts'); + const { describeMiddlewareChains } = httpRuntimeModule; - const responseMessage = { + const responseMessage: any = { type: hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_RESPONSE, message: { requestId: event.message.requestId, @@ -371,7 +381,10 @@ async function middlewareChainsRequestHandler(event) { } } -module.exports = serverItcHandlers; +export default serverItcHandlers; // Named exports so consumers (e.g., MCP listChanged) can subscribe via `schemaHandler.addListener(fn)`. -module.exports.schemaHandler = schemaHandler; -module.exports.resourceHandler = resourceHandler; +export { schemaHandler }; +export { resourceHandler }; + +// Compiled consumers dispatch directly by event type, including `.schema`. +if (typeof module !== 'undefined') Object.assign(module.exports, serverItcHandlers); diff --git a/server/jobs/jobOwnership.ts b/server/jobs/jobOwnership.ts index dbb1d24f27..cab09e436a 100644 --- a/server/jobs/jobOwnership.ts +++ b/server/jobs/jobOwnership.ts @@ -3,7 +3,7 @@ import { getDatabases } from '../../resources/databases.ts'; import * as hdbTerms from '../../utility/hdbTerms.ts'; import log from '../../utility/logging/harper_logger.ts'; -import * as manageThreads from '../threads/manageThreads.js'; +import { processIncarnation } from '../threads/processIncarnation.ts'; import { updateJob } from './jobs.ts'; /** @@ -11,7 +11,7 @@ import { updateJob } from './jobs.ts'; * dead job look alive. Every thread of one Harper process carries the same value, and any restart * produces a new one. `undefined` on a thread started without one. */ -export const JOB_OWNER_INSTANCE_ID: string | undefined = manageThreads.processIncarnation; +export const JOB_OWNER_INSTANCE_ID: string | undefined = processIncarnation; /** Stripped from `get_job` responses. */ export const JOB_OWNER_ATTRIBUTES = ['owner_instance', 'owner_pid'] as const; diff --git a/server/jobs/jobProcess.ts b/server/jobs/jobProcess.ts index 4d1f0a0950..b3f0eb72be 100644 --- a/server/jobs/jobProcess.ts +++ b/server/jobs/jobProcess.ts @@ -18,9 +18,11 @@ import cloneDeep from 'lodash/cloneDeep.js'; import { pathToFileURL } from 'node:url'; import { join } from 'node:path'; import { parentPort } from 'node:worker_threads'; -import { notifyJobCleanupComplete } from '../threads/manageThreads.js'; +import { notifyJobCleanupComplete } from '../threads/manageThreads.ts'; import { getEnvBuiltInComponents } from './../../components/Application.ts'; import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; +import * as env from '../../utility/environment/environmentManager.ts'; +import { runStartup } from '../../utility/lifecycle.ts'; const JOB_NAME = process.env[(hdbTerms as any).PROCESS_NAME_ENV_PROP] as string; const JOB_ID = JOB_NAME.substring(4); @@ -34,8 +36,10 @@ const JOB_ID = JOB_NAME.substring(4); let jobObj: any = { id: JOB_ID, request: undefined }; let exitCode = 0; try { + env.initSync(); harperLogger.notify('Starting job:', JOB_ID); globalSchema.setSchemaDataToGlobal(); + await runStartup(); for (const { packageIdentifier } of getEnvBuiltInComponents()) { if (packageIdentifier.startsWith('@/')) { diff --git a/server/jobs/jobRunner.ts b/server/jobs/jobRunner.ts index 039f0a554c..8c87d9c823 100644 --- a/server/jobs/jobRunner.ts +++ b/server/jobs/jobRunner.ts @@ -1,7 +1,5 @@ 'use strict'; -import { join } from 'node:path'; - import * as hdbUtil from '../../utility/common_utils.ts'; import * as hdbTerms from '../../utility/hdbTerms.ts'; import * as bulkLoad from '../../dataLayer/bulkLoad.ts'; @@ -10,11 +8,13 @@ import * as jobs from './jobs.ts'; import * as hdbExport from '../../dataLayer/export.ts'; import * as hdbDelete from '../../dataLayer/delete.ts'; import * as rocksdbBackup from '../../dataLayer/rocksdbBackup.ts'; -import * as threadsStart from '../threads/manageThreads.js'; +import * as threadsStart from '../threads/manageThreads.ts'; import * as transactionLog from '../../utility/logging/transactionLog.ts'; import * as restart from '../../bin/restart.ts'; -import { parentPort, isMainThread } from 'worker_threads'; -import { onMessageByType } from '../threads/manageThreads.js'; +import { parentPort, isMainThread } from 'node:worker_threads'; +import { onMessageByType } from '../threads/manageThreads.ts'; +import { join } from 'node:path'; +import { RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; class RunnerMessage { job: any; @@ -142,7 +142,7 @@ async function runJob(runnerMessage: any, operation: any) { async function launchJobThread(job_id: any) { log.trace('launching job thread:', job_id); if (isMainThread) { - threadsStart.startWorker(join(__dirname, './jobProcess.js'), { + threadsStart.startWorker(join(RUNTIME_SRC_ROOT, `server/jobs/jobProcess${RUNTIME_FILE_EXT}`), { autoRestart: false, name: 'job', env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${job_id}` }, @@ -157,7 +157,7 @@ async function launchJobThread(job_id: any) { if (isMainThread) { onMessageByType(hdbTerms.ITC_EVENT_TYPES.START_JOB, async (message) => { try { - threadsStart.startWorker(join(__dirname, './jobProcess.js'), { + threadsStart.startWorker(join(RUNTIME_SRC_ROOT, `server/jobs/jobProcess${RUNTIME_FILE_EXT}`), { autoRestart: false, name: 'job', env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${message.jobId}` }, diff --git a/server/jobs/jobs.ts b/server/jobs/jobs.ts index 402141b5e5..f9c272d27f 100644 --- a/server/jobs/jobs.ts +++ b/server/jobs/jobs.ts @@ -1,4 +1,5 @@ -'use strict'; +import { loadNativePackage } from '../../utility/packageUtils.js'; +import { loadRuntimeModule } from '../../utility/packageUtils.js'; /** * The jobs class is used to enable operations on the jobs system table. The jobHandler function is the only @@ -18,7 +19,7 @@ import UpdateObject from '../../dataLayer/UpdateObject.ts'; import log from '../../utility/logging/harper_logger.ts'; import Insert_Object from '../../dataLayer/InsertObject.ts'; import * as hdbUtil from '../../utility/common_utils.ts'; -import { promisify } from 'util'; +import { promisify } from 'node:util'; import * as fileLoadValidator from '../../validation/fileLoadValidator.ts'; import * as rocksdbBackup from '../../dataLayer/rocksdbBackup.ts'; import bulkDeleteValidator from '../../validation/bulkDeleteValidator.ts'; @@ -58,7 +59,7 @@ export async function handleGetJobsByStartDate(jsonBody: any) { let result = await getJobsInDateRange(jsonBody); log.trace(`Searching for jobs from ${jsonBody.from_date} to ${jsonBody.to_date}`); if (result && result.length > 0) { - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); for (let currRes of result) { if (currRes.start_datetime) { currRes.start_datetime_converted = moment(currRes.start_datetime); @@ -243,7 +244,7 @@ export async function addJob(jsonBody: any) { * @returns {Promise<*>} */ export async function getJobsInDateRange(jsonBody: any) { - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); let parsedFromDate = moment(jsonBody.from_date, moment.ISO_8601); let parsedToDate = moment(jsonBody.to_date, moment.ISO_8601); @@ -259,7 +260,7 @@ export async function getJobsInDateRange(jsonBody: any) { try { if (!pSqlEvaluate) { - const hdbSql = require('../../sqlTranslator/index'); + const hdbSql = loadRuntimeModule('sqlTranslator/index'); pSqlEvaluate = promisify(hdbSql.evaluateSQL); } // Harper's own statement, not the caller's. A carrier is not equivalent here — see DESIGN.md. diff --git a/server/liveSubscriptionAuth.ts b/server/liveSubscriptionAuth.ts index b0baadd62c..eb9fe2c078 100644 --- a/server/liveSubscriptionAuth.ts +++ b/server/liveSubscriptionAuth.ts @@ -1,3 +1,4 @@ +import * as userRuntimeModule from '../security/user.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import { coalesceRefresh } from '../utility/coalesceRefresh.ts'; import type { UserChange } from '../security/user.ts'; @@ -174,7 +175,7 @@ function ensureStarted(): void { } if (!userChangeListenerInstalled) { try { - securityUser ??= require('../security/user'); + securityUser ??= userRuntimeModule; securityUser!.onUserChange(onUserChanged); userChangeListenerInstalled = true; } catch (error) { @@ -194,7 +195,7 @@ function stopIfIdle(): void { function indexRole(group: PrincipalGroup): void { let roleId: unknown; try { - securityUser ??= require('../security/user'); + securityUser ??= userRuntimeModule; roleId = securityUser!.userRecordVersions(group.username).roleId; } catch { roleId = ANY_ROLE; diff --git a/server/loadRootComponents.js b/server/loadRootComponents.ts similarity index 76% rename from server/loadRootComponents.js rename to server/loadRootComponents.ts index e54623763c..23de4a89a0 100644 --- a/server/loadRootComponents.js +++ b/server/loadRootComponents.ts @@ -1,31 +1,33 @@ -const { isMainThread, workerData } = require('worker_threads'); -const { getTables } = require('../resources/databases.ts'); -const { +import { loadRuntimeModule } from '../utility/packageUtils.js'; +import * as jobOwnershipRuntimeModule from './jobs/jobOwnership.ts'; +import { isMainThread, workerData } from 'node:worker_threads'; +import { getTables } from '../resources/databases.ts'; +import { loadComponentDirectories, loadComponent, readyComponentModules, startSecretCustodyOnMainThread, -} = require('../components/componentLoader.ts'); -const { resetResources } = require('../resources/Resources.ts'); -const configUtils = require('../config/configUtils.ts'); -const { dirname } = require('path'); -const { loadCertificates } = require('../security/keys.ts'); -const { installApplications, recoverInterruptedActivations } = require('../components/Application.ts'); -const { failClosedReleases, liveDeploymentId, RejectedReleaseError } = require('../components/releaseCertification.ts'); -const { errorForLog } = require('../utility/logging/harper_logger.ts'); -const { CONFIG_PARAMS } = require('../utility/hdbTerms.ts'); +} from '../components/componentLoader.ts'; +import { resetResources } from '../resources/Resources.ts'; +import * as configUtils from '../config/configUtils.ts'; +import { dirname } from 'node:path'; +import { loadCertificates } from '../security/keys.ts'; +import { installApplications, recoverInterruptedActivations } from '../components/Application.ts'; +import { failClosedReleases, liveDeploymentId, RejectedReleaseError } from '../components/releaseCertification.ts'; +import { errorForLog } from '../utility/logging/harper_logger.ts'; +import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; -let loadedComponents = new Map(); +let loadedComponents: any = new Map(); /** * Map every component directory to the reason activation recovery could not run. Used only when the scan * fails globally, where the safe answer is "none of these are known-good" rather than "all of these are". */ -async function failEveryComponentClosed(cause) { - const failures = new Map(); +async function failEveryComponentClosed(cause?: any) { + const failures: any = new Map(); const reason = cause instanceof Error ? cause : new Error(`Activation recovery could not run: ${String(cause)}`); try { - const { readdir } = require('node:fs/promises'); + const { readdir } = await import('node:fs/promises'); const componentsRoot = configUtils.getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); for (const entry of await readdir(componentsRoot, { withFileTypes: true })) { if (!entry.name.startsWith('.') && (entry.isDirectory() || entry.isSymbolicLink())) { @@ -41,7 +43,7 @@ async function failEveryComponentClosed(cause) { * This is main entry point for loading the main set of global server modules that power Harper. * @returns {Promise} */ -async function loadRootComponents(isWorkerThread = false) { +async function loadRootComponents(isWorkerThread: any = false) { // Interrupted activations are settled FIRST, before installApplications() — not merely before the // component scan. installApplications() installs whatever the root config names, so a candidate left // half-swapped by a crash has to be resolved before that runs, or boot reinstalls over it. Failures are @@ -67,7 +69,7 @@ async function loadRootComponents(isWorkerThread = false) { // reads the config a restore publishes. if (isMainThread && !process.env.HARPER_SAFE_MODE) { try { - await require('../components/canaryRollout.ts').resolveInterruptedCertifications(); + await loadRuntimeModule('components/canaryRollout').resolveInterruptedCertifications(); } catch (error) { console.error(errorForLog(error)); } @@ -92,9 +94,7 @@ async function loadRootComponents(isWorkerThread = false) { // An unreconciled job row is misleading, but it is no reason to refuse to start. This function re-runs // on every root component reload; the sweep itself holds the once-per-process guard. if (isMainThread) { - require('./jobs/jobOwnership.ts') - .reconcileInterruptedJobsOnce() - .catch((error) => console.error(errorForLog(error))); + jobOwnershipRuntimeModule.reconcileInterruptedJobsOnce().catch((error) => console.error(errorForLog(error))); } await loadCertificates(); @@ -107,7 +107,7 @@ async function loadRootComponents(isWorkerThread = false) { }); if (!process.env.HARPER_SAFE_MODE) { // once the global plugins are loaded, we now load all the CF and run applications (and their components) - const readyComponentPromises = new WeakMap(); + const readyComponentPromises: any = new WeakMap(); await loadComponentDirectories( loadedComponents, resources, @@ -127,7 +127,7 @@ async function loadRootComponents(isWorkerThread = false) { */ async function componentsFailedClosed() { const componentsRoot = configUtils.getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); - const { processIncarnation } = require('./threads/manageThreads.js'); + const { processIncarnation } = loadRuntimeModule('server/threads/manageThreads'); let failClosed; try { failClosed = await failClosedReleases(componentsRoot, processIncarnation); @@ -136,11 +136,14 @@ async function componentsFailedClosed() { failClosed = await failEveryComponentClosed(error); } // Each holds only while the release it refused is still the live one. - for (const [component, { deploymentId, reason }] of Object.entries(workerData?.failClosed ?? {})) { + for (const [component, { deploymentId, reason }] of Object.entries(workerData?.failClosed ?? {}) as [ + string, + { deploymentId: string; reason: string }, + ][]) { if ((await liveDeploymentId(componentsRoot, component).catch(() => undefined)) !== deploymentId) continue; failClosed.set(component, new RejectedReleaseError(component, deploymentId, reason)); } return failClosed; } -module.exports.loadRootComponents = loadRootComponents; +export { loadRootComponents }; diff --git a/server/mqtt.ts b/server/mqtt.ts index ee5e17cca1..a0a82fc958 100644 --- a/server/mqtt.ts +++ b/server/mqtt.ts @@ -1,6 +1,7 @@ // for now we are using mqtt-packet, but we may implement some of this ourselves, particularly packet generation so that // we can implement more efficient progressive buffer allocation. -import { parser as makeParser, generate } from 'mqtt-packet'; +import _mqtt_packet from 'mqtt-packet'; +const { parser: makeParser, generate } = _mqtt_packet; import { getSession, DurableSubscriptionsSession } from './DurableSubscriptionsSession.ts'; import { getSuperUser } from '../security/user.ts'; import { getDeserializer } from './serverHelpers/contentTypes.ts'; @@ -16,9 +17,10 @@ import { get } from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS, AUTH_AUDIT_STATUS, AUTH_AUDIT_TYPES } from '../utility/hdbTerms.ts'; import { loggerWithTag } from '../utility/logging/logger.ts'; import { forComponent as loggerForComponent } from '../utility/logging/harper_logger.ts'; -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; import { verifyCertificate } from '../security/certificateVerification/index.ts'; import { registerShutdownDrain } from '../components/shutdownDrain.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { toCloseReason } from './serverHelpers/webSocketCloseReason.ts'; import { DatabaseGenerationChangedError, ResumeHistoryUnavailableError } from '../utility/errors/hdbError.ts'; import { @@ -33,6 +35,9 @@ const authEventLog = loggerWithTag('auth-event'); const mqttLog = loggerForComponent('mqtt'); let AUTHORIZE_LOCAL = get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? process.env.DEV_MODE; +onStartup(() => { + AUTHORIZE_LOCAL = get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? process.env.DEV_MODE; +}); export function bypassAuth() { AUTHORIZE_LOCAL = true; } diff --git a/server/operationsServer.ts b/server/operationsServer.ts index 28d3ede83b..f22a432b88 100644 --- a/server/operationsServer.ts +++ b/server/operationsServer.ts @@ -1,17 +1,26 @@ // @ts-nocheck -import cluster from 'cluster'; +import cluster from 'node:cluster'; import zlib from 'node:zlib'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} import * as terms from '../utility/hdbTerms.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { realExit } from './threads/workerProcessGuard.ts'; -import fastify, { FastifyInstance, FastifyReply, FastifyRequest, FastifyServerOptions } from 'fastify'; +import fastify, { + type FastifyInstance, + type FastifyReply, + type FastifyRequest, + type FastifyServerOptions, +} from 'fastify'; import fastifyCors, { type FastifyCorsOptions } from '@fastify/cors'; import fastifyCompress from '@fastify/compress'; import fastifyStatic from '@fastify/static'; -import requestTimePlugin from './serverHelpers/requestTimePlugin.js'; -import guidePath from 'path'; +import requestTimePlugin from './serverHelpers/requestTimePlugin.ts'; +import guidePath from 'node:path'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; import * as globalSchema from '../utility/globalSchema.ts'; import * as commonUtils from '../utility/common_utils.ts'; @@ -24,7 +33,7 @@ import { handlePostRequest, serverErrorHandler, reqBodyValidationHandler, -} from './serverHelpers/serverHandlers.js'; +} from './serverHelpers/serverHandlers.ts'; import { registerFastifyInstance } from './http.ts'; import { registerContentHandlers } from './serverHelpers/contentTypes.ts'; import { getConfigObj } from '../config/configUtils.ts'; @@ -34,8 +43,8 @@ type ParsedSqlObject = any; import { generateJsonApi } from '../resources/openApi.ts'; import { Resources } from '../resources/Resources.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; -import { sendItcEvent } from './threads/itc.js'; -import { onMessageByType } from './threads/manageThreads.js'; +import { sendItcEvent } from './threads/itc.ts'; +import { onMessageByType } from './threads/manageThreads.ts'; const DEFAULT_HEADERS_TIMEOUT = 60000; const REQ_MAX_BODY_SIZE = env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_MAXREQUESTBODYSIZE) ?? 1024 * 1024 * 1024; //this defaults to 1GB in bytes @@ -50,7 +59,7 @@ export { operationsServer as startOnMainThread }; /** * Builds a Harper server. */ -async function operationsServer(options: ServerOptions & { resources?: Resources }) { +async function operationsServer(options: ServerOptions & { resources?: Resources } = {}) { try { harperLogger.debug('In Fastify server' + process.cwd()); harperLogger.debug(`Running with NODE_ENV set as: ${process.env.NODE_ENV}`); diff --git a/server/serverHelpers/contentTypes.ts b/server/serverHelpers/contentTypes.ts index 93c8f96477..9cbe431730 100644 --- a/server/serverHelpers/contentTypes.ts +++ b/server/serverHelpers/contentTypes.ts @@ -1,7 +1,7 @@ import { streamAsJSON, stringify, parse } from './JSONStream.ts'; import { pack, unpack, encodeIter } from 'msgpackr'; import { decode, Encoder, EncoderStream } from 'cbor-x'; -import { createBrotliCompress, brotliCompress, constants } from 'zlib'; +import { createBrotliCompress, brotliCompress, constants } from 'node:zlib'; import { ClientError } from '../../utility/errors/hdbError.ts'; import stream, { Readable, Transform } from 'node:stream'; import { server } from '../Server.ts'; @@ -18,9 +18,10 @@ import { Blob } from '../../resources/blob.ts'; // TODO: Only load this if fastify is loaded import fp from 'fastify-plugin'; import { parseMultipartRequest, releaseUnreadUpload } from './multipartParser.ts'; -const SERIALIZATION_BIGINT = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false; -const JSONStringify = SERIALIZATION_BIGINT ? stringify : JSON.stringify; -const JSONParse = SERIALIZATION_BIGINT ? parse : JSON.parse; +import { onStartup } from '../../utility/lifecycle.ts'; +let JSONStringify: typeof JSON.stringify = stringify; +let JSONParse: typeof JSON.parse = parse; +let COMPRESSION_THRESHOLD: number | undefined; const streamStartup = Symbol('streamStartup'); const serializedStreamError = Symbol('serializedStreamError'); @@ -259,6 +260,22 @@ const genericHandler = { }; mediaTypes.set('*/*', genericHandler); mediaTypes.set('', genericHandler); +function initializeSerialization() { + const previousStringify = JSONStringify; + const useBigInt = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false; + JSONStringify = useBigInt ? stringify : JSON.stringify; + JSONParse = useBigInt ? parse : JSON.parse; + for (const codec of mediaTypes.values()) { + if (codec.serialize === previousStringify) codec.serialize = JSONStringify; + } + COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD); +} +try { + initializeSerialization(); +} catch (error) { + if (!(error instanceof ReferenceError)) throw error; +} +onStartup(initializeSerialization); // try to JSON parse, but since we don't know for sure, this will return the body // otherwise function tryJSONParse(input) { @@ -432,7 +449,6 @@ export function findBestSerializer(incomingMessage) { return { serializer: bestSerializer, type: bestType, parameters: bestParameters }; } -const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD); const brotliParams = (mode: number) => ({ params: { [constants.BROTLI_PARAM_MODE]: mode, [constants.BROTLI_PARAM_QUALITY]: 2 }, }); diff --git a/server/serverHelpers/operationDispatchState.ts b/server/serverHelpers/operationDispatchState.ts new file mode 100644 index 0000000000..fd37a3c62a --- /dev/null +++ b/server/serverHelpers/operationDispatchState.ts @@ -0,0 +1,6 @@ +export interface LocalOperationDispatch { + chooseOperation: (body: any, bypassAuth?: boolean) => Function; + processLocalTransaction: (req: any, operationFunction: Function) => Promise; +} + +export const operationDispatchState: { local?: LocalOperationDispatch } = {}; diff --git a/server/serverHelpers/registeredOperations.ts b/server/serverHelpers/registeredOperations.ts index e62ed51124..f83f4a6ba0 100644 --- a/server/serverHelpers/registeredOperations.ts +++ b/server/serverHelpers/registeredOperations.ts @@ -1,3 +1,4 @@ +import { operationDispatchState, type LocalOperationDispatch } from './operationDispatchState.ts'; /** * Cross-thread bridge for `server.registerOperation()` (#1736). * @@ -25,13 +26,14 @@ import * as terms from '../../utility/hdbTerms.ts'; import * as env from '../../utility/environment/environmentManager.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; import { ServerError } from '../../utility/errors/hdbError.ts'; -import { sendItcEvent } from '../threads/itc.js'; -import { hasThreadExited, onMessageByType, onThreadExit } from '../threads/manageThreads.js'; +import { sendItcEvent } from '../threads/itc.ts'; +import { hasThreadExited, onMessageByType, onThreadExit } from '../threads/manageThreads.ts'; import { registerWorkerGrantableOperation, unregisterWorkerGrantableOperation, } from '../../utility/operationPermissions.ts'; import { runWithOperationAuthorizationBypass } from './operationAuthorizationState.ts'; +import { onStartup } from '../../utility/lifecycle.ts'; const operationLog = harperLogger.loggerWithTag('operation'); @@ -47,18 +49,17 @@ const NON_FORWARDABLE_FIELDS = ['baseRequest', 'baseResponse', 'fastifyResponse' // Bound how long the main thread waits for a worker to finish a forwarded operation. Past the // operations-API connection timeout the client socket is gone anyway; this just prevents a // wedged-but-alive worker from leaking pending forwards forever. -const EXECUTE_TIMEOUT_MS = env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_TIMEOUT) || 120_000; +let EXECUTE_TIMEOUT_MS = env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_TIMEOUT) || 120_000; +onStartup(() => { + EXECUTE_TIMEOUT_MS = env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_TIMEOUT) || 120_000; +}); // Dispatch functions injected by serverUtilities at its module load (it statically imports this // module, so a plain import here would be a cycle; a runtime require of a .ts path doesn't // survive the dist build). A worker can only receive an execute request after announcing a // registration — which goes through serverUtilities — so these are always set on that path. -let localDispatch: { - chooseOperation: (body: any, bypassAuth?: boolean) => Function; - processLocalTransaction: (req: any, operationFunction: Function) => Promise; -}; -export function setLocalOperationDispatch(dispatch: typeof localDispatch) { - localDispatch = dispatch; +export function setLocalOperationDispatch(dispatch: LocalOperationDispatch) { + operationDispatchState.local = dispatch; } /** name -> threadIds of workers that registered it (main thread only) */ @@ -264,6 +265,7 @@ export async function operationExecuteRequestHandler(event: { const { requestId, body, bypassAuth, originator } = event.message; let response; try { + const localDispatch = operationDispatchState.local; if (!localDispatch) throw new ServerError('This worker thread cannot execute operations', 503); // Authorization state travels in the trusted same-process ITC envelope, never in the // caller-controlled operation body. This preserves server.operation(..., false) across diff --git a/server/serverHelpers/requestTimePlugin.js b/server/serverHelpers/requestTimePlugin.ts similarity index 86% rename from server/serverHelpers/requestTimePlugin.js rename to server/serverHelpers/requestTimePlugin.ts index c369b48ec1..67e956741d 100644 --- a/server/serverHelpers/requestTimePlugin.js +++ b/server/serverHelpers/requestTimePlugin.ts @@ -1,10 +1,10 @@ -const { recordAction, recordActionBinary } = require('../../resources/analytics/write.ts'); -const fp = require('fastify-plugin'); +import { recordAction, recordActionBinary } from '../../resources/analytics/write.ts'; +import fp from 'fastify-plugin'; const ESTIMATED_HEADER_SIZE = 200; // it is very expensive to actually measure HTTP response header size (we change it // ourselves) with an unacceptable performance penalty, so we estimate this part -module.exports = fp( +export default fp( function (fastify, opts, done) { // eslint-disable-next-line require-await fastify.addHook('onResponse', async (request, reply) => { @@ -12,15 +12,15 @@ module.exports = fp( let _time = reply.elapsedTime; }); // eslint-disable-next-line require-await - fastify.addHook('onSend', async (request, reply, payload) => { + fastify.addHook('onSend', async (request, reply, payload: any) => { let responseTime = reply.elapsedTime; let startTransfer = performance.now(); - let config = reply.request.routeOptions; + let config: any = reply.request.routeOptions; let action; let type; let method; if (config.config?.isOperation) { - action = request.body?.operation; + action = (request.body as any)?.operation; type = 'operation'; } else { action = config.url; diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.ts similarity index 80% rename from server/serverHelpers/serverHandlers.js rename to server/serverHelpers/serverHandlers.ts index 20fdbf134c..4241175471 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.ts @@ -1,29 +1,28 @@ -'use strict'; +import * as terms from '../../utility/hdbTerms.ts'; +import * as hdbUtil from '../../utility/common_utils.ts'; +import harperLogger from '../../utility/logging/harper_logger.ts'; +import { realExit } from '../threads/workerProcessGuard.ts'; +import { ClientError, handleHDBError, hdbErrors } from '../../utility/errors/hdbError.ts'; +import { isMainThread } from 'node:worker_threads'; +import { Readable } from 'node:stream'; -const terms = require('../../utility/hdbTerms.ts'); -const hdbUtil = require('../../utility/common_utils.ts'); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { realExit } = require('../threads/workerProcessGuard.ts'); -const { ClientError, handleHDBError, hdbErrors } = require('../../utility/errors/hdbError.ts'); -const { isMainThread } = require('worker_threads'); -const { Readable } = require('stream'); +import * as _os from 'node:os'; +let os = _os; -const os = require('os'); - -const auth = require('../../security/fastifyAuth.ts'); +import * as auth from '../../security/fastifyAuth.ts'; // this is a hack to suppress a deprecation warning and can be removed once `auth.authorize` // is converted to an async function -function pAuthorize(req, resp) { - return new Promise((resolve, reject) => { +function pAuthorize(req?: any, resp?: any) { + return new Promise((resolve, reject) => { auth.authorize(req, resp, (err, user) => (err ? reject(err) : resolve(user))); }); } -const serverUtilities = require('./serverUtilities.ts'); -const { applyImpersonation } = require('../../security/impersonation.ts'); -const { createGzip, constants } = require('zlib'); -const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter.ts'); +import * as serverUtilities from './serverUtilities.ts'; +import { applyImpersonation } from '../../security/impersonation.ts'; +import { createGzip, constants } from 'node:zlib'; +import { ProgressEmitter, createSSEResponseStream } from './progressEmitter.ts'; // Operations that support `Accept: text/event-stream` for live streaming. The handler // attaches a ProgressEmitter as req.body.progress so the operation can emit events; the @@ -31,13 +30,13 @@ const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter. // single-response shape because progress is undefined on that path. deploy_component / // get_deployment stream a bounded run and end; read_log tails the log and stays open until // the client disconnects (the emitter's abort signal), so subscribers see new lines live. -const SSE_PROGRESS_OPERATIONS = new Set([ +const SSE_PROGRESS_OPERATIONS: any = new Set([ terms.OPERATIONS_ENUM.DEPLOY_COMPONENT, terms.OPERATIONS_ENUM.GET_DEPLOYMENT, terms.OPERATIONS_ENUM.READ_LOG, ]); -const NO_AUTH_OPERATIONS = [ +const NO_AUTH_OPERATIONS: any = [ terms.OPERATIONS_ENUM.CREATE_AUTHENTICATION_TOKENS, terms.OPERATIONS_ENUM.LOGIN, terms.OPERATIONS_ENUM.LOGOUT, @@ -46,9 +45,9 @@ const NO_AUTH_OPERATIONS = [ terms.OPERATIONS_ENUM.EXCHANGE_OIDC_TOKEN, ]; -const UNSAFE_REQUEST_BODY_PROPERTIES = ['__proto__', 'constructor', 'prototype']; +const UNSAFE_REQUEST_BODY_PROPERTIES: any = ['__proto__', 'constructor', 'prototype']; -function validateRequestBodyProperties(body) { +function validateRequestBodyProperties(body?: any) { if (!body || typeof body !== 'object') { throw new ClientError('Invalid request body', 400); } @@ -59,7 +58,7 @@ function validateRequestBodyProperties(body) { } } -function handleServerUncaughtException(err) { +function handleServerUncaughtException(err?: any) { let message = `Found an uncaught exception with message: ${err.message}. ${os.EOL}Stack: ${err.stack} ${ os.EOL }Terminating ${isMainThread ? 'HDB' : 'thread'}.`; @@ -71,16 +70,16 @@ function handleServerUncaughtException(err) { } // The errors handlePostRequest logged, per request; see server/DESIGN.md. -const errorsLoggedByRequest = new WeakMap(); -const LEVELS_ABOVE_ERROR = new Set([terms.LOG_LEVELS.FATAL, terms.LOG_LEVELS.NOTIFY]); +const errorsLoggedByRequest: any = new WeakMap(); +const LEVELS_ABOVE_ERROR: any = new Set([terms.LOG_LEVELS.FATAL, terms.LOG_LEVELS.NOTIFY]); -function serverErrorHandler(error, req, resp) { +function serverErrorHandler(error?: any, req?: any, resp?: any) { // Fastify passes a handler's reason through as-is, so a rejection with none arrives here as nullish. error ??= handleHDBError(new Error('The request failed without an error'), undefined, 500); if (!errorsLoggedByRequest.get(req)?.has(error)) harperLogger[error.logLevel || 'info'](error); if (error.statusCode) { if (typeof error.http_resp_msg !== 'object') { - const body = { error: error.http_resp_msg || error.message }; + const body: any = { error: error.http_resp_msg || error.message }; // surface the machine-readable signal for errors that declare retryability (e.g. a // still-building index, IndexRebuildingError) so API callers can distinguish and retry if (error.retryable !== undefined) { @@ -98,7 +97,7 @@ function serverErrorHandler(error, req, resp) { return resp.code(statusCode).send(error.message ? { error: error.message } : error); } -function reqBodyValidationHandler(req, resp, done) { +function reqBodyValidationHandler(req?: any, resp?: any, done?: any) { if (!req.body || Object.keys(req.body).length === 0 || typeof req.body !== 'object') { const validationErr = handleHDBError(new Error(), 'Invalid JSON.', hdbErrors.HTTP_STATUS_CODES.BAD_REQUEST); done(validationErr, null); @@ -114,7 +113,7 @@ function reqBodyValidationHandler(req, resp, done) { done(); } -function authHandler(req, resp, done) { +function authHandler(req?: any, resp?: any, done?: any) { let user; const isAuthOperation = !NO_AUTH_OPERATIONS.includes(req.body.operation); @@ -149,7 +148,7 @@ function authHandler(req, resp, done) { } } -function authAndEnsureUserOnRequest(req, resp, done) { +function authAndEnsureUserOnRequest(req?: any, resp?: any, done?: any) { pAuthorize(req, resp) .then((userData) => { req.hdb_user = userData; @@ -163,7 +162,7 @@ function authAndEnsureUserOnRequest(req, resp, done) { }); } -async function handlePostRequest(req, res, _bypassAuth = false) { +async function handlePostRequest(req?: any, res?: any, _bypassAuth: any = false) { let operation_function; try { @@ -186,7 +185,7 @@ async function handlePostRequest(req, res, _bypassAuth = false) { // `Accept` parsing only checks for the text/event-stream token; allow comma-separated // values like `text/event-stream, application/json` and quality params per RFC 7231. if (req.headers?.accept?.includes('text/event-stream') && SSE_PROGRESS_OPERATIONS.has(req.body.operation)) { - const emitter = new ProgressEmitter(); + const emitter: any = new ProgressEmitter(); req.body.progress = emitter; res.header('Content-Type', 'text/event-stream'); res.header('Cache-Control', 'no-cache'); @@ -194,9 +193,9 @@ async function handlePostRequest(req, res, _bypassAuth = false) { return createSSEResponseStream(emitter, () => serverUtilities.processLocalTransaction(req, operation_function)); } - let result = await serverUtilities.processLocalTransaction(req, operation_function); - if (result instanceof Readable && result.headers) { - for (let [name, value] of result.headers) { + let result: any = await serverUtilities.processLocalTransaction(req, operation_function); + if (result instanceof Readable && (result as any).headers) { + for (let [name, value] of (result as any).headers) { res.header(name, value); } // fastify-compress has one job. I don't know why it can't do it. So we compress here to @@ -205,7 +204,11 @@ async function handlePostRequest(req, res, _bypassAuth = false) { // mislabel a gzip:false tar or double-compress a gzip:true one). Streams marked // `preCompressed` (e.g. a stored .tar.gz payload) are passed through as-is — // recompressing them wastes CPU for zero gain. - if (req.headers['accept-encoding']?.includes('gzip') && !result.noCompression && !result.preCompressed) { + if ( + req.headers['accept-encoding']?.includes('gzip') && + !(result as any).noCompression && + !(result as any).preCompressed + ) { res.header('content-encoding', 'gzip'); const gzip = createGzip({ level: constants.Z_BEST_SPEED }); // go fast // .pipe() does not tear down across the pipe in either direction, so wire both: @@ -234,7 +237,7 @@ async function handlePostRequest(req, res, _bypassAuth = false) { } } -module.exports = { +export { authHandler, authAndEnsureUserOnRequest, handlePostRequest, diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 9cd8b3a00e..beddc7349c 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -1,3 +1,4 @@ +import { loadRuntimeModule } from '../../utility/packageUtils.js'; import * as search from '../../dataLayer/search.ts'; import * as bulkLoad from '../../dataLayer/bulkLoad.ts'; import * as schema from '../../dataLayer/schema.ts'; @@ -6,7 +7,7 @@ import * as delete_ from '../../dataLayer/delete.ts'; import readAuditLog from '../../dataLayer/readAuditLog.ts'; import * as user from '../../security/user.ts'; import * as role from '../../security/role.ts'; -import customFunctionOperations from '../../components/operations.js'; +import * as customFunctionOperations from '../../components/operations.ts'; import { setMcpQuotaHandler } from '../../components/mcp/quota.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; import readLog from '../../utility/logging/readLog.ts'; @@ -18,7 +19,7 @@ import * as terms from '../../utility/hdbTerms.ts'; import { hdbErrors, handleHDBError } from '../../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; import * as restart from '../../bin/restart.ts'; -import * as util from 'util'; +import * as util from 'node:util'; import * as insert from '../../dataLayer/insert.ts'; import * as globalSchema from '../../utility/globalSchema.ts'; import { systemInformation } from '../../utility/environment/systemInformation.ts'; @@ -53,10 +54,14 @@ import { runWithOperationAuthorizationBypass } from './operationAuthorizationSta import { stripSuppliedParsedSqlObject } from './requestSanitization.ts'; const pSearchSearch = util.promisify(search.search); +let sqlModule; +function getSqlModule() { + return (sqlModule ??= loadRuntimeModule('sqlTranslator/index')); +} let pEvaluateSql: (sql: string) => Promise; function evaluateSQL(command) { if (!pEvaluateSql) { - const sql = require('../../sqlTranslator/index'); + const sql = getSqlModule(); pEvaluateSql = util.promisify(sql.evaluateSQL); } return pEvaluateSql(command); @@ -179,7 +184,6 @@ export async function processLocalTransaction(req: OperationRequest, operationFu export const OPERATION_FUNCTION_MAP = initializeOperationFunctionMap(); -server.operation = operation; export type OperationDefinition = { name: string; execute: (operation: any) => any | Promise; @@ -205,7 +209,7 @@ const declaredPermissionNames = new Set(); * Register an operation function with the server. * @param operationDefinition */ -server.registerOperation = (operationDefinition: OperationDefinition) => { +function registerOperation(operationDefinition: OperationDefinition) { const { name, execute, requiresSuperUser } = operationDefinition; let handler = execute; if (requiresSuperUser === undefined) { @@ -233,7 +237,7 @@ server.registerOperation = (operationDefinition: OperationDefinition) => { // so the main thread can forward calls here (#1736), and can mirror the role-allowlist mark that // registerOperationPermission above made only in this thread's scope. if (!isMainThread) announceRegisteredOperation(name, requiresSuperUser !== undefined); -}; +} // Register the durable MCP quota policy as a function (see components/mcp/quota.ts). Worker-local, // like the tool dispatch that consults it, so no cross-thread announcement is needed. @@ -319,7 +323,7 @@ export function chooseOperation(json: OperationRequestBody, bypassAuth = false) try { if (isSqlOperation || hasNestedSqlSearch) { - const sql = require('../../sqlTranslator/index'); + const sql = getSqlModule(); const sqlStatement = isSqlOperation ? json.sql : nestedSearch.sql; // Before this dispatch's own parse is assigned, so a body-supplied object cannot survive it. stripSuppliedParsedSqlObject(json); @@ -849,3 +853,6 @@ function initializeOperationFunctionMap(): Map Promise | void }[] diff --git a/server/threads/heldStart.ts b/server/threads/heldStart.ts index 199126e123..a27c16f901 100644 --- a/server/threads/heldStart.ts +++ b/server/threads/heldStart.ts @@ -38,7 +38,8 @@ export async function beginHeldStart(requests: HeldStartRequest[]): Promise + loaded: Map, + onReported?: () => void ): Promise { while (requests.some(({ component }) => bootVerdictOf(component).outcome === 'pending')) { await delay(DEFERRED_LOAD_POLL_MS); @@ -63,5 +64,6 @@ export async function awaitAdmission( }; parentPort.on('message', onMessage); parentPort.postMessage({ type: ITC_EVENT_TYPES.CHILD_COMPONENT_VERDICT, components }); + onReported?.(); }); } diff --git a/server/threads/isolatedApplications.ts b/server/threads/isolatedApplications.ts index 08cb63cb46..fa0dde8401 100644 --- a/server/threads/isolatedApplications.ts +++ b/server/threads/isolatedApplications.ts @@ -10,7 +10,7 @@ * metadata names the application and its hosts for the fronting proxy to route by. */ import { isMainThread, workerData } from 'node:worker_threads'; -import { getWorkerIndex } from './manageThreads.js'; +import { getWorkerIndex } from './manageThreads.ts'; import { getConfigObj, getConfigPath } from '../../config/configUtils.ts'; import * as env from '../../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; diff --git a/server/threads/itc.js b/server/threads/itc.ts similarity index 72% rename from server/threads/itc.js rename to server/threads/itc.ts index cffeada555..80151c9470 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.ts @@ -1,25 +1,24 @@ -'use strict'; - -const hdbUtils = require('../../utility/common_utils.ts'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { threadId } = require('worker_threads'); -const { +import * as hdbTerms from '../../utility/hdbTerms.ts'; +// Inline isEmpty to break the common_utils→databases→itc→common_utils circular dep. +const isEmpty = (value: unknown) => value === undefined || value === null; +import { ITC_ERRORS } from '../../utility/errors/commonErrors.ts'; +import { threadId as _threadId } from 'node:worker_threads'; +const threadId = _threadId; +import { onMessageFromWorkers, - broadcastWithAcknowledgement, - broadcastWithStrictAcknowledgement, -} = require('./manageThreads.js'); + broadcastWithAcknowledgement as _broadcastWithAcknowledgement, + broadcastWithStrictAcknowledgement as _broadcastWithStrictAcknowledgement, +} from './manageThreads.ts'; +let broadcastWithAcknowledgement = _broadcastWithAcknowledgement; +const broadcastWithStrictAcknowledgement = _broadcastWithStrictAcknowledgement; -module.exports = { - sendItcEvent, - sendItcEventStrict, - validateEvent, - SchemaEventMsg, -}; +export { sendItcEvent, sendItcEventStrict, validateEvent, SchemaEventMsg }; +import * as serverItcHandlersModule from '../itc/serverHandlers.ts'; +import hdbLogger from '../../utility/logging/harper_logger.ts'; let serverItcHandlers; const RESTORE_CLOSE_ACK_TIMEOUT_MS = 30000; onMessageFromWorkers(async (event, sender) => { - serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); + serverItcHandlers = serverItcHandlers || serverItcHandlersModule.default; let error; try { validateEvent(event); @@ -27,7 +26,6 @@ onMessageFromWorkers(async (event, sender) => { await serverItcHandlers[event.type](event); } } catch (caught) { - const hdbLogger = require('../../utility/logging/harper_logger.ts'); hdbLogger.error('ITC event handler failed', caught); error = { name: caught?.name, @@ -50,7 +48,7 @@ onMessageFromWorkers(async (event, sender) => { * @param event * @param {boolean|'active'} includeJobWorkers */ -function sendItcEvent(event, includeJobWorkers = false) { +function sendItcEvent(event, includeJobWorkers: boolean | 'active' = false) { // Always stamp originator so handlers can send direct responses back. // The main thread's threadId is 0 (worker_threads convention); parentPort.threadId // is set to 0 in workers, so sendToThread(0, ...) routes back to main. @@ -79,7 +77,7 @@ function sendItcEvent(event, includeJobWorkers = false) { } /** @param {boolean|'active'} includeJobWorkers */ -function sendItcEventStrict(event, timeout, includeJobWorkers = false) { +function sendItcEventStrict(event, timeout, includeJobWorkers: boolean | 'active' = false) { if (event.message) event.message.originator = threadId; return broadcastWithStrictAcknowledgement(event, timeout, includeJobWorkers); } @@ -94,15 +92,15 @@ function validateEvent(event) { return ITC_ERRORS.INVALID_ITC_DATA_TYPE; } - if (!event.hasOwnProperty('type') || hdbUtils.isEmpty(event.type)) { + if (!event.hasOwnProperty('type') || isEmpty(event.type)) { return ITC_ERRORS.MISSING_TYPE; } - if (!event.hasOwnProperty('message') || hdbUtils.isEmpty(event.message)) { + if (!event.hasOwnProperty('message') || isEmpty(event.message)) { return ITC_ERRORS.MISSING_MSG; } - if (!event.message.hasOwnProperty('originator') || hdbUtils.isEmpty(event.message.originator)) { + if (!event.message.hasOwnProperty('originator') || isEmpty(event.message.originator)) { return ITC_ERRORS.MISSING_ORIGIN; } diff --git a/server/threads/logRotationTransport.ts b/server/threads/logRotationTransport.ts index 4ef44e8c1f..60b4771b22 100644 --- a/server/threads/logRotationTransport.ts +++ b/server/threads/logRotationTransport.ts @@ -5,7 +5,7 @@ import { threadId } from 'node:worker_threads'; import { setRotationTransport } from '../../utility/logging/logGenerationCoordinator.ts'; -import { broadcast, onMessageByType, onThreadExit } from './manageThreads.js'; +import { broadcast, onMessageByType, onThreadExit } from './manageThreads.ts'; // Assigned to the `threads` global by manageThreads. declare const threads: { diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.ts similarity index 90% rename from server/threads/manageThreads.js rename to server/threads/manageThreads.ts index c55833ab9c..77cdc001e0 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.ts @@ -1,42 +1,49 @@ -'use strict'; - -// Must run before any component code is loaded so that process.exit() called -// from component code (e.g. Next.js's `unhandledRejection` handler) is -// intercepted in workers. -const { realExit } = require('./workerProcessGuard.ts'); - -const { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } = require('worker_threads'); -const { spawnSync } = require('node:child_process'); -const { readdirSync, readFileSync, readlinkSync } = require('node:fs'); -const { setTimeout: delay } = require('node:timers/promises'); -const { confirmWindowsProcessTreeGone, ROOT_SPAWN_ALLOWANCE_MS } = require('./windowsProcessTree.ts'); -const { activeWorkerPools } = require('./workerPools.ts'); -const { join, isAbsolute, extname } = require('path'); -const { pathToFileURL } = require('url'); -const { server } = require('../Server.ts'); -const { totalmem } = require('os'); -const { setHeapSnapshotNearHeapLimit } = typeof globalThis.Bun !== 'undefined' ? {} : require('v8'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const envMgr = require('../../utility/environment/environmentManager.ts'); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { randomBytes } = require('crypto'); -const { _assignPackageExport } = require('../../globals.js'); -const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); -const { resolvePreloadModules } = require('./resolvePreload.ts'); -const { resolveThreadHeapMemoryMb } = require('./threadHeapMemory.ts'); -const { getConfigPath } = require('../../config/configUtils.ts'); -const { resolveWatchTarget } = require('../../utility/watchPath.ts'); -const { +import { loadRuntimeModule } from '../../utility/packageUtils.js'; +import { + listenersByType, + messagesQueuedByType, + messageListeners, + threadExitListeners, + workerHooks, +} from './threadMessageState.ts'; +// Install the exit guard before component dependencies evaluate. +import { realExit } from './workerProcessGuard.ts'; +import * as shutdownDrainRuntimeModule from '../../components/shutdownDrain.ts'; +import { onStartup } from '../../utility/lifecycle.ts'; + +import { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } from 'node:worker_threads'; +import { spawnSync } from 'node:child_process'; +import { readdirSync, readFileSync, readlinkSync } from 'node:fs'; +import { setTimeout as delay } from 'node:timers/promises'; +import { confirmWindowsProcessTreeGone, ROOT_SPAWN_ALLOWANCE_MS } from './windowsProcessTree.ts'; +import { activeWorkerPools } from './workerPools.ts'; +import { join, isAbsolute, extname } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { server } from '../Server.ts'; +import { totalmem } from 'node:os'; +import { setHeapSnapshotNearHeapLimit as _setHeapSnapshotNearHeapLimit } from 'node:v8'; +const setHeapSnapshotNearHeapLimit = typeof globalThis.Bun !== 'undefined' ? undefined : _setHeapSnapshotNearHeapLimit; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import * as envMgr from '../../utility/environment/environmentManager.ts'; +import harperLogger from '../../utility/logging/harper_logger.ts'; +import { randomBytes } from 'node:crypto'; +import { _assignPackageExport } from '../../globals.js'; +import { PACKAGE_ROOT, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; +import { resolvePreloadModules } from './resolvePreload.ts'; +import { resolveThreadHeapMemoryMb } from './threadHeapMemory.ts'; +import { getConfigPath } from '../../config/configUtils.ts'; +import { resolveWatchTarget } from '../../utility/watchPath.ts'; +import { databaseDropPreparationSnapshot, handleDatabaseDropPreparationOwnerExit, -} = require('../../resources/databaseDropPreparation.ts'); -const { +} from '../../resources/databaseDropPreparation.ts'; +import { DIRECTORY_POLLING_FALLBACK_OPTIONS, claimLostNativeWatchError, guardedWatch, isWatcherExhaustionError, warnWatcherFallback, -} = require('../../utility/watcherFallback.ts'); +} from '../../utility/watcherFallback.ts'; let importModules; function getImportModules() { if (importModules === undefined) @@ -59,9 +66,9 @@ function getRequireModules() { } const isBun = typeof globalThis.Bun !== 'undefined'; const MB = 1024 * 1024; -const workers = []; // these are our child workers that we are managing +const workers: any = []; // these are our child workers that we are managing let processShuttingDown = false; -const connectedPorts = []; // these are all known connected worker ports (siblings, children, parents) +const connectedPorts: any = []; // these are all known connected worker ports (siblings, children, parents) const MAX_UNEXPECTED_RESTARTS = 50; // Threads get 10s to die before they're forced. In dev (`harper dev`) we widen this: a reload's old // worker may be disposing a native runtime (e.g. @harperfast/vite's rolldown dev server) and forcing it @@ -105,12 +112,12 @@ _assignPackageExport('threads', connectedPorts); // Worker-side: (re)arm the self-exit backstop `delay` ms out, but never earlier than a drain deadline // already requested via extendShutdownDeadline (so ordering between the SHUTDOWN handler and the drain // extension doesn't matter). -function armSelfExit(delay) { +function armSelfExit(delay?: any) { if (selfExitDrainDeadline) { // selfExitDrainDeadline is already clamped to the configured ceiling (see boundedTerminateDelay), // but adding threadTerminationTimeout headroom on top can push the sum back over the max timer // value at the extreme end of that ceiling, silently defeating the overflow guard. - const { MAX_TIMER_MS } = require('../../components/shutdownDrain.ts'); + const { MAX_TIMER_MS } = shutdownDrainRuntimeModule; delay = Math.max( delay, Math.min(Math.max(0, selfExitDrainDeadline - Date.now()) + threadTerminationTimeout, MAX_TIMER_MS) @@ -130,7 +137,7 @@ function armSelfExit(delay) { // the main thread to extend its external force-terminate timer to match. Called from the shutdown path // only when there is real work to drain (see threadServer / shutdownDrain), so an unrelated hang is // still force-killed on the normal short timeout. -function extendShutdownDeadline(deadlineMs) { +function extendShutdownDeadline(deadlineMs?: any) { selfExitDrainDeadline = Math.max(selfExitDrainDeadline, deadlineMs); if (selfExitTimer) armSelfExit(0); // re-arm honoring the (now recorded) drain deadline try { @@ -157,19 +164,17 @@ function notifyJobCleanupComplete() { } } -const listenersByType = new Map(); -const messagesQueuedByType = new Map(); const { promise: whenThreadsStarted, resolve: threadsHaveStarted } = Promise.withResolvers(); -const initialRestartNumber = workerData?.restartNumber || 1; +export let restartNumber = workerData?.restartNumber || 1; // Identifies this process incarnation, where the PID cannot: a container reuses PID 1. Minted once // on the main thread and carried to workers, so live siblings agree on it — one derived per thread // would not. `undefined` on a worker started without it; consumers must fall back, not treat that // as a mismatch. -const processIncarnation = workerData ? workerData.processIncarnation : randomBytes(8).toString('hex'); +import { processIncarnation } from './processIncarnation.ts'; // Every value in this literal is a bare identifier: Node's CommonJS export scan, which supplies the // named bindings an ES module import can use, stops reading the literal at the first value that is not. -module.exports = { +export { startWorker, restartWorkers, canaryVerdictTimeoutMs, @@ -207,6 +212,7 @@ module.exports = { restoreShutdownDeadline, notifyJobCleanupComplete, beginProcessShutdown, + isProcessShuttingDown, registerWorkerDataProvider, onThreadExit, hasThreadExited, @@ -223,20 +229,15 @@ module.exports = { setCertificationHandler, setCanaryVerdictTimeout, setRootComponentsReload, - restartNumber: initialRestartNumber, processIncarnation, whenThreadsStarted, threadsHaveStarted, // Assigned further down once defined. TypeScript 7 only treats keys of this literal as exports and // types them from it, so only a value that cannot be built before the literal belongs here. - sendToThread: undefined, - getThreadInfo: undefined, - getRunningIsolatedApplications: undefined, - watchDir: undefined, }; -connectedPorts.onMessageByType = onMessageByType; -connectedPorts.sendToThread = function (threadId, message) { +(connectedPorts as any).onMessageByType = onMessageByType; +(connectedPorts as any).sendToThread = function (threadId?: any, message?: any) { if (!message?.type) throw new Error('A message with a type must be provided'); const port = connectedPorts.find((port) => port.threadId === threadId); if (!port) return false; @@ -253,14 +254,14 @@ connectedPorts.sendToThread = function (threadId, message) { }; // Direct thread-to-thread send, so a worker can reach a sibling (e.g. the record lock owner worker) // without a hop through main. Returns false when no port for the thread is connected. -module.exports.sendToThread = connectedPorts.sendToThread; +export const sendToThread = connectedPorts.sendToThread; // make sure this is set on all threads, including the main thread (this is no-op // if it was already with the execArgv below) if (envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) setHeapSnapshotNearHeapLimit(1); let isMainWorker; -function setTerminateTimeout(newTimeout) { +function setTerminateTimeout(newTimeout?: any) { threadTerminationTimeout = newTimeout; } function getWorkerIndex() { @@ -272,7 +273,7 @@ function getWorkerIndex() { * dedicated worker for an isolated one -- never index 0, so node-wide duties stay on the pool, and * never a dedicated worker for anything but its own application. */ -function isApplicationPrimaryWorker(applicationName) { +function isApplicationPrimaryWorker(applicationName?: any) { if (workerData?.isolatedApplication !== undefined) return applicationName === workerData.isolatedApplication; return getWorkerIndex() === 0; } @@ -281,8 +282,8 @@ function isApplicationPrimaryWorker(applicationName) { * dedicated worker. Registered by openBranchDatabase, so the ownership predicates below can tell them * from the shared stores every thread has open. */ -const branchStorePaths = new Set(); -function markBranchStorePath(path, isBranch = true) { +const branchStorePaths: any = new Set(); +function markBranchStorePath(path?: any, isBranch: any = true) { if (isBranch) branchStorePaths.add(path); else branchStorePaths.delete(path); } @@ -291,12 +292,12 @@ function markBranchStorePath(path, isBranch = true) { * reclamation, audit cleanup). Shared stores belong to the last pool worker; a dedicated worker * maintains only its own branch stores, which no other thread has open. */ -function ownsStoreMaintenance(storePath) { +function ownsStoreMaintenance(storePath?: any) { if (workerData?.isolatedApplication !== undefined) return branchStorePaths.has(storePath); return getWorkerIndex() === getWorkerCount() - 1; } /** The "worker 0" counterpart: expiration eviction for `storePath`. */ -function ownsStoreExpiration(storePath) { +function ownsStoreExpiration(storePath?: any) { if (workerData?.isolatedApplication !== undefined) return branchStorePaths.has(storePath); return getWorkerIndex() === 0; } @@ -318,7 +319,7 @@ function applicationWorkerIndex() { return workerData?.isolatedApplication !== undefined ? 0 : getWorkerIndex(); } /** Every started worker dedicated to `application`, including a replacement still booting. */ -function workersForApplication(application) { +function workersForApplication(application?: any) { return workers.filter((worker) => worker.application === application); } /** @@ -326,9 +327,9 @@ function workersForApplication(application) { * the same backstop the rolling restart uses (FORCE_EXIT on Bun, where terminate() segfaults), and * resolve once it has exited. Marked as shut down first so its exit does not start a replacement. */ -function stopWorker(worker) { +function stopWorker(worker?: any) { worker.wasShutdown = true; - return new Promise((resolve) => { + return new Promise((resolve) => { const armTerminate = (delay) => setTimeout(() => { harperLogger.warn('Thread did not voluntarily terminate, terminating from the outside', worker.threadId); @@ -343,7 +344,7 @@ function stopWorker(worker) { let timeout = armTerminate(threadTerminationTimeout * 2); worker.extendTerminateDeadline = (deadlineMs) => { clearTimeout(timeout); - const { boundedTerminateDelay, getShutdownDrainCeilingMs } = require('../../components/shutdownDrain.ts'); + const { boundedTerminateDelay, getShutdownDrainCeilingMs } = shutdownDrainRuntimeModule; timeout = armTerminate( boundedTerminateDelay(deadlineMs, Date.now(), threadTerminationTimeout * 2, getShutdownDrainCeilingMs()) ); @@ -351,24 +352,24 @@ function stopWorker(worker) { worker.on('exit', () => { clearTimeout(timeout); worker.extendTerminateDeadline = undefined; - resolve(); + resolve(undefined); }); try { - worker.postMessage({ restartNumber: module.exports.restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN }); + worker.postMessage({ restartNumber: restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN }); } catch { clearTimeout(timeout); - resolve(); // already gone + resolve(undefined); // already gone } }); } function getWorkerCount() { return workerData ? workerData.workerCount : isMainWorker ? 1 : undefined; } -function isEligibleBroadcastRecipient(port) { +function isEligibleBroadcastRecipient(port?: any) { return !port.isJobWorker; } function getEligibleBroadcastRecipientThreadIds() { - const recipientThreadIds = new Set(); + const recipientThreadIds: any = new Set(); for (const port of connectedPorts) { if (isEligibleBroadcastRecipient(port) && port.threadId !== undefined) { recipientThreadIds.add(port.threadId); @@ -376,9 +377,9 @@ function getEligibleBroadcastRecipientThreadIds() { } return recipientThreadIds; } -function setMainIsWorker(isWorker) { +function setMainIsWorker(isWorker?: any) { isMainWorker = isWorker; - module.exports.threadsHaveStarted(); + threadsHaveStarted(undefined); } let workerCount = 1; // should be assigned when workers are created @@ -386,7 +387,7 @@ let workerCount = 1; // should be assigned when workers are created // Covers the keys startWorker spreads below plus keys read elsewhere: `noServerStart` is set by // the embedding entry point (index.ts) and read by threadServer.js to skip startServers(); a // provider shadowing it would wedge HTTP worker startup. -const RESERVED_WORKER_DATA_KEYS = [ +const RESERVED_WORKER_DATA_KEYS: any = [ 'addPorts', 'addThreadIds', 'addPortIsJobWorkers', @@ -405,7 +406,7 @@ const RESERVED_WORKER_DATA_KEYS = [ 'poolIndex', '__proto__', // never a legitimate payload name; spread would define it as an own property ]; -const workerDataProviders = new Map(); +const workerDataProviders: any = new Map(); /** * Register a provider that contributes an extra `workerData` property to every worker spawned * from this thread. `provider(options)` receives the startWorker options (`options.name` is the @@ -414,7 +415,7 @@ const workerDataProviders = new Map(); * returns a non-cloneable value is logged and skipped so it can never break a spawn. * Returns a function that unregisters the provider. */ -function registerWorkerDataProvider(name, provider) { +function registerWorkerDataProvider(name?: any, provider?: any) { if (RESERVED_WORKER_DATA_KEYS.includes(name) || workerDataProviders.has(name)) { throw new Error(`workerData provider name '${name}' is already in use`); } @@ -432,7 +433,7 @@ function registerWorkerDataProvider(name, provider) { // setProperty() clones each value as it records it, so this provider cannot hit the log-and-skip // path below — which for this one would mean spawning the worker on the on-disk config. registerWorkerDataProvider('configOverrides', () => envMgr.getConfigOverrides()); -function collectProvidedWorkerData(options) { +function collectProvidedWorkerData(options?: any) { if (workerDataProviders.size === 0) return undefined; let provided; for (const [name, provider] of workerDataProviders) { @@ -478,7 +479,7 @@ if (!parentPort) { worker.postMessage({ type: RUNNING_ISOLATED_APPLICATIONS, requestId: message.requestId, - applications: runningIsolatedApplicationsGetter(), + applications: workerHooks.runningApplications(), }); }); onMessageByType(RESOURCE_REPORT, (message, worker) => { @@ -502,18 +503,22 @@ if (!parentPort) { worker?.extendTerminateDeadline?.(message.deadlineMs); }); } -// postMessage type listeners that are registered in other ways or can be registered later -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_REQUEST, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_RESPONSE, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_REQUEST, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_RESPONSE, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_REGISTERED, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_REQUEST, null); -listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_RESPONSE, null); +// Queue messages for late listeners without replacing registrations made during cyclic imports. +for (const type of [ + hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, + hdbTerms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE, + hdbTerms.ITC_EVENT_TYPES.SCHEMA, + hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, + hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_REQUEST, + hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_RESPONSE, + hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_REQUEST, + hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_RESPONSE, + hdbTerms.ITC_EVENT_TYPES.OPERATION_REGISTERED, + hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_REQUEST, + hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_RESPONSE, +]) { + if (!listenersByType.has(type)) listenersByType.set(type, null); +} // These request/response functions register their own one-shot parentPort listener per // call rather than going through onMessageByType, so without this, every reply would also reach // addPort's permanent dispatcher as an "unregistered" type: notifyMessageListeners would warn and @@ -536,7 +541,7 @@ const certifications = new Map(); // complete before then too. const settledCertifications = new Map(); const settledKey = (component, deploymentId) => `${component}\u0000${deploymentId}`; -const heldStarts = new Set(); +const heldStarts = new Set(); const failClosedInMemory = new Map(); let certificationHandler; const REQUESTER_RELEASE_TIMEOUT_MS = 10000; @@ -704,11 +709,11 @@ async function untilNoCertificationArmed() { /** A deploy bracket of main's own (components/deployLifecycle.ts) over a complete tree: it holds watchers, not loads. */ function pauseWatchersOf(component) { - return require('../../components/deployLifecycle.ts').broadcastDeployStart(component, { watchersOnly: true }); + return loadRuntimeModule('components/deployLifecycle').broadcastDeployStart(component, { watchersOnly: true }); } function resumeWatchersOf(component, bracket) { - require('../../components/deployLifecycle.ts').broadcastDeployEnd(component, bracket); + loadRuntimeModule('components/deployLifecycle').broadcastDeployEnd(component, bracket); } function errorMessageOf(error) { @@ -742,7 +747,7 @@ function verdictDecision(certification, components) { }; } -async function decideCertification(certification, decision, canary) { +async function decideCertification(certification, decision, canary?: any) { if (certification.decision || certification.deciding) return; certification.deciding = true; certification.refusing = refusesRelease(decision); @@ -765,7 +770,7 @@ async function decideCertification(certification, decision, canary) { } /** Every held start loading a release about to be refused stops first, so no restore runs under one still holding it. */ -function stopHeldStartsOf(certification, canary) { +function stopHeldStartsOf(certification, canary?: any) { const stopping = [...heldStarts].filter((held) => held.gated.includes(certification)); if (canary && heldStarts.has(canary) && !stopping.includes(canary)) stopping.push(canary); return Promise.all(stopping.map((held) => stopHeldStart(held, certification))); @@ -817,7 +822,7 @@ function heldRequests(gated, checks) { } function holdStart(worker, gated, startOptions) { - const held = { + const held: any = { worker, gated, checks: startOptions.check ?? [], @@ -997,7 +1002,7 @@ function loadFailure(entry, component = entry.component) { }`; } -function stopHeldStart(held, refused) { +function stopHeldStart(held, refused?: any) { if (held.stopping) return held.stopping; held.settled = true; held.worker.stoppedByGate = true; @@ -1051,11 +1056,11 @@ function releaseCertificationRequester(component, deploymentId) { noteAnswered(); } -let answered = Promise.withResolvers(); +let answered = Promise.withResolvers(); /** A deploy stopped answering: it released, left, or its worker exited. */ function noteAnswered() { answered.resolve(); - answered = Promise.withResolvers(); + answered = Promise.withResolvers(); } /** @@ -1212,9 +1217,9 @@ function registerCertificationRequests() { }); } -function startWorker(path, options = {}, startOptions = {}) { +function startWorker(path?: any, options: any = {}, startOptions: any = {}) { if (processShuttingDown) { - const error = new Error('Cannot start a worker while the Harper process is shutting down'); + const error: any = new Error('Cannot start a worker while the Harper process is shutting down'); error.code = 'ERR_HARPER_PROCESS_SHUTTING_DOWN'; throw error; } @@ -1241,10 +1246,10 @@ function startWorker(path, options = {}, startOptions = {}) { // https://plaid.com/blog/how-we-parallelized-our-node-service-by-30x/ const maxYoungMemory = Math.min(Math.max(maxOldMemory >> 6, 16), 64); - const channelsToConnect = []; - const portsToSend = []; + const channelsToConnect: any = []; + const portsToSend: any = []; for (let existingPort of connectedPorts) { - const channel = new MessageChannel(); + const channel: any = new MessageChannel(); channel.existingPort = existingPort; channelsToConnect.push(channel); portsToSend.push(channel.port2); @@ -1262,6 +1267,7 @@ function startWorker(path, options = {}, startOptions = {}) { ]; // an explicit execArgv replaces inheritance, so `node --enable-source-maps` would not reach workers otherwise if (!isBun && process.sourceMapsEnabled) execArgv.push('--enable-source-maps'); + if (!isBun && RUNTIME_FILE_EXT === '.ts') execArgv.push('--conditions=typestrip'); if (!isBun && envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) execArgv.push('--heapsnapshot-near-heap-limit=1'); // Preload configured modules (e.g. an APM agent like dd-trace) before the worker's entry @@ -1285,7 +1291,7 @@ function startWorker(path, options = {}, startOptions = {}) { const gated = openCertificationsPlacedBy(options); const certify = heldRequests(gated, startOptions.check); - const worker = new Worker(isAbsolute(path) ? path : join(PACKAGE_ROOT, path), { + const worker: any = new Worker(isAbsolute(path) ? path : join(PACKAGE_ROOT, path), { resourceLimits: { maxOldGenerationSizeMb: maxOldMemory, maxYoungGenerationSizeMb: maxYoungMemory, @@ -1304,8 +1310,8 @@ function startWorker(path, options = {}, startOptions = {}) { isolatedApplication: options.application, workerPools: activeWorkerPools(), poolIndex: options.poolIndex, - restartNumber: module.exports.restartNumber, - processIncarnation: module.exports.processIncarnation, + restartNumber: restartNumber, + processIncarnation: processIncarnation, ticketKeys: getTicketKeys(), databaseDropPreparations: databaseDropPreparationSnapshot(), certify, @@ -1347,7 +1353,7 @@ function startWorker(path, options = {}, startOptions = {}) { workers.splice(workers.indexOf(worker), 1); // Its native handles are closed by now, so the origin-floor bounds it published are stale; every // thread retires them for the databases it has open, since this one need not have them all. - const { ORIGIN_FLOOR_RETIRE, retireOriginFloorSlots } = require('../../resources/originClosedFloor.ts'); + const { ORIGIN_FLOOR_RETIRE, retireOriginFloorSlots } = loadRuntimeModule('resources/originClosedFloor'); retireOriginFloorSlots(exitedThreadId); broadcast({ type: ORIGIN_FLOOR_RETIRE, threadId: exitedThreadId }); if ( @@ -1378,7 +1384,7 @@ function startWorker(path, options = {}, startOptions = {}) { return worker; } -const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP, hdbTerms.THREAD_TYPES.REPLICATION]; +const OVERLAPPING_RESTART_TYPES: any = [hdbTerms.THREAD_TYPES.HTTP, hdbTerms.THREAD_TYPES.REPLICATION]; /** * Restart all the worker threads @@ -1400,14 +1406,14 @@ const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP, hdbTerms.THREAD_T * behavior, explicit undefined restarts the shared pool only, a name restarts only that application's * dedicated worker, and '*' restarts all. */ -let replacementRestarts = Promise.resolve(); +let replacementRestarts: Promise = Promise.resolve(); async function restartWorkers( - name = null, - maxWorkersDown = Math.max(Math.floor(workerCount / 8), 1), // restart 1/8 of the threads at a time, but at least 1 - startReplacementThreads = true, - onProgress = null, - application = undefined, - certification = undefined + name: any = null, + maxWorkersDown: any = Math.max(Math.floor(workerCount / 8), 1), // restart 1/8 of the threads at a time, but at least 1 + startReplacementThreads: any = true, + onProgress: any = null, + application: any = undefined, + certification: any = undefined ) { if (arguments.length < 5) application = '*'; if (isMainThread) { @@ -1427,7 +1433,14 @@ async function restartWorkers( } } -async function replaceWorkers(name, maxWorkersDown, startReplacementThreads, onProgress, application, certification) { +async function replaceWorkers( + name, + maxWorkersDown, + startReplacementThreads, + onProgress, + application, + certification?: any +) { { // Declining is not the same as delegating: a caller reporting on the restart must not read this // as "another thread is completing it". @@ -1440,9 +1453,7 @@ async function replaceWorkers(name, maxWorkersDown, startReplacementThreads, onP } catch (e) { harperLogger.error('Unable to reestablish current working directory', e); } - const freshlyStarted = new Set(); // dedicated workers the reconcile just started: already on the new code - // problematic cyclic dependency, bind late - const { resetRestartNeeded } = require('../../components/requestRestart.ts'); + const freshlyStarted: any = new Set(); // dedicated workers the reconcile just started: already on the new code // One process-wide bit cannot say WHICH application is pending, so it may only be cleared by a // restart that demonstrably covers every worker the bit could stand for. A restart scoped to one // application leaves the whole pool on its old code; a pool restart replaces every pool worker and @@ -1452,20 +1463,23 @@ async function replaceWorkers(name, maxWorkersDown, startReplacementThreads, onP // which is every node that uses no isolated application, i.e. the historic behavior unchanged. const coversEveryWorker = application === '*' || (application === undefined && !workers.some((worker) => worker.application)); - if (coversEveryWorker) resetRestartNeeded(); + // An unregistered restart module cannot have set the flag. Keep shutdown marking synchronous. + if (coversEveryWorker) workerHooks.resetRestartNeeded?.(); // This is here to prevent circular dependencies if (startReplacementThreads) { - const loadRootComponents = rootComponentsReload ?? require('../loadRootComponents.js').loadRootComponents; + const loadRootComponents = rootComponentsReload ?? (await import('../loadRootComponents.ts')).loadRootComponents; + if (processShuttingDown) return { declined: true }; // Installing and loading every root component reports nothing and can outlast a caller's idle // window on its own (a cold npm cache, a large dependency graph), so beat while it runs. The // caller's absolute ceiling is what bounds a load that never finishes. const loading = setInterval(() => onProgress?.(), RESTART_PROGRESS_HEARTBEAT_MS).unref(); try { await loadRootComponents(); + if (processShuttingDown) return { declined: true }; // isolated applications added or removed by the reload get their dedicated worker started or // stopped; a crash-looping newcomer can take a while, so the heartbeat runs through this too try { - for (const application of (await isolatedWorkerReconciler?.()) ?? []) freshlyStarted.add(application); + for (const application of (await workerHooks.reconcile?.()) ?? []) freshlyStarted.add(application); } catch (error) { harperLogger.error('Could not reconcile isolated application workers', error); } @@ -1475,7 +1489,7 @@ async function replaceWorkers(name, maxWorkersDown, startReplacementThreads, onP onProgress?.(); } - module.exports.restartNumber++; + restartNumber++; // `Infinity` is shutdownWorkers' "all at once" sentinel and must survive. if (typeof maxWorkersDown !== 'number' || Number.isNaN(maxWorkersDown)) { maxWorkersDown = 1; @@ -1485,10 +1499,10 @@ async function replaceWorkers(name, maxWorkersDown, startReplacementThreads, onP maxWorkersDown = maxWorkersDown * workers.length; } // make a copy of the workers before iterating them, as the workers array mutates a lot during this - let waitingToFinish = []; // promises for workers we are replacing, spliced as each is replaced + let waitingToFinish: any = []; // promises for workers we are replacing, spliced as each is replaced // Every replacement that was started without being awaited first, so this function can still // resolve only once each one is accepting connections. - let replacementsStarting = []; + let replacementsStarting: any = []; // A replacement that never came up leaves the pool in one of two very different states, and a // caller waiting on this restart needs them apart: the pre-start path keeps the *old* worker // serving old code, while a replacement that exits after its predecessor is already gone only @@ -1614,7 +1628,7 @@ async function replaceWorkers(name, maxWorkersDown, startReplacementThreads, onP // we leave the existing worker in place, and a background retry succeeding later would push the // pool over its configured worker count. Re-enabled once it has started. newWorker.wasShutdown = true; - let started = await new Promise((resolve) => { + let started = await new Promise((resolve) => { // Generous backstop so a replacement that deadlocks during init can't wedge the whole // restart forever. Far longer than any legitimate startup, so it never fires in practice. const giveUp = () => { @@ -1877,7 +1891,7 @@ function postShutdown(worker) { if (!workers.includes(worker)) return false; try { worker.postMessage({ - restartNumber: module.exports.restartNumber, + restartNumber: restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, }); } catch (err) { @@ -1890,8 +1904,8 @@ function postShutdown(worker) { } /** Resolves once a worker asked to shut down has exited, forcing it after the backstop it may extend. */ -function whenShutDownWorkerExits(worker, onProgress, onExit) { - return new Promise((resolve) => { +function whenShutDownWorkerExits(worker, onProgress, onExit?: any) { + return new Promise((resolve) => { // in case the exit inside the thread doesn't timeout, force it from the outside const armTerminate = (delay) => setTimeout(() => { @@ -1916,7 +1930,7 @@ function whenShutDownWorkerExits(worker, onProgress, onExit) { // Clamp the worker-requested deadline to the configured ceiling (and to a finite value) so a // buggy/rogue message can't defer the force-kill unboundedly; a shrink (drain-done reset) // passes through untouched. See boundedTerminateDelay for the arithmetic + its unit tests. - const { boundedTerminateDelay, getShutdownDrainCeilingMs } = require('../../components/shutdownDrain.ts'); + const { boundedTerminateDelay, getShutdownDrainCeilingMs } = loadRuntimeModule('components/shutdownDrain'); const delay = boundedTerminateDelay( deadlineMs, Date.now(), @@ -1955,7 +1969,7 @@ function setHeldReplacementTimeout(timeoutMs) { /** A held replacement's verdict, the predecessor's retirement where they cannot share a port, and its bind. */ function heldReplacementTimeoutMs() { if (heldReplacementTimeoutOverride !== undefined) return heldReplacementTimeoutOverride; - const { getShutdownDrainCeilingMs } = require('../../components/shutdownDrain.ts'); + const { getShutdownDrainCeilingMs } = loadRuntimeModule('components/shutdownDrain'); return canaryVerdictTimeoutMs() + Math.max(threadTerminationTimeout * 2, 60000) + getShutdownDrainCeilingMs(); } @@ -1966,8 +1980,8 @@ function heldReplacementTimeoutMs() { * @param newWorker The replacement worker * @returns {Promise} whether the worker reported that it started */ -function whenWorkerStarted(newWorker) { - return new Promise((resolve) => { +function whenWorkerStarted(newWorker?: any) { + return new Promise((resolve) => { const cleanup = () => { clearTimeout(timeout); newWorker.off('message', startListener); @@ -2004,13 +2018,16 @@ function whenWorkerStarted(newWorker) { newWorker.on('exit', exitListener); }); } -function shutdownWorkers(name) { +function shutdownWorkers(name?: any) { return restartWorkers(name, Infinity, false, null, '*'); } function beginProcessShutdown() { processShuttingDown = true; } -async function shutdownWorkersNow(name) { +function isProcessShuttingDown() { + return processShuttingDown; +} +async function shutdownWorkersNow(name?: any) { if (name == null) beginProcessShutdown(); shutdownWorkers(name); // set the state of all the workers to shut down. this should finish the important stuff synchronously if (isBun) { @@ -2029,29 +2046,26 @@ async function shutdownWorkersNow(name) { * The restart scope on the wire: a worker cannot post `undefined` and have it mean "the pool" (a * message with no scope at all means "everything", as every pre-existing sender intends). */ -function encodeRestartScope(application) { +function encodeRestartScope(application?: any) { return application === undefined ? '' : application; // '' is not a legal application name } -function decodeRestartScope(message) { +function decodeRestartScope(message?: any) { if (message.scope === undefined) return '*'; return message.scope === '' ? undefined : message.scope; } -let isolatedWorkerReconciler = null; -let runningIsolatedApplicationsGetter = () => []; /** Registered by socketRouter: starts and stops dedicated workers to match the root config's isolated applications. */ -function setIsolatedWorkerReconciler(reconcile) { - isolatedWorkerReconciler = reconcile; +function setIsolatedWorkerReconciler(reconcile?: any) { + workerHooks.reconcile = reconcile; } -function setRunningIsolatedApplicationsGetter(getter) { - runningIsolatedApplicationsGetter = getter; +function setRunningIsolatedApplicationsGetter(getter?: any) { + workerHooks.runningApplications = getter; } -const messageListeners = []; -function onMessageFromWorkers(listener) { +function onMessageFromWorkers(listener?: any) { messageListeners.push(listener); } -function onMessageByType(type, listener) { +function onMessageByType(type?: any, listener?: any) { let listeners = listenersByType.get(type); if (!listeners) listenersByType.set(type, (listeners = [])); listeners.push(listener); @@ -2066,7 +2080,7 @@ function onMessageByType(type, listener) { } const MAX_SYNC_BROADCAST = 10; -async function broadcast(message, includeSelf) { +async function broadcast(message?: any, includeSelf?: any) { let count = 0; for (let port of connectedPorts) { try { @@ -2074,7 +2088,7 @@ async function broadcast(message, includeSelf) { if (count++ > MAX_SYNC_BROADCAST) { // posting messages can be somewhat expensive, so we yield the event turn occassionally to not cause any delays. count = 0; - await new Promise(setImmediate); + await new Promise(setImmediate); } } catch (error) { harperLogger.error(`Unable to send message to worker`, error); @@ -2085,13 +2099,13 @@ async function broadcast(message, includeSelf) { } } -const awaitingResponses = new Map(); +const awaitingResponses: any = new Map(); let nextId = 1; // Backstop so a wedged-but-alive worker (one whose event loop is blocked and never acks, yet // whose port hasn't closed) can't hang a mutating admin/DDL op forever. Ordinary broadcasts happen // after the durable write and proceed best-effort; strict preparation broadcasts reject on timeout. const DEFAULT_ACK_TIMEOUT_MS = 30000; -function settleAcknowledgementsForClosedWorker(matches, jobCleanupComplete, exitConfirmed) { +function settleAcknowledgementsForClosedWorker(matches?: any, jobCleanupComplete?: any, exitConfirmed?: any) { for (const [, ackHandler] of awaitingResponses) { if (!matches(ackHandler.port)) continue; if (ackHandler.allowNormalJobExit && !jobCleanupComplete && !exitConfirmed) continue; @@ -2099,11 +2113,15 @@ function settleAcknowledgementsForClosedWorker(matches, jobCleanupComplete, exit } } -function settleAcknowledgementsForClosedPort(port, jobCleanupComplete = false, exitConfirmed = false) { +function settleAcknowledgementsForClosedPort(port?: any, jobCleanupComplete: any = false, exitConfirmed: any = false) { settleAcknowledgementsForClosedWorker((candidate) => candidate === port, jobCleanupComplete, exitConfirmed); } -function settleAcknowledgementsForClosedThread(threadId, jobCleanupComplete = false, exitConfirmed = false) { +function settleAcknowledgementsForClosedThread( + threadId?: any, + jobCleanupComplete: any = false, + exitConfirmed: any = false +) { settleAcknowledgementsForClosedWorker( (candidate) => candidate.threadId === threadId, jobCleanupComplete, @@ -2113,20 +2131,20 @@ function settleAcknowledgementsForClosedThread(threadId, jobCleanupComplete = fa /** @param {boolean|'active'} includeJobWorkers */ function broadcastWithAcknowledgement( - message, - timeout = DEFAULT_ACK_TIMEOUT_MS, - strict = false, - includeJobWorkers = false + message?: any, + timeout: any = DEFAULT_ACK_TIMEOUT_MS, + strict: any = false, + includeJobWorkers: any = false ) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { let waitingCount = 0; let timer; let initializing = true; - const failures = []; + const failures: any = []; // Tracks the handlers still awaiting an ack for THIS broadcast. Doubles as an // idempotency guard: a port's handler runs at most once whether it's driven by an ack, // the close listener, or the timeout below. - const pending = new Set(); + const pending: any = new Set(); const finish = () => { if (timer) { clearTimeout(timer); @@ -2140,14 +2158,14 @@ function broadcastWithAcknowledgement( reject(failures[0]); return; } - const error = new AggregateError(failures, 'A worker could not prepare for the schema change'); + const error: any = new AggregateError(failures, 'A worker could not prepare for the schema change'); for (const property of ['name', 'code', 'statusCode', 'retryable']) { const value = failures[0][property]; if (property === 'name' && value === 'Error') continue; if (value != null && failures.every((failure) => failure[property] === value)) error[property] = value; } reject(error); - } else resolve(); + } else resolve(undefined); }; for (let port of connectedPorts) { // Ordinary schema gossip excludes job workers to avoid re-entrant waits. Destructive @@ -2164,7 +2182,7 @@ function broadcastWithAcknowledgement( ackHandler = (response) => { if (!pending.delete(ackHandler)) return; // already settled for this port if (response?.error) { - const error = new Error( + const error: any = new Error( `Worker ${port.threadId} could not prepare for the schema change: ${response.error.message ?? response.error}` ); error.cause = response.error; @@ -2221,7 +2239,7 @@ function broadcastWithAcknowledgement( if (timeout > 0) { timer = setTimeout(() => { timer = undefined; - const stuck = []; + const stuck: any = []; for (let ackHandler of [...pending]) { stuck.push(ackHandler.port); ackHandler( @@ -2252,7 +2270,11 @@ function broadcastWithAcknowledgement( } /** @param {boolean|'active'} includeJobWorkers */ -function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS, includeJobWorkers = false) { +function broadcastWithStrictAcknowledgement( + message?: any, + timeout: any = DEFAULT_ACK_TIMEOUT_MS, + includeJobWorkers: any = false +) { return broadcastWithAcknowledgement(message, timeout, true, includeJobWorkers); } @@ -2266,7 +2288,7 @@ function getOsThreadId() { } } -function readTaskFile(tid, name) { +function readTaskFile(tid?: any, name?: any) { try { return readFileSync(`/proc/self/task/${tid}/${name}`, 'utf8').trim(); } catch { @@ -2276,13 +2298,13 @@ function readTaskFile(tid, name) { // Kernel-side view of one thread. Every field is best-effort and reported individually, since // wchan/syscall need ptrace read access that a hardened container may deny while stat is open. -function finiteOrUndefined(value) { +function finiteOrUndefined(value?: any) { const number = Number(value); return Number.isFinite(number) ? number : undefined; } -function readOsThreadState(tid) { - const thread = { tid }; +function readOsThreadState(tid?: any) { + const thread: any = { tid }; const stat = readTaskFile(tid, 'stat'); if (stat !== undefined) { // Fields after the parenthesized comm, so state is [0], utime/stime [11]/[12], starttime [19]. @@ -2307,8 +2329,8 @@ function readOsThreadState(tid) { return thread; } -function snapshotWorkerThread(worker) { - const snapshot = { at: Date.now() }; +function snapshotWorkerThread(worker?: any) { + const snapshot: any = { at: Date.now() }; if (worker.resources?.updated) snapshot.sinceResourceReport = snapshot.at - worker.resources.updated; const eventLoop = worker.performance?.eventLoopUtilization?.(); if (eventLoop) snapshot.eventLoop = { idle: eventLoop.idle, active: eventLoop.active }; @@ -2316,12 +2338,12 @@ function snapshotWorkerThread(worker) { return snapshot; } -function describeThreadState(thread) { +function describeThreadState(thread?: any) { return `state=${thread.state ?? '?'} wchan=${thread.wchan ?? '?'} syscall=${thread.syscall ?? '?'}`; } -function describeSnapshot(snapshot) { - const parts = [ +function describeSnapshot(snapshot?: any) { + const parts: any = [ snapshot.sinceResourceReport === undefined ? 'no resource report received' : `last resource report ${snapshot.sinceResourceReport}ms ago`, @@ -2339,12 +2361,12 @@ function describeSnapshot(snapshot) { return parts.join('; '); } -function describeDelta(before, after) { +function describeDelta(before?: any, after?: any) { return Number.isFinite(before) && Number.isFinite(after) ? `+${after - before}` : '?'; } -function describeProgress(first, second) { - const parts = []; +function describeProgress(first?: any, second?: any) { + const parts: any = []; if (first.eventLoop && second.eventLoop) parts.push( `event loop active +${Math.round(second.eventLoop.active - first.eventLoop.active)}ms idle +${Math.round(second.eventLoop.idle - first.eventLoop.idle)}ms` @@ -2362,7 +2384,7 @@ function describeProgress(first, second) { // "parked on a lock" (no CPU ticks, no context switches) from "spinning". const STUCK_WORKER_SAMPLE_INTERVAL_MS = 1000; const STUCK_WORKER_DIAGNOSTIC_COOLDOWN_MS = 30000; -function logStuckWorkerDiagnostics(worker) { +function logStuckWorkerDiagnostics(worker?: any) { if (!worker || !workers.includes(worker)) return; const now = Date.now(); if (worker.stuckDiagnosticAt !== undefined && now - worker.stuckDiagnosticAt < STUCK_WORKER_DIAGNOSTIC_COOLDOWN_MS) @@ -2386,7 +2408,7 @@ function logStuckWorkerDiagnostics(worker) { }, STUCK_WORKER_SAMPLE_INTERVAL_MS).unref(); } -function sendThreadInfo(targetWorker) { +function sendThreadInfo(targetWorker?: any) { targetWorker.postMessage({ type: THREAD_INFO, workers: getChildWorkerInfo(), @@ -2413,15 +2435,14 @@ function getChildWorkerInfo() { * @param worker * @param message */ -function recordResourceReport(worker, message) { +function recordResourceReport(worker?: any, message?: any) { worker.resources = message; // we want to record when this happens so we know if it has reported recently worker.resources.updated = Date.now(); } -let monitorListener; -function setMonitorListener(listener) { - monitorListener = listener; +function setMonitorListener(listener?: any) { + workerHooks.monitorListener = listener; } const MONITORING_INTERVAL = 1000; @@ -2430,15 +2451,15 @@ const MONITORING_INTERVAL = 1000; const PINNED_ELU_UTILIZATION_THRESHOLD = 0.99; const PINNED_ELU_SUSTAINED_MS = 30_000; const PINNED_ELU_SUSTAINED_TICKS = Math.ceil(PINNED_ELU_SUSTAINED_MS / MONITORING_INTERVAL); -module.exports.PINNED_ELU_UTILIZATION_THRESHOLD = PINNED_ELU_UTILIZATION_THRESHOLD; -module.exports.PINNED_ELU_SUSTAINED_TICKS = PINNED_ELU_SUSTAINED_TICKS; +export { PINNED_ELU_UTILIZATION_THRESHOLD }; +export { PINNED_ELU_SUSTAINED_TICKS }; -function describePinnedWorker(worker) { +function describePinnedWorker(worker?: any) { const identity = [worker.name, worker.application].filter(Boolean).join('/'); return `Worker thread ${worker.threadId}${identity ? ` (${identity})` : ''}`; } -function checkPinnedWorkerELU(worker, recentELU) { +function checkPinnedWorkerELU(worker?: any, recentELU?: any) { const { utilization } = recentELU; // idle can briefly read negative while a worker tears down, producing a nonsense ratio // outside [0, 1]; treat that tick as unmeasured rather than counting or resetting on it. @@ -2461,7 +2482,7 @@ function checkPinnedWorkerELU(worker, recentELU) { } } -function sampleWorkerELU(worker) { +function sampleWorkerELU(worker?: any) { if (!isBun && worker.performance?.eventLoopUtilization) { let current_ELU = worker.performance.eventLoopUtilization(); let recent_ELU; @@ -2491,7 +2512,7 @@ function startMonitoring() { // utilization levels (last second) and so we don't have to make these calls to frequently setInterval(() => { for (let worker of workers) sampleWorkerELU(worker); - if (monitorListener) monitorListener(); + if (workerHooks.monitorListener) workerHooks.monitorListener(); }, MONITORING_INTERVAL).unref(); } const REPORTING_INTERVAL = 1000; @@ -2499,12 +2520,13 @@ const REPORTING_INTERVAL = 1000; if (parentPort && workerData?.addPorts) { // Main thread always has threadId 0 (worker_threads convention). Stamp it on // parentPort so sendToThread(0, ...) and similar lookups can route back to main. + // @ts-expect-error - stamping threadId on MessagePort for routing purposes parentPort.threadId = 0; addPort(parentPort); const osThreadId = getOsThreadId(); if (osThreadId !== undefined) parentPort.postMessage({ type: OS_THREAD_ID, osThreadId }); for (let i = 0, l = workerData.addPorts.length; i < l; i++) { - let port = workerData.addPorts[i]; + let port: any = workerData.addPorts[i]; port.threadId = workerData.addThreadIds[i]; addPort(port, false, workerData.addPortIsJobWorkers?.[i]); } @@ -2520,7 +2542,7 @@ if (parentPort && workerData?.addPorts) { }); }, REPORTING_INTERVAL).unref(); getThreadInfo = (timeoutMs) => - new Promise((resolve, reject) => { + new Promise((resolve, reject) => { // Request thread info from the parent thread and wait for it to respond with info on all threads. let timeout; parentPort.on('message', receiveThreadInfo); @@ -2531,7 +2553,7 @@ if (parentPort && workerData?.addPorts) { reject(error); return; } - function receiveThreadInfo(message) { + function receiveThreadInfo(message?: any) { if (message.type === THREAD_INFO) { cleanup(); resolve(message.workers); @@ -2544,7 +2566,7 @@ if (parentPort && workerData?.addPorts) { if (timeoutMs != null) { timeout = setTimeout(() => { cleanup(); - const error = new Error(`Timed out waiting for thread information after ${timeoutMs}ms`); + const error: any = new Error(`Timed out waiting for thread information after ${timeoutMs}ms`); error.code = 'ERR_THREAD_INFO_TIMEOUT'; reject(error); }, timeoutMs); @@ -2552,7 +2574,7 @@ if (parentPort && workerData?.addPorts) { }); let nextRunningIsolatedApplicationsRequestId = 0; getRunningIsolatedApplications = (timeoutMs) => - new Promise((resolve, reject) => { + new Promise((resolve, reject) => { const requestId = ++nextRunningIsolatedApplicationsRequestId; let timeout; parentPort.on('message', receiveApplications); @@ -2563,7 +2585,7 @@ if (parentPort && workerData?.addPorts) { reject(error); return; } - function receiveApplications(message) { + function receiveApplications(message?: any) { if (message.type === RUNNING_ISOLATED_APPLICATIONS && message.requestId === requestId) { cleanup(); resolve(message.applications); @@ -2576,7 +2598,7 @@ if (parentPort && workerData?.addPorts) { if (timeoutMs != null) { timeout = setTimeout(() => { cleanup(); - const error = new Error(`Timed out waiting for isolated application topology after ${timeoutMs}ms`); + const error: any = new Error(`Timed out waiting for isolated application topology after ${timeoutMs}ms`); error.code = 'ERR_ISOLATED_APPLICATIONS_TIMEOUT'; reject(error); }, timeoutMs); @@ -2584,7 +2606,7 @@ if (parentPort && workerData?.addPorts) { }); let awaitTerminationRequestId = 0; awaitProcessGroupTermination = (ownerThreadId, signal) => - new Promise((resolve) => { + new Promise((resolve) => { // Deliberately no timeout: a contender must not reclaim a dead owner's lock while its // process group might still be alive and mutating files, so this mirrors the unbounded // wait Application.ts's waitForConfirmedTermination uses for the same reason. A caller @@ -2597,33 +2619,32 @@ if (parentPort && workerData?.addPorts) { function cleanup() { parentPort.off('message', receiveConfirmation); } - function receiveConfirmation(message) { + function receiveConfirmation(message?: any) { if (message.type === PROCESS_GROUP_TERMINATION_CONFIRMED && message.requestId === requestId) { cleanup(); - resolve(); + resolve(undefined); } } }); } else { getThreadInfo = getChildWorkerInfo; getRunningIsolatedApplications = isMainThread - ? () => runningIsolatedApplicationsGetter() + ? () => workerHooks.runningApplications() : () => { - const error = new Error('No channel to the main thread for isolated application topology'); + const error: any = new Error('No channel to the main thread for isolated application topology'); error.code = 'ERR_ISOLATED_APPLICATIONS_UNAVAILABLE'; return Promise.reject(error); }; awaitProcessGroupTermination = (ownerThreadId) => - pendingProcessGroupTerminations.get(ownerThreadId) ?? Promise.resolve(); + pendingProcessGroupTerminations.get(ownerThreadId) ?? Promise.resolve(undefined); } -module.exports.getThreadInfo = getThreadInfo; -module.exports.getRunningIsolatedApplications = getRunningIsolatedApplications; +export { getThreadInfo }; +export { getRunningIsolatedApplications }; // Listeners notified when a connected thread's port closes (worker exit/restart), so // modules holding per-thread state (e.g. registeredOperations' registry and in-flight // forwards) can clean up. -const threadExitListeners = []; -function onThreadExit(listener) { +function onThreadExit(listener?: any) { threadExitListeners.push(listener); } @@ -2631,28 +2652,28 @@ function onThreadExit(listener) { // regardless of which of the two removal paths below observes it first. Node worker_threads // ids are monotonically increasing and never reused within a process, so this never needs // pruning (unbounded growth is one entry per worker restart over the process lifetime). -const notifiedDeadThreadIds = new Set(); -const processGroupsByThread = new Map(); +const notifiedDeadThreadIds: any = new Set(); +const processGroupsByThread: any = new Map(); // A dead thread's process groups are killed asynchronously (SIGKILL/taskkill only queue the // request). While that termination is in flight, isThreadRunning must keep reporting the owner // as alive — otherwise a contender can delete the dead worker's lock claim and start a new // preparation before its old process tree is confirmed gone, reopening the concurrent-writer // window component-preparation locking exists to close. -const pendingProcessGroupTerminations = new Map(); +const pendingProcessGroupTerminations: any = new Map(); const PROCESS_GROUP_TERMINATION_POLL_MS = 25; const ZOMBIE_GROUP_MEMBER_SCAN_INTERVAL_MS = 1000; const PROCESS_GROUP_LIVENESS_WARNING_MS = 30000; // Bounds isThreadRunning's own wait (below); Application.ts's waitForConfirmedTermination stays // deliberately unbounded. const THREAD_RUNNING_TERMINATION_BACKSTOP_MS = PROCESS_GROUP_LIVENESS_WARNING_MS; -const zombieGroupScanTimes = new Map(); -const processGroupLivenessStates = new Map(); +const zombieGroupScanTimes: any = new Map(); +const processGroupLivenessStates: any = new Map(); // When each group's root was registered — by then it was already running, which is what lets the // Windows scan tell our root from a later process that recycled its PID. -const processGroupSpawnedAt = new Map(); +const processGroupSpawnedAt: any = new Map(); let processGroupRegistrationGeneration = 0; -function processGroupExists(processGroupId) { +function processGroupExists(processGroupId?: any) { try { process.kill(-processGroupId, 0); return true; @@ -2661,11 +2682,11 @@ function processGroupExists(processGroupId) { } } -function processProbeError(error) { +function processProbeError(error?: any) { return error?.code ?? error?.message ?? String(error ?? 'unknown error'); } -function processGroupLeaderState(processGroupId, platform, readStat) { +function processGroupLeaderState(processGroupId?: any, platform?: any, readStat?: any) { if (platform !== 'linux') { return { state: 'unknown', reason: `zombie-process detection is unavailable on ${platform}` }; } @@ -2684,7 +2705,7 @@ function processGroupLeaderState(processGroupId, platform, readStat) { return { state: fields[0] === 'Z' ? 'zombie' : 'alive' }; } -function scanLinuxProcessGroup(processGroupId, readDirectory, readStat) { +function scanLinuxProcessGroup(processGroupId?: any, readDirectory?: any, readStat?: any) { let processIds; try { processIds = readDirectory('/proc'); @@ -2721,7 +2742,7 @@ function scanLinuxProcessGroup(processGroupId, readDirectory, readStat) { return { isAlive: false }; } -function keepProcessGroupAlive(processGroupId, reason, observationTime, warn) { +function keepProcessGroupAlive(processGroupId?: any, reason?: any, observationTime?: any, warn?: any) { let livenessState = processGroupLivenessStates.get(processGroupId); if (!livenessState) { livenessState = { observedAt: observationTime, reason, warned: false }; @@ -2738,12 +2759,12 @@ function keepProcessGroupAlive(processGroupId, reason, observationTime, warn) { return true; } -function clearProcessGroupLivenessState(processGroupId) { +function clearProcessGroupLivenessState(processGroupId?: any) { zombieGroupScanTimes.delete(processGroupId); processGroupLivenessStates.delete(processGroupId); } -function isProcessGroupAlive(processGroupId, options) { +function isProcessGroupAlive(processGroupId?: any, options?: any) { const platform = options?.platform ?? process.platform; const groupExists = options?.processGroupExists ?? processGroupExists; const readDirectory = options?.readDirectory ?? readdirSync; @@ -2778,11 +2799,11 @@ function isProcessGroupAlive(processGroupId, options) { return false; } -function processGroupIsAlive(processGroupId) { +function processGroupIsAlive(processGroupId?: any) { return isProcessGroupAlive(processGroupId); } -async function waitForProcessGroupExit(processGroupId, registration) { +async function waitForProcessGroupExit(processGroupId?: any, registration?: any) { while (true) { const currentRegistration = processGroupSpawnedAt.get(processGroupId); if (currentRegistration !== undefined && currentRegistration !== registration) return; @@ -2799,7 +2820,7 @@ async function waitForProcessGroupExit(processGroupId, registration) { // way the wait confirms via the process table — reclamation must not proceed on a guess. The // root was created inside the spawner's spawn() call, whose start and return times travel with the // registration so the cross-thread hop adds nothing to the window before it. -function waitForWindowsGroupExit(processGroupId, spawn, killedAt) { +function waitForWindowsGroupExit(processGroupId?: any, spawn?: any, killedAt?: any) { return confirmWindowsProcessTreeGone( { rootPid: processGroupId, @@ -2812,7 +2833,13 @@ function waitForWindowsGroupExit(processGroupId, spawn, killedAt) { ); } -function addProcessGroup(ownerThreadId, processGroupId, spawnedAt, spawnStartedAt, registrationGeneration) { +function addProcessGroup( + ownerThreadId?: any, + processGroupId?: any, + spawnedAt?: any, + spawnStartedAt?: any, + registrationGeneration?: any +) { if (!Number.isInteger(processGroupId) || processGroupId <= 0) return; const previousRegistration = processGroupSpawnedAt.get(processGroupId); if ( @@ -2835,7 +2862,7 @@ function addProcessGroup(ownerThreadId, processGroupId, spawnedAt, spawnStartedA }); } -function removeProcessGroup(ownerThreadId, processGroupId, registrationGeneration) { +function removeProcessGroup(ownerThreadId?: any, processGroupId?: any, registrationGeneration?: any) { const currentRegistration = processGroupSpawnedAt.get(processGroupId); const sameOwnerNewerGeneration = currentRegistration?.ownerThreadId === ownerThreadId && @@ -2869,9 +2896,9 @@ function removeProcessGroup(ownerThreadId, processGroupId, registrationGeneratio // path has no guarantee the async loop gets even one scan in before `process.exit()` runs, and by // then this function has already dropped the registration the exit handler would otherwise have // caught, so the blind kill has to fire here instead. -function terminateProcessGroupsForThread(ownerThreadId, { fromExitHandler = false } = {}) { +function terminateProcessGroupsForThread(ownerThreadId?: any, { fromExitHandler = false } = {}) { const processGroups = processGroupsByThread.get(ownerThreadId); - if (!processGroups) return pendingProcessGroupTerminations.get(ownerThreadId) ?? Promise.resolve(); + if (!processGroups) return pendingProcessGroupTerminations.get(ownerThreadId) ?? Promise.resolve(undefined); processGroupsByThread.delete(ownerThreadId); // Membership is not ownership: a PID is reusable the moment its process exits. const groupIds = [...processGroups].filter((processGroupId) => { @@ -2881,7 +2908,7 @@ function terminateProcessGroupsForThread(ownerThreadId, { fromExitHandler = fals ); return false; }); - const killedAt = new Map(); + const killedAt: any = new Map(); for (const processGroupId of groupIds) { try { if (process.platform === 'win32') { @@ -2922,7 +2949,7 @@ function terminateProcessGroupsForThread(ownerThreadId, { fromExitHandler = fals // `spawnedAt` / `spawnStartedAt`: the caller's clock as its spawn() of the group's root returned and // as it was called — the root was created between the two. -function registerProcessGroup(processGroupId, spawnedAt = Date.now(), spawnStartedAt) { +function registerProcessGroup(processGroupId?: any, spawnedAt: any = Date.now(), spawnStartedAt?: any) { const registrationGeneration = ++processGroupRegistrationGeneration; if (isMainThread) addProcessGroup(threadId, processGroupId, spawnedAt, spawnStartedAt, registrationGeneration); else { @@ -2937,7 +2964,7 @@ function registerProcessGroup(processGroupId, spawnedAt = Date.now(), spawnStart return registrationGeneration; } -function unregisterProcessGroup(processGroupId, registrationGeneration) { +function unregisterProcessGroup(processGroupId?: any, registrationGeneration?: any) { if (isMainThread) removeProcessGroup(threadId, processGroupId, registrationGeneration); else parentPort?.postMessage({ type: UNREGISTER_PROCESS_GROUP, processGroupId, registrationGeneration }); } @@ -2954,8 +2981,8 @@ class ProcessGroupTerminationUnconfirmedError extends Error { // "not running": componentPreparationLock's ownerIsAlive/ownerLivenessConfirmed already treat a // throwing isOwnerAlive as "can't confirm — don't steal the claim, don't renew the deadline", so // the lock's own bounded wait is what eventually fails the waiter. -async function awaitConfirmedProcessGroupTermination(ownerThreadId) { - const abortController = new AbortController(); +async function awaitConfirmedProcessGroupTermination(ownerThreadId?: any) { + const abortController: any = new AbortController(); let timedOut = false; const timer = setTimeout(() => { timedOut = true; @@ -2965,7 +2992,7 @@ async function awaitConfirmedProcessGroupTermination(ownerThreadId) { try { await Promise.race([ awaitProcessGroupTermination(ownerThreadId, abortController.signal), - new Promise((resolve) => abortController.signal.addEventListener('abort', resolve, { once: true })), + new Promise((resolve) => abortController.signal.addEventListener('abort', resolve, { once: true })), ]); } finally { clearTimeout(timer); @@ -2978,7 +3005,7 @@ async function awaitConfirmedProcessGroupTermination(ownerThreadId) { } } -async function isThreadRunning(ownerThreadId, timeoutMs = THREAD_INFO_REQUEST_TIMEOUT_MS) { +async function isThreadRunning(ownerThreadId?: any, timeoutMs: any = THREAD_INFO_REQUEST_TIMEOUT_MS) { if (ownerThreadId === threadId || ownerThreadId === 0) return true; if ((await getThreadInfo(timeoutMs)).some((worker) => worker.threadId === ownerThreadId)) return true; // The thread itself is gone, but it may still own process groups whose forced termination is @@ -2999,11 +3026,11 @@ if (isMainThread) { * reads the tombstone `notifyThreadExit` records below — callers on the exit path need an answer * without awaiting process-group confirmation. */ -function hasThreadExited(threadId) { +function hasThreadExited(threadId?: any) { return notifiedDeadThreadIds.has(threadId); } -function notifyThreadExit(deadThreadId) { +function notifyThreadExit(deadThreadId?: any) { if (deadThreadId == null || notifiedDeadThreadIds.has(deadThreadId)) return; notifiedDeadThreadIds.add(deadThreadId); handleDatabaseDropPreparationOwnerExit(deadThreadId); @@ -3016,7 +3043,7 @@ function notifyThreadExit(deadThreadId) { } } -function removePort(port, deadThreadId) { +function removePort(port?: any, deadThreadId?: any) { // A sibling may already have announced this dead thread and removed its port. Process-group // cleanup must still run when the authoritative close/exit event reaches this thread. if (deadThreadId != null) terminateProcessGroupsForThread(deadThreadId); @@ -3047,7 +3074,7 @@ function removePort(port, deadThreadId) { } } -function addPort(port, keepRef, isJobWorker) { +function addPort(port?: any, keepRef?: any, isJobWorker?: any) { if (isJobWorker) port.isJobWorker = true; connectedPorts.push(port); // Capture threadId now — Bun resets port.threadId to -1 by the time 'exit' fires. @@ -3110,7 +3137,7 @@ function addPort(port, keepRef, isJobWorker) { if (keepRef) port.refCount = 100; else port.unref(); } -function notifyMessageListeners(message, port) { +function notifyMessageListeners(message?: any, port?: any) { for (let listener of messageListeners) { listener(message, port); } @@ -3135,11 +3162,12 @@ function notifyMessageListeners(message, port) { } } } +export let watchDir; if (isMainThread) { let beforeRestart, queuedRestart; - let changedFiles = new Set(); - const ignoredPaths = ['node_modules', '.git']; - const watchDir = async (dir, beforeRestartCallback) => { + let changedFiles: any = new Set(); + const ignoredPaths: any = ['node_modules', '.git']; + watchDir = async (dir, beforeRestartCallback) => { if (beforeRestartCallback) beforeRestart = beforeRestartCallback; const watchTarget = resolveWatchTarget(dir); let usingPolling = watchTarget.mustPoll; @@ -3185,11 +3213,11 @@ if (isMainThread) { }; openWatcher(); }; - module.exports.watchDir = watchDir; if (process.env.WATCH_DIR) watchDir(process.env.WATCH_DIR); } else { onMessageByType(hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, async (message) => { - module.exports.restartNumber = message.restartNumber; + restartNumber = message.restartNumber; + beginProcessShutdown(); parentPort.unref(); // remove this handle armSelfExit(threadTerminationTimeout); }); @@ -3204,4 +3232,6 @@ if (isMainThread) registerCertificationRequests(); // Required here, not from logging, which must never import this module; last in the file because it // calls onThreadExit. -require('./logRotationTransport.ts'); +onStartup(async () => { + await import('./logRotationTransport.ts'); +}); diff --git a/server/threads/processIncarnation.ts b/server/threads/processIncarnation.ts new file mode 100644 index 0000000000..2258431b75 --- /dev/null +++ b/server/threads/processIncarnation.ts @@ -0,0 +1,6 @@ +import { randomBytes } from 'node:crypto'; +import { workerData } from 'node:worker_threads'; + +export const processIncarnation: string | undefined = workerData + ? workerData.processIncarnation + : randomBytes(8).toString('hex'); diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index 1dff9e420a..031e31d46d 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -1,3 +1,4 @@ +import { RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; import { startWorker, setMonitorListener, @@ -7,7 +8,8 @@ import { setRunningIsolatedApplicationsGetter, workersForApplication, stopWorker, -} from './manageThreads.js'; + isProcessShuttingDown, +} from './manageThreads.ts'; import { presentIsolatedApplicationNames, isolatedApplicationRefusal, @@ -51,6 +53,7 @@ if (isMainThread) { export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean) { const workerSlots = []; + if (isProcessShuttingDown()) return []; let dedicatedPools: { type: string; count: number }[] = []; // Crash-path defense: a hard crash can skip a worker's exit-time UDS cleanup and leave stale // mirror files behind. This runs before any worker below can start (and thus before any mirror @@ -64,7 +67,9 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean // determine database storage path" before main() ever runs. if (isMainThread && !sweptSocketsDirectory) { sweptSocketsDirectory = true; - (await import('../http.ts')).cleanupSocketsDirectory(); + const { cleanupSocketsDirectory } = await import('../http.ts'); + if (isProcessShuttingDown()) return []; + cleanupSocketsDirectory(); } recordHostname().catch((err) => harperLogger.error?.('Error recording hostname for analytics:', err)); // Drive transaction-log cooling from the main thread (the registry is a @@ -82,15 +87,18 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean workerSlots.push(slot); poolSlots.push(slot); } else { - const { loadRootComponents } = require('../loadRootComponents.js'); + const { loadRootComponents } = await import('../loadRootComponents.ts'); + if (isProcessShuttingDown()) return []; if (threadCount === 0) { if (configuredReplicationThreads() > 0) harperLogger.warn( 'replication.threads is ignored because threads.count is 0: replication runs on the main thread' ); setMainIsWorker(true); - const threadServer = require('./threadServer.js'); + const threadServer = await import('./threadServer.ts'); + if (isProcessShuttingDown()) return []; await threadServer.startServers(); + if (isProcessShuttingDown()) return []; // startServers() schedules listener startup after loading components; await its cached // batch so a bind failure reaches bin/run.ts and exits non-zero in single-thread mode too. await threadServer.listenOnPorts(); @@ -101,11 +109,14 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean setActiveWorkerPools(dedicatedPools.map((pool) => pool.type)); } await loadRootComponents(); - const { listenOnPorts } = require('./threadServer.js'); + if (isProcessShuttingDown()) return []; + const { listenOnPorts } = await import('./threadServer.ts'); + if (isProcessShuttingDown()) return []; await listenOnPorts(); // Windows does not support SO_REUSEPORT, so only a single HTTP worker is supported. if (process.platform === 'win32') threadCount = 1; } + if (isProcessShuttingDown()) return []; poolSize = threadCount; nextIsolatedIndex = Math.max(nextIsolatedIndex, threadCount); const isolated = admittedIsolatedApplications([...isolatedSlots.keys()]); @@ -260,6 +271,7 @@ async function reconcileIsolatedWorkersNow(): Promise { for (const application of stoppedApplications) cleanupApplicationSockets(application); } const started: string[] = []; + if (isProcessShuttingDown()) return started; const heapShareCount = poolSize + dedicatedWorkerCount + wanted.size; for (const slot of poolSlots) slot.setHeapShareCount(heapShareCount); for (const slot of isolatedSlots.values()) slot.setHeapShareCount(heapShareCount); @@ -434,7 +446,7 @@ function startWorkerSlot( } }, }; - startWorker(join(__dirname, './threadServer.js'), workerOptions); + startWorker(join(RUNTIME_SRC_ROOT, `server/threads/threadServer${RUNTIME_FILE_EXT}`), workerOptions); // Stop of a dedicated worker whose application is gone: every worker carrying the application, // a crashed one's replacement still booting included, so none is left running the removed app. let shutdownPromise: Promise | undefined; diff --git a/server/threads/threadMessageState.ts b/server/threads/threadMessageState.ts new file mode 100644 index 0000000000..185d4fa758 --- /dev/null +++ b/server/threads/threadMessageState.ts @@ -0,0 +1,10 @@ +export const listenersByType = new Map void> | null>(); +export const messagesQueuedByType = new Map(); +export const messageListeners: Array<(message: any, port?: any) => void> = []; +export const threadExitListeners: Array<(...args: any[]) => void> = []; +export const workerHooks: { + reconcile: (() => any) | null; + runningApplications: () => any[]; + monitorListener?: () => void; + resetRestartNeeded?: () => void; +} = { reconcile: null, runningApplications: () => [] }; diff --git a/server/threads/threadServer.js b/server/threads/threadServer.ts similarity index 77% rename from server/threads/threadServer.js rename to server/threads/threadServer.ts index d1bf628d67..73c9263c29 100644 --- a/server/threads/threadServer.js +++ b/server/threads/threadServer.ts @@ -1,98 +1,115 @@ -'use strict'; - -const { isMainThread, parentPort, threadId, workerData } = require('node:worker_threads'); -const { createServer: createSocketServer } = require('node:net'); -const { unlinkSync, existsSync, renameSync } = require('node:fs'); -const { join, dirname } = require('node:path'); +declare const Bun: any; +import * as loadRootComponentsRuntimeModule from '../loadRootComponents.ts'; +import * as databasesRuntimeModule from '../../resources/databases.ts'; +import * as uwsServerRuntimeModule from '../serverHelpers/uwsServer.ts'; +import * as inspector from 'node:inspector'; +import { isMainThread, parentPort, threadId, workerData } from 'node:worker_threads'; +import { createServer as createSocketServer } from 'node:net'; +import { unlinkSync, existsSync, renameSync, realpathSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT, loadRuntimeModule } from '../../utility/packageUtils.js'; let componentsLoadedResolve; -exports.whenComponentsLoaded = new Promise((resolve) => { +export const whenComponentsLoaded: any = new Promise((resolve) => { componentsLoadedResolve = resolve; }); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const env = require('../../utility/environment/environmentManager.ts'); -const terms = require('../../utility/hdbTerms.ts'); -const { server } = require('../Server.ts'); -let { createServer: createSecureSocketServer } = require('node:tls'); -const { +import harperLogger from '../../utility/logging/harper_logger.ts'; +import * as env from '../../utility/environment/environmentManager.ts'; +import * as terms from '../../utility/hdbTerms.ts'; +import { server } from '../Server.ts'; +import { createServer as _createSecureSocketServer } from 'node:tls'; +let createSecureSocketServer = _createSecureSocketServer; +import { restartNumber, getWorkerIndex, extendShutdownDeadline, restoreShutdownDeadline, -} = require('./manageThreads.js'); -const { + beginProcessShutdown, + isProcessShuttingDown, +} from './manageThreads.ts'; +import { runShutdownDrains, shutdownDrainsHaveWork, getShutdownDrainCeilingMs, -} = require('../../components/shutdownDrain.ts'); -const { realExit } = require('./workerProcessGuard.ts'); -const { +} from '../../components/shutdownDrain.ts'; +import { realExit } from './workerProcessGuard.ts'; +import { thisThreadsIsolatedApplication, applicationSocketName, shouldStartUwsListenerHere, -} = require('./isolatedApplications.ts'); -const { +} from './isolatedApplications.ts'; +import { shouldBindListenerHere, isDedicatedPoolWorker, poolMemberIndex, claimListener, listenerOwner, -} = require('./workerPools.ts'); -const { isBun } = require('../serverHelpers/Request.ts'); -const { getDomainSocketPathMaxBytes, isDomainSocketPathTooLong } = require('../../utility/domainSocket.ts'); -const { createTLSSelector, getEffectiveTlsCiphers } = require('../../security/keys.ts'); -const { startupLog } = require('../../bin/run.ts'); -const { SERVERS, setPortServerMap, portServer, socketOptionDefaults } = require('../serverRegistry.ts'); -const httpComponent = require('../http.ts'); -const { withProxyProtocol } = require('../serverHelpers/proxyProtocol.ts'); -const globals = require('../../globals.js'); -const { whenScopesClosed } = require('../../components/scopeShutdown.ts'); +} from './workerPools.ts'; +import { isBun } from '../serverHelpers/Request.ts'; +import { getDomainSocketPathMaxBytes, isDomainSocketPathTooLong } from '../../utility/domainSocket.ts'; +import { createTLSSelector, getEffectiveTlsCiphers } from '../../security/keys.ts'; +import { startupLog } from '../../bin/run.ts'; +import { SERVERS, setPortServerMap, portServer, socketOptionDefaults } from '../serverRegistry.ts'; +import * as httpComponent from '../http.ts'; +import { withProxyProtocol } from '../serverHelpers/proxyProtocol.ts'; +import * as globals from '../../globals.js'; +import { whenScopesClosed } from '../../components/scopeShutdown.ts'; +import { getComponentName } from '../../components/componentLoader.ts'; +import { onStartup } from '../../utility/lifecycle.ts'; -const debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG); +let debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG); +let inspectorInitialized = false; const isWindows = process.platform === 'win32'; server.socket = onSocket; -if (!isBun) { - if (debugThreads) { - let port; - if (isMainThread) { - port = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_PORT) ?? 9229; - const closeInspector = () => { +function initializeInspector() { + if (inspectorInitialized) return; + inspectorInitialized = true; + debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG); + if (!isBun) { + if (debugThreads) { + let port; + if (isMainThread) { + port = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_PORT) ?? 9229; + const closeInspector = () => { + try { + inspector.close(); + } catch (error) { + harperLogger.info('Could not close debugger', error); + } + }; + for (const signal of ['SIGINT', 'SIGTERM', 'SIGQUIT', 'exit']) { + process.on(signal, closeInspector); + } + } else { + const startingPort = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_STARTINGPORT); + if (startingPort && getWorkerIndex() >= 0) { + port = startingPort + getWorkerIndex(); + } + } + if (port) { + const host = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_HOST); + const waitForDebugger = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_WAITFORDEBUGGER); try { - require('inspector').close(); + inspector.open(port, host, waitForDebugger); } catch (error) { - harperLogger.info('Could not close debugger', error); + harperLogger.trace(`Could not start debugging on port ${port}, you may already be debugging:`, error.message); } - }; - for (const signal of ['SIGINT', 'SIGTERM', 'SIGQUIT', 'exit']) { - process.on(signal, closeInspector); } - } else { - const startingPort = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_STARTINGPORT); - if (startingPort && getWorkerIndex() >= 0) { - port = startingPort + getWorkerIndex(); - } - } - if (port) { - const host = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_HOST); - const waitForDebugger = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_WAITFORDEBUGGER); + } else if (process.env.DEV_MODE && isMainThread) { try { - require('inspector').open(port, host, waitForDebugger); + inspector.open(9229); } catch (error) { - harperLogger.trace(`Could not start debugging on port ${port}, you may already be debugging:`, error.message); + if (restartNumber <= 1) + harperLogger.trace('Could not start debugging on port 9229, you may already be debugging:', error.message); } } - } else if (process.env.DEV_MODE && isMainThread) { - try { - require('inspector').open(9229); - } catch (error) { - if (restartNumber <= 1) - harperLogger.trace('Could not start debugging on port 9229, you may already be debugging:', error.message); - } } } +if (debugThreads !== undefined && !isMainThread) initializeInspector(); +onStartup(initializeInspector); -process.on('uncaughtException', (error) => { +process.on('uncaughtException', (error: any) => { if (error.isHandled) return; if (error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED') return; // that's what network connections do if (error.message === 'write EIO') return; // that means the terminal is closed @@ -102,16 +119,15 @@ process.on('uncaughtException', (error) => { // handler is registered. Without this, any async path that rejects without being caught // (e.g. a cache-update commit error when the caller has already resolved) will kill the // worker thread. Mirror the uncaughtException behavior: log and continue. -process.on('unhandledRejection', (reason) => { +process.on('unhandledRejection', (reason: any) => { if (reason?.isHandled) return; harperLogger.error('unhandledRejection', reason); }); -env.initSync(); -exports.globals = globals; -exports.listenOnDomainSocket = listenOnDomainSocket; -exports.listenOnPorts = listenOnPorts; -exports.startServers = startServers; -exports.closeServers = closeServers; +export { globals }; +export { listenOnDomainSocket }; +export { listenOnPorts }; +export { startServers }; +export { closeServers }; let serversClosing = false; function closeServers() { @@ -127,9 +143,9 @@ function closeServers() { } } // Give pending requests time to finish, then exit - return new Promise((resolve) => setTimeout(resolve, 5000).unref()); + return new Promise((resolve) => setTimeout(resolve, 5000).unref()); } - const promises = []; + const promises: any = []; for (let port in SERVERS) { const server = SERVERS[port]; if (server.closeIdleConnections) { @@ -166,7 +182,7 @@ function closeServers() { } // And we tell the server not to accept any more incoming connections promises.push( - new Promise((resolve) => { + new Promise((resolve) => { server.close?.(() => { resolve(); }); @@ -182,7 +198,49 @@ function closeServers() { return Promise.all(promises); } +let shutdownPromise: Promise; +function shutdownServers() { + if (shutdownPromise) return shutdownPromise; + const drainDeadline = Date.now() + getShutdownDrainCeilingMs(); + const extendedForDrain = shutdownDrainsHaveWork(); + if (extendedForDrain) extendShutdownDeadline(drainDeadline); + // Wait for application scopes to finish closing before exiting — some dispose a native + // runtime asynchronously (e.g. @harperfast/vite's rolldown dev server), and exiting the + // worker while that runtime is still live crashes the process. The manageThreads backstop + // timers still bound this if a scope's disposal hangs. + shutdownPromise = runShutdownDrains(drainDeadline) + .then(() => { + if (extendedForDrain) restoreShutdownDeadline(); + }) + .then(() => closeServers()) + .then(() => whenScopesClosed()) + .then(async () => { + const { branchDatabasesHaveWork, closeBranchDatabases } = databasesRuntimeModule; + if (!branchDatabasesHaveWork()) return closeBranchDatabases(); + extendShutdownDeadline(Date.now() + getShutdownDrainCeilingMs()); + try { + await closeBranchDatabases(); + } finally { + restoreShutdownDeadline(); + } + }) + .then(() => { + if (!isMainThread) realExit(0); + }); + // Clean up per-thread UDS socket and metadata files + httpComponent.cleanupUdsFiles(); + if (!isBun && (debugThreads || process.env.DEV_MODE)) { + try { + inspector.close(); + } catch (error) { + harperLogger.info('Could not close debugger', error); + } + } + return shutdownPromise; +} + function startServers() { + if (isProcessShuttingDown()) return shutdownServers(); // A worker that has not yet posted child_started owns no ref'd handle: addPort() // (manageThreads) unrefs parentPort, component watchers are persistent:false, and the // reporting timers are unref'd. An await inside loadRootComponents whose completion @@ -209,86 +267,43 @@ function startServers() { } reportStartupPhase(startupPhase); let listening; - // A worker held for nothing still reports its load before it binds, where main answers it: one already loading when a - // release went live is not admitted on that load. const heldFor = workerData?.certify ?? (workerData?.reportsLoad ? [] : undefined); - const heldStart = heldFor ? require('./heldStart.ts') : undefined; - const loaded = Promise.resolve(heldStart?.beginHeldStart(heldFor)) - .then((loadedGenerations) => - require('../loadRootComponents.js') - .loadRootComponents(true) - .then(() => { - require('../../components/deployLifecycle.ts').deployLifecycle._componentsLoaded(); - return loadedGenerations; - }) - ) - .then(async (loadedGenerations) => { - parentPort - ?.on('message', (message) => { - if (message.type === terms.ITC_EVENT_TYPES.SHUTDOWN) { - harperLogger.trace('received shutdown request', threadId); - // shutdown (for these threads) means stop listening for incoming requests (finish what we are working) and - // close connections as possible, then let the event loop complete. - // First, gracefully drain any in-flight work registered by components — notably a - // replication blob *send* streaming to a peer, which is cheaper to finish than to interrupt - // (interrupting leaves the peer's copy diverged until it re-requests). The drain waits only - // on work still making progress, bounded by an absolute deadline. When there is real work to - // drain we push the termination backstops out to that deadline first so the drain isn't cut - // short, then restore the normal short backstop once draining is done — so any later hang - // (closeServers / scope disposal) is still force-killed on the normal timeout, and a worker - // with no such work is never affected. - const drainDeadline = Date.now() + getShutdownDrainCeilingMs(); - const extendedForDrain = shutdownDrainsHaveWork(); - if (extendedForDrain) extendShutdownDeadline(drainDeadline); - // Wait for application scopes to finish closing before exiting — some dispose a native - // runtime asynchronously (e.g. @harperfast/vite's rolldown dev server), and exiting the - // worker while that runtime is still live crashes the process. The manageThreads backstop - // timers still bound this if a scope's disposal hangs. - runShutdownDrains(drainDeadline) - .then(() => { - if (extendedForDrain) restoreShutdownDeadline(); - }) - .then(() => closeServers()) - .then(() => whenScopesClosed()) - .then(async () => { - const { branchDatabasesHaveWork, closeBranchDatabases } = require('../../resources/databases.ts'); - if (!branchDatabasesHaveWork()) return closeBranchDatabases(); - extendShutdownDeadline(Date.now() + getShutdownDrainCeilingMs()); - try { - await closeBranchDatabases(); - } finally { - restoreShutdownDeadline(); - } - }) - .then(() => { - realExit(0); - }); - // Clean up per-thread UDS socket and metadata files - httpComponent.cleanupUdsFiles(); - if (!isBun && (debugThreads || process.env.DEV_MODE)) { - try { - require('inspector').close(); - } catch (error) { - harperLogger.info('Could not close debugger', error); - } - } - } - }) - .ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles - if (heldStart) { - reportStartupPhase('awaiting admission'); - await heldStart.awaitAdmission(heldFor, loadedGenerations); - } - reportStartupPhase('binding listeners'); - listening = listenOnPorts(); - }); - componentsLoadedResolve(loaded); - const started = loaded + const heldStart = heldFor ? loadRuntimeModule('server/threads/heldStart') : undefined; + let admitted: Promise | undefined; + let loaded: Promise; + parentPort?.on('message', (message) => { + if (message.type !== terms.ITC_EVENT_TYPES.SHUTDOWN) return; + beginProcessShutdown(); + harperLogger.trace('received shutdown request', threadId); + void loaded + .then(shutdownServers, shutdownServers) + .catch((error) => harperLogger.error('Error shutting down HTTP worker', error)); + }); + loaded = Promise.resolve(heldStart?.beginHeldStart(heldFor)).then(async (loadedGenerations) => { + await loadRootComponentsRuntimeModule.loadRootComponents(true); + loadRuntimeModule('components/deployLifecycle').deployLifecycle._componentsLoaded(); + if (heldStart) { + reportStartupPhase('awaiting admission'); + await new Promise((resolve, reject) => { + admitted = heldStart.awaitAdmission(heldFor, loadedGenerations, resolve); + admitted.catch(reject); + }); + } + }); + const ready = loaded.then(async () => { + await admitted; + if (isProcessShuttingDown()) return shutdownServers(); + reportStartupPhase('binding listeners'); + listening = listenOnPorts(); + }); + componentsLoadedResolve(ready); + const started = ready .then(() => listening) .then(() => { // An isolated worker is reachable only through a bound mirror. if (thisThreadsIsolatedApplication() && !httpComponent.hasUdsMirror()) throw new Error(`Isolated application ${thisThreadsIsolatedApplication()} has no UDS mirror`); + if (isProcessShuttingDown()) return shutdownServers(); reportStartupPhase('ready'); if (getWorkerIndex() === 0) { try { @@ -300,6 +315,7 @@ function startServers() { parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED }); }) .catch((error) => { + if (isProcessShuttingDown()) return shutdownServers(); const failedPhase = startupPhase; reportStartupPhase(`failed during ${failedPhase}`); harperLogger.fatal(`HTTP worker startup failed during ${failedPhase}`, error); @@ -332,7 +348,7 @@ function startServers() { * A per-thread mirror binds at a temp name and is renamed over its published path: libuv unlinks a * pipe server's bound path on close whoever owns it by then (server/DESIGN.md). */ -function listenOnDomainSocket(port, server) { +function listenOnDomainSocket(port?: any, server?: any) { const bindPath = server.isPerThreadSocket && !isWindows ? mirrorBindPath(port) : port; const overlong = [port, bindPath].find((path) => isDomainSocketPathTooLong(path)); if (overlong) { @@ -343,8 +359,8 @@ function listenOnDomainSocket(port, server) { return Promise.resolve({ port, failed: true }); } if (bindPath === port && existsSync(port)) unlinkSync(port); - return new Promise((resolve, reject) => { - function onError(error) { + return new Promise((resolve, reject) => { + function onError(error?: any) { reject(error); } function onListening() { @@ -376,13 +392,14 @@ function listenOnDomainSocket(port, server) { } let mirrorBindSequence = 0; -function mirrorBindPath(socketPath) { +function mirrorBindPath(socketPath?: any) { return join(dirname(socketPath), `.${threadId}.${++mirrorBindSequence}`); } const POOL_BIND_RETRY_MS = 60_000; let listening; function listenOnPorts() { + if (isProcessShuttingDown()) return Promise.resolve([]); if (isBun) return listenOnPortsBun(); if (listening) return Promise.all(listening); // already set up listening = []; @@ -445,7 +462,7 @@ function listenOnPorts() { continue; let bindRetryDeadline; listening.push( - new Promise((resolve, reject) => { + new Promise((resolve, reject) => { server .listen(listen_on, () => { resolve({ port, name: server.name, protocol_name: server.protocol_name }); @@ -496,7 +513,7 @@ function listenOnPorts() { const cfg = uwsServeConfigs[key]; if (!shouldStartUwsListenerHere(cfg) || !shouldBindListenerHere(key)) continue; // dedicated worker: its own mirrors only if (cfg.socketPath && existsSync(cfg.socketPath)) unlinkSync(cfg.socketPath); - const { createUwsServer } = require('../serverHelpers/uwsServer.ts'); + const { createUwsServer } = uwsServerRuntimeModule; listening.push( createUwsServer(cfg).then(({ close }) => { // Register a minimal server-like entry so closeServers() can tear it down. uWS's @@ -527,7 +544,7 @@ function listenOnPorts() { * collisions have been ruled out (reusePort sharing, or the main thread's first-bind ordering), * so this is unambiguously external. */ -function logExternalBindConflict(port, err) { +function logExternalBindConflict(port?: any, err?: any) { // `port` is a string key from `for..in SERVERS`, but portServer may be keyed by the numeric // port setPortServerMap() was called with, so fall back to a numeric lookup. const registered = portServer.get(port) ?? portServer.get(Number(port)); @@ -563,7 +580,7 @@ async function listenOnPortsBun() { } else { portNumber = +port; } - const serveOptions = { + const serveOptions: any = { port: portNumber, // Respect the per-server reusePort decision made in http.ts (the operations API // opts out so it stays exclusive); fall back to the platform default otherwise. @@ -578,7 +595,7 @@ async function listenOnPortsBun() { // Wait for TLS certs to be loaded const defaultContext = await config.tlsSelector.ready; if (defaultContext) { - serveOptions.tls = { + const tlsOpts: any = { cert: defaultContext.options.cert, key: defaultContext.options.key, }; @@ -587,9 +604,10 @@ async function listenOnPortsBun() { if (ca) { if (Array.isArray(ca)) ca = ca.filter((entry) => typeof entry === 'string'); if (typeof ca === 'string' || (Array.isArray(ca) && ca.length > 0)) { - serveOptions.tls.ca = ca; + tlsOpts.ca = ca; } } + serveOptions.tls = tlsOpts; } // Set up listener for cert updates to reload TLS const pseudoServer = config.pseudoServer; @@ -597,7 +615,7 @@ async function listenOnPortsBun() { pseudoServer.secureContextsListeners.push(() => { const updatedCtx = config.tlsSelector.defaultContext; if (updatedCtx && SERVERS[port]?.reload) { - const tlsUpdate = { + const tlsUpdate: any = { cert: updatedCtx.options.cert, key: updatedCtx.options.key, }; @@ -669,7 +687,7 @@ async function listenOnPortsBun() { } } // Also start any non-HTTP servers (raw socket servers) that were registered in SERVERS - const listening = []; + const listening: any = []; for (let port in SERVERS) { const server = SERVERS[port]; // Skip Bun servers (they're already listening) and config objects @@ -695,7 +713,7 @@ async function listenOnPortsBun() { continue; } listening.push( - new Promise((resolve, reject) => { + new Promise((resolve, reject) => { server .listen({ port: portNum, host: rawHostname || (isMac ? '0.0.0.0' : '::') }, () => { resolve({ port }); @@ -718,8 +736,26 @@ async function listenOnPortsBun() { } return Promise.all(listening); } -if (!isMainThread && !workerData?.noServerStart) { - startServers(); +if ( + !isMainThread && + !workerData?.noServerStart && + process.argv[1] && + existsSync(process.argv[1]) && + realpathSync(process.argv[1]) === join(RUNTIME_SRC_ROOT, `server/threads/threadServer${RUNTIME_FILE_EXT}`) +) { + // The shared module graph also loads here in job and user workers, which must not bind listeners. + (async () => { + parentPort?.ref(); + env.initSync(); + const { runStartup } = await import('../../utility/lifecycle.ts'); + await runStartup(); + if (isProcessShuttingDown()) return shutdownServers(); + await startServers(); + })().catch((err) => { + if (isProcessShuttingDown()) return shutdownServers(); + harperLogger.fatal('Worker failed to start', err); + realExit(1); + }); } /** @@ -727,9 +763,8 @@ if (!isMainThread && !workerData?.noServerStart) { * @param listener * @param options */ -function onSocket(listener, options) { - let getComponentName = require('../../components/componentLoader.ts').getComponentName; - let socketServer; +function onSocket(listener?: any, options?: any) { + let socketServer: any; const { threadType } = options; for (const port of [options.securePort, options.port]) { if (!port) continue; @@ -762,7 +797,7 @@ function onSocket(listener, options) { // mutable binding made every secure-port metadata write read `secureContexts` off the plain // TCP server (undefined) and publish an empty `certificates:` list — the #1998 bug that let // an SNI-routing proxy (Symphony) fall back to the node certificate on 8883. - const secureSocketServer = createSecureSocketServer( + const secureSocketServer: any = createSecureSocketServer( { rejectUnauthorized: Boolean(options.mtls?.required), requestCert: Boolean(options.mtls), @@ -808,7 +843,7 @@ function onSocket(listener, options) { // socket.authorized/remoteAddress at connection time, so the data-interception // approach the HTTP UDS mirror uses (enableProxyProtocol) would apply the // forwarded mTLS identity too late. - const udsServer = createSocketServer({ ...socketOptionDefaults }, withProxyProtocol(listener)); + const udsServer: any = createSocketServer({ ...socketOptionDefaults }, withProxyProtocol(listener)); udsServer.isPerThreadSocket = true; if (threadType) claimListener(udsPath, threadType); diff --git a/sqlEngine/binder/defaultDatabase.ts b/sqlEngine/binder/defaultDatabase.ts index 3a98c39cc6..50133d5087 100644 --- a/sqlEngine/binder/defaultDatabase.ts +++ b/sqlEngine/binder/defaultDatabase.ts @@ -1,3 +1,4 @@ +import * as databasesRuntimeModule from '../../resources/databases.ts'; /** * Default-database resolution for schema-unqualified SQL table references. * @@ -25,7 +26,7 @@ export function _setDatabasesLoader(loader: (() => DatabaseRegistry) | null): vo export function loadDatabases(): DatabaseRegistry { if (_databasesLoader) return _databasesLoader(); - const mod = require('../../resources/databases.js'); + const mod = databasesRuntimeModule; return mod.getDatabases(); } diff --git a/sqlEngine/diff/differential.ts b/sqlEngine/diff/differential.ts index 7a872646e6..99c92c8f9e 100644 --- a/sqlEngine/diff/differential.ts +++ b/sqlEngine/diff/differential.ts @@ -1,3 +1,4 @@ +import * as indexCjsModule from '../../sqlTranslator/index.ts'; /** * Differential test harness. * @@ -31,7 +32,7 @@ interface SqlTranslator { } function loadLegacy(): SqlTranslator { - return require('../../sqlTranslator/index.js'); + return indexCjsModule; } function runLegacy(sql: string, opts: DifferentialOptions): Promise { diff --git a/sqlEngine/errors.ts b/sqlEngine/errors.ts index ff2a4353f8..bc7d9acd56 100644 --- a/sqlEngine/errors.ts +++ b/sqlEngine/errors.ts @@ -12,7 +12,7 @@ * The router does not fall back on it — it surfaces to the caller. */ -import { ClientError } from '../utility/errors/hdbError.js'; +import { ClientError } from '../utility/errors/hdbError.ts'; export class EngineUnsupportedError extends ClientError { reason: string; diff --git a/sqlEngine/executor/runMutation.ts b/sqlEngine/executor/runMutation.ts index 0d3e8bacb5..d328c2e097 100644 --- a/sqlEngine/executor/runMutation.ts +++ b/sqlEngine/executor/runMutation.ts @@ -1,3 +1,4 @@ +import * as transactionRuntimeModule from '../../resources/transaction.ts'; /** * Phase 4 mutation executors: INSERT / UPDATE / DELETE. * @@ -175,7 +176,7 @@ export function _setTransactionRunner(runner: TransactionRunner | null): void { function getTransactionRunner(): TransactionRunner { if (_transactionRunner) return _transactionRunner; // eslint-disable-next-line @typescript-eslint/no-var-requires - const mod = require('../../resources/transaction.js'); + const mod = transactionRuntimeModule; return mod.transaction as TransactionRunner; } diff --git a/sqlEngine/parser/parse.ts b/sqlEngine/parser/parse.ts index 3339262134..aee2326e97 100644 --- a/sqlEngine/parser/parse.ts +++ b/sqlEngine/parser/parse.ts @@ -1,3 +1,4 @@ +import alasql from 'alasql'; /** * Thin wrapper around alasql.parse. * @@ -6,8 +7,6 @@ * pipeline only sees the IR. */ -const alasql = require('alasql'); - export interface AlaSqlParseResult { ast: unknown; variant: string; diff --git a/sqlEngine/router.ts b/sqlEngine/router.ts index eb26b5ad91..01277c4ec2 100644 --- a/sqlEngine/router.ts +++ b/sqlEngine/router.ts @@ -1,3 +1,4 @@ +import * as harper_loggerCjsModule from '../utility/logging/harper_logger.ts'; /** * Router that dispatches a SQL request to either the new engine or the legacy * AlaSQL-based path based on the sql.engine config flag. @@ -29,7 +30,7 @@ export interface RouteOptions { function getLogger(): { info: (msg: string) => void; warn: (msg: string) => void } { try { - const harperLogger = require('../utility/logging/harper_logger.js'); + const harperLogger = harper_loggerCjsModule; return harperLogger.loggerWithTag ? harperLogger.loggerWithTag('sql-engine') : harperLogger; } catch { return { info: () => {}, warn: () => {} }; diff --git a/sqlTranslator/SelectValidator.ts b/sqlTranslator/SelectValidator.ts index ba9cb439d2..03cffe4547 100644 --- a/sqlTranslator/SelectValidator.ts +++ b/sqlTranslator/SelectValidator.ts @@ -1,7 +1,7 @@ 'use strict'; import RecursiveIterator from 'recursive-iterator'; -import * as alasql from 'alasql'; +import alasql from 'alasql'; import clone from 'clone'; import * as commonUtils from '../utility/common_utils.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; @@ -100,7 +100,7 @@ class SelectValidator { } //let theTable = clone(table); - let schemaTable = databases[table.databaseid][table.tableid]; + let schemaTable: any = databases[table.databaseid][table.tableid]; /*TODO rather than putting every attribute in an array we will create a Map there will be a map element for every table and every table alias (this will create duplicate map elements) this will have downstream effects in comparison functions like findColumn*/ schemaTable.attributes.forEach((attribute) => { diff --git a/sqlTranslator/alasqlFunctionImporter.ts b/sqlTranslator/alasqlFunctionImporter.ts index 14c9d7d4ad..9660d22a1c 100644 --- a/sqlTranslator/alasqlFunctionImporter.ts +++ b/sqlTranslator/alasqlFunctionImporter.ts @@ -4,9 +4,9 @@ * PUrpose of this is to set up a central module to define and import custom functions into alasql */ -import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.js'; +import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.ts'; import * as dateFunctions from '../utility/functions/date/dateFunctions.js'; -import * as geo from '../utility/functions/geo.js'; +import * as geo from '../utility/functions/geo.ts'; //import the custom function, need to define an upper and lower case version of the function so it is parsed properly in alasql export default function (alasql: any) { diff --git a/sqlTranslator/deleteTranslator.ts b/sqlTranslator/deleteTranslator.ts index 725b9b3e0a..c7b3b22fd4 100644 --- a/sqlTranslator/deleteTranslator.ts +++ b/sqlTranslator/deleteTranslator.ts @@ -1,8 +1,8 @@ -import * as alasql from 'alasql'; +import alasql from 'alasql'; import * as search from '../dataLayer/search.ts'; import log from '../utility/logging/harper_logger.ts'; import harperBridge from '../dataLayer/harperBridge/harperBridge.ts'; -import * as util from 'util'; +import * as util from 'node:util'; import * as hdbUtils from '../utility/common_utils.ts'; import * as terms from '../utility/hdbTerms.ts'; import * as globalSchema from '../utility/globalSchema.ts'; diff --git a/sqlTranslator/index.ts b/sqlTranslator/index.ts index cdd44b2b31..049a007689 100644 --- a/sqlTranslator/index.ts +++ b/sqlTranslator/index.ts @@ -1,14 +1,14 @@ 'use strict'; import * as insert from '../dataLayer/insert.ts'; -import * as util from 'util'; +import * as util from 'node:util'; const cbInsertInsert = util.callbackify(insert.insert); import { search } from '../dataLayer/search.ts'; import { update } from '../dataLayer/update.ts'; const cbUpdateUpdate = util.callbackify(update); import { convertDelete as deleteTranslator } from './deleteTranslator.ts'; const cbDeleteTranslator = util.callbackify(deleteTranslator); -import * as alasql from 'alasql'; +import alasql from 'alasql'; import * as opAuth from '../utility/operation_authorization.ts'; import logger from '../utility/logging/harper_logger.ts'; import alasqlFunctionImporter from './alasqlFunctionImporter.ts'; diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 0071918b9e..d8edbacc92 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -4,9 +4,10 @@ * AST SQL values such as attributes, tables, etc. **/ -import * as alasql from 'alasql'; +import alasql from 'alasql'; import RecursiveIterator from 'recursive-iterator'; -const harperLogger = require('../utility/logging/harper_logger').default || require('../utility/logging/harper_logger'); +import _harperLogger from '../utility/logging/harper_logger.ts'; +const harperLogger = _harperLogger; import * as hdbUtils from '../utility/common_utils.ts'; import * as terms from '../utility/hdbTerms.ts'; import { findDatabasesWithTable, loadDatabaseRegistry } from '../sqlEngine/binder/defaultDatabase.ts'; diff --git a/unitTests/apiTests/setupTestApp.mjs b/unitTests/apiTests/setupTestApp.mjs index f110b65ace..f6b620314e 100644 --- a/unitTests/apiTests/setupTestApp.mjs +++ b/unitTests/apiTests/setupTestApp.mjs @@ -1,7 +1,7 @@ import { setupTestDBPath, ensureSystemTables } from '../testUtils.js'; -import { fileURLToPath } from 'url'; +import { fileURLToPath } from 'node:url'; import hdbTerms from '#src/utility/hdbTerms'; -import { join } from 'path'; +import { join } from 'node:path'; import axios from 'axios'; import { encode } from 'cbor-x'; import analytics from '#src/resources/analytics/write'; @@ -48,6 +48,12 @@ function makeString() { let createdRecords; let serverStarted; export async function setupTestApp() { + // Run production startup hooks before applying the API-test auth overrides. + if (typeof process !== 'undefined' && !serverStarted) { + const { runStartup } = await import('#src/utility/lifecycle'); + await runStartup(); + } + analytics.setAnalyticsEnabled(false); bypassAuth(); bypassAuthMQTT(); @@ -128,6 +134,8 @@ export async function setupTestApp() { } else { const { startHTTPThreads } = await import('#src/server/threads/socketRouter'); serverStarted = await startHTTPThreads(config.threads || 0); + const { whenComponentsLoaded } = await import('#src/server/threads/threadServer'); + await whenComponentsLoaded; } try { seed = 0; // reset the seed to make sure we are deterministic here diff --git a/unitTests/dataLayer/bulkLoad.test.js b/unitTests/dataLayer/bulkLoad.test.js index 96df92ca41..2c1770059e 100644 --- a/unitTests/dataLayer/bulkLoad.test.js +++ b/unitTests/dataLayer/bulkLoad.test.js @@ -227,7 +227,7 @@ describe.skip('Test bulkLoad.js', () => { it('Test csvDataLoad with attr-level perms issues - returns errors', async function () { PermissionResponseObject_rw.prototype.getPermsResponse = () => get_perms_resp_stub(); - const getPermsError_rw = bulkLoad_rewire.__set__('PermissionResponseObject_js_1', { + const getPermsError_rw = bulkLoad_rewire.__set__('PermissionResponseObject_ts_1', { default: PermissionResponseObject_rw, }); let result; diff --git a/unitTests/dataLayer/delete.test.js b/unitTests/dataLayer/delete.test.js index 92eff69f3e..b580cbe644 100644 --- a/unitTests/dataLayer/delete.test.js +++ b/unitTests/dataLayer/delete.test.js @@ -186,7 +186,7 @@ describe('Tests for delete.js', () => { // and silently delete every record in the table (reported as "1 of 1"). The validator must // reject it up front, before deleteRecords is ever reached. let bridge_spy = sandbox.stub().resolves(); - let revert = _delete.__set__('harperBridge', { deleteRecords: bridge_spy }); + let revert = _delete.__set__('harperBridge_ts_1', { default: { deleteRecords: bridge_spy } }); let delete_obj = testUtils.deepClone(DELETE_OBJ_TEST); delete_obj.hash_values = [8, null, 9]; let test_err_result = await testUtils.testError( @@ -209,7 +209,7 @@ describe('Tests for delete.js', () => { expected_response.deleted_hashes = []; expected_response.skipped_hashes = [8, 9]; let delete_records_stub = sandbox.stub().resolves(expected_response); - let revert = _delete.__set__('harperBridge', { deleteRecords: delete_records_stub }); + let revert = _delete.__set__('harperBridge_ts_1', { default: { deleteRecords: delete_records_stub } }); let result = await _delete.deleteRecord(DELETE_RECORDS_TEST); expect(delete_records_stub).to.have.been.calledWith(DELETE_RECORDS_TEST); diff --git a/unitTests/dataLayer/insert.test.js b/unitTests/dataLayer/insert.test.js index 77813c2805..a3aa401671 100644 --- a/unitTests/dataLayer/insert.test.js +++ b/unitTests/dataLayer/insert.test.js @@ -72,7 +72,7 @@ describe('Test insert module', () => { }); beforeEach(() => { - insert_rw.__set__('harperBridge', { upsertRecords: bridge_upsert_stub }); + insert_rw.__set__('harperBridge_ts_1', { default: { upsertRecords: bridge_upsert_stub } }); }); afterEach(async () => { @@ -177,7 +177,7 @@ describe('Test insert module', () => { let bridge_insert_stub = sandbox.stub().resolves(bridge_insert_resp_test); before(() => { - insert_rw.__set__('harperBridge', { createRecords: bridge_insert_stub }); + insert_rw.__set__('harperBridge_ts_1', { default: { createRecords: bridge_insert_stub } }); }); after(() => { @@ -226,7 +226,7 @@ describe('Test insert module', () => { let bridge_update_stub = sandbox.stub(); before(() => { - insert_rw.__set__('harperBridge', { updateRecords: bridge_update_stub }); + insert_rw.__set__('harperBridge_ts_1', { default: { updateRecords: bridge_update_stub } }); }); after(() => { diff --git a/unitTests/dataLayer/readAuditLog.test.js b/unitTests/dataLayer/readAuditLog.test.js index 8e52dd13b4..1bebd64e74 100644 --- a/unitTests/dataLayer/readAuditLog.test.js +++ b/unitTests/dataLayer/readAuditLog.test.js @@ -69,8 +69,10 @@ describe.skip('test readAuditLog module', () => { it('test happy path', async () => { let stub = sandbox.stub().resolves([]); - let rw_stub = rw_read_audit_log.__set__('harperBridge', { - readAuditLog: stub, + let rw_stub = rw_read_audit_log.__set__('harperBridge_ts_1', { + default: { + readAuditLog: stub, + }, }); let obj = new ReadAuditLogObject('dev', 'test', 'timestamp'); diff --git a/unitTests/resources/caching.test.js b/unitTests/resources/caching.test.js index 2b411b37e3..55b0e25ea0 100644 --- a/unitTests/resources/caching.test.js +++ b/unitTests/resources/caching.test.js @@ -6,7 +6,7 @@ const { table } = require('#src/resources/databases'); const { Resource } = require('#src/resources/Resource'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { RequestTarget } = require('#src/resources/RequestTarget'); -const { VERSION_REUSED } = require('#src/resources/RecordEncoder'); +const { VERSION_REUSED, entryMap } = require('#src/resources/RecordEncoder'); const { INVALIDATED } = require('#src/resources/Table'); const { exportIdMapping } = require('#src/resources/nodeIdMapping'); const { transaction } = require('#src/resources/transaction'); @@ -1340,25 +1340,54 @@ describe('Caching', () => { it('Bigger stampede is handled', async function () { this.timeout(5000); + let interval; try { timer = 2; CachingTable.setTTLExpiration(100); // don't evict during this test since it will clear the history let i = 0; sourceRequests = 0; let results = []; - let interval = setInterval(async () => { - i++; - if (i % 16 == 1) CachingTable.invalidate(23); - else { - // clearing the cache kind of emulates what another thread would see - if (i % 4 == 0) CachingTable.primaryStore.cache.clear(); - let raw_result = CachingTable.get(23); - let result = await raw_result; - results.push(result); + let cacheMisses = 0; + const failures = []; + const reads = []; + interval = setInterval(() => { + try { + i++; + if (i % 16 == 1) + reads.push(Promise.resolve(CachingTable.invalidate(23)).catch((error) => failures.push(error))); + else { + let evictedEntry; + if (i % 4 == 0) { + const store = CachingTable.primaryStore; + const cached = store.getEntry(23); + if (cached?.value && !store.hasLock(23)) { + // Without its memoized Entry, the private value cache cannot vouch for this record. + assert.equal(entryMap.delete(cached.value), true); + evictedEntry = cached; + } + } + let raw_result = CachingTable.get(23); + reads.push( + Promise.resolve(raw_result) + .then((result) => { + if (evictedEntry) { + assert.notStrictEqual(result, evictedEntry.value); + if (entryMap.get(result)?.version === evictedEntry.version) cacheMisses++; + } + results.push(result); + }) + .catch((error) => failures.push(error)) + ); + } + } catch (error) { + failures.push(error); } }, 1); await new Promise((resolve) => setTimeout(resolve, 3000)); clearInterval(interval); + await Promise.all(reads); + assert.deepEqual(failures, []); + assert(cacheMisses > 40, `Only ${cacheMisses} actual cache misses`); for (let result of results) { assert.equal(result.name, 'name 23'); } @@ -1374,6 +1403,7 @@ describe('Caching', () => { assert(entry.localTime > 1); } } finally { + clearInterval(interval); timer = 0; } }); diff --git a/unitTests/resources/create.test.js b/unitTests/resources/create.test.js index 2426ff0d6f..25ca162f4a 100644 --- a/unitTests/resources/create.test.js +++ b/unitTests/resources/create.test.js @@ -1,6 +1,7 @@ require('../testUtils'); const assert = require('assert'); const { Worker } = require('worker_threads'); +const { once } = require('node:events'); const { setupTestDBPath } = require('../testUtils'); const { table } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); @@ -65,7 +66,21 @@ describe('Create records', () => { let id_after = CreateTest.getNewId(); assert(Math.abs(id_before - id_after) > 1000000); }); - after(() => { - test_thread.terminate(); + after(async () => { + if (!test_thread || test_thread.threadId === -1) return; + // Graceful shutdown — let the worker close its rocksdb handles on its own + // event loop rather than relying on Worker.terminate(), which under + // rocksdb-js triggers a native finalizer crash during the next test's + // process-wide handle setup. + const exited = once(test_thread, 'exit', { signal: AbortSignal.timeout(5000) }); + const fallback = setTimeout(() => { + test_thread.terminate().catch(() => {}); + }, 1000); + try { + test_thread.postMessage({ type: 'shutdown' }); + await exited; + } finally { + clearTimeout(fallback); + } }); }); diff --git a/unitTests/security/authCredentialDeferral.test.js b/unitTests/security/authCredentialDeferral.test.js index 57306fa356..4244cfdfc3 100644 --- a/unitTests/security/authCredentialDeferral.test.js +++ b/unitTests/security/authCredentialDeferral.test.js @@ -16,6 +16,7 @@ const resourcesModule = require('#src/resources/Resources'); const tokenAuthentication = require('#src/security/tokenAuthentication'); const { databases } = require('#src/resources/databases'); const { authentication } = require('#src/security/auth'); +const { runStartup } = require('#src/utility/lifecycle'); const HARPER_OWNED = '/Ledger/1'; const HARPER_OWNED_PUBLIC = '/PublicNotice/1'; @@ -488,7 +489,7 @@ describe('#2703 principal resolution failures become decisions, not thrown error const SESSION_COOKIE = `example_com-hdb-session=${SESSION_ID}`; const CERT_CN = 'svc.example.com'; - const sessionTable = databases.system.hdb_session; + let sessionTable; let originalGetUser; let originalValidateOperationToken; @@ -554,6 +555,8 @@ describe('#2703 principal resolution failures become decisions, not thrown error } before(async () => { + await runStartup(); + sessionTable = databases.system.hdb_session; originalGetUser = serverModule.server.getUser; originalValidateOperationToken = tokenAuthentication.validateOperationToken; diff --git a/unitTests/security/jsLoader/jsLoader.test.js b/unitTests/security/jsLoader/jsLoader.test.js index fc0b1afa76..56b900c2c7 100644 --- a/unitTests/security/jsLoader/jsLoader.test.js +++ b/unitTests/security/jsLoader/jsLoader.test.js @@ -228,7 +228,9 @@ describe('SES loading', () => { const sesIndex = readFileSync(join(dirname(require.resolve('ses/package.json')), 'index.js'), 'utf8'); const loaderSource = readFileSync(join(__dirname, '..', '..', '..', 'security', 'jsLoader.ts'), 'utf8'); const sesShims = [...sesIndex.matchAll(/^import '\.\/src\/([\w-]+-shim\.js)';$/gm)].map((match) => match[1]); - const loadedShims = [...loaderSource.matchAll(/require\('ses\/([\w-]+-shim\.js)'\)/g)].map((match) => match[1]); + const loadedShims = [...loaderSource.matchAll(/(?:require|nativeRequire)\('ses\/([\w-]+-shim\.js)'\)/g)].map( + (match) => match[1] + ); expect(sesShims).to.have.length.above(0); expect(loadedShims).to.deep.equal(sesShims); }); diff --git a/unitTests/server/threads/fixtures/admission-scope.cjs b/unitTests/server/threads/fixtures/admission-scope.cjs new file mode 100644 index 0000000000..4dabcfc1f1 --- /dev/null +++ b/unitTests/server/threads/fixtures/admission-scope.cjs @@ -0,0 +1,39 @@ +const { parentPort, workerData } = require('node:worker_threads'); +const { setTimeout: delay } = require('node:timers/promises'); + +exports.handleApplication = async (scope) => { + try { + scope.server.socket((socket) => socket.end('admitted'), { + port: workerData.probePort, + name: 'admission-scope', + ...(workerData.probePool ? { threadType: 'replication' } : {}), + }); + if (workerData.probePool) { + const { SERVERS } = require('#src/server/serverRegistry'); + SERVERS[workerData.probePort].noReusePort = true; + SERVERS[workerData.probePort].on('error', (error) => { + if (error.code === 'EADDRINUSE') parentPort.postMessage({ type: 'admission-bind-error' }); + }); + } + scope.on('close', async () => { + await delay(workerData.probePool ? 1000 : 30); + parentPort.postMessage({ type: 'admission-scope-disposed' }); + }); + const { whenComponentsLoaded } = await import('#src/server/threads/threadServer'); + whenComponentsLoaded.then(() => parentPort.postMessage({ type: 'admission-components-loaded' })); + parentPort.postMessage({ type: 'admission-scope-loaded' }); + if (workerData.holdProbeLoad) { + await new Promise((resolve) => { + const release = (message) => { + if (message.type !== 'release-admission-probe') return; + parentPort.off('message', release); + resolve(); + }; + parentPort.on('message', release); + }); + } + } catch (error) { + parentPort.postMessage({ type: 'admission-fixture-error', message: error.message }); + throw error; + } +}; diff --git a/unitTests/server/threads/fixtures/component-shutdown.cjs b/unitTests/server/threads/fixtures/component-shutdown.cjs new file mode 100644 index 0000000000..95b565f15e --- /dev/null +++ b/unitTests/server/threads/fixtures/component-shutdown.cjs @@ -0,0 +1,23 @@ +const { parentPort } = require('node:worker_threads'); + +exports.handleApplication = async (scope) => { + scope.on( + 'close', + () => + new Promise((resolve) => { + setTimeout(() => { + parentPort.postMessage({ type: 'component-disposed' }); + resolve(); + }, 30); + }) + ); + parentPort.postMessage({ type: 'component-held' }); + await new Promise((resolve) => { + const release = (message) => { + if (message.type !== 'release-component') return; + parentPort.off('message', release); + resolve(); + }; + parentPort.on('message', release); + }); +}; diff --git a/unitTests/server/threads/fixtures/embedded-layout.cjs b/unitTests/server/threads/fixtures/embedded-layout.cjs new file mode 100644 index 0000000000..cb1b0ea809 --- /dev/null +++ b/unitTests/server/threads/fixtures/embedded-layout.cjs @@ -0,0 +1,90 @@ +const assert = require('node:assert'); +const { cpSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync } = require('node:fs'); +const { join } = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { Worker } = require('node:worker_threads'); +const { once } = require('node:events'); +const { pathToFileURL } = require('node:url'); + +exports.probeEmbeddedRuntime = async (sourceRoot) => { + const product = join(process.env.ROOTPATH, '.embedded-product'); + const runtime = join(product, 'dist/core'); + const metadata = JSON.parse(readFileSync(join(sourceRoot, 'package.json'), 'utf8')); + metadata.name = 'embedded-layout-fixture'; + mkdirSync(product, { recursive: true }); + writeFileSync(join(product, 'package.json'), JSON.stringify(metadata)); + cpSync(join(sourceRoot, 'dist'), runtime, { recursive: true }); + const outerCli = join(product, 'dist/bin/harper.js'); + mkdirSync(join(product, 'dist/bin'), { recursive: true }); + writeFileSync( + outerCli, + "console.log('outer-cli'); const { harper } = require('../core/bin/harper.js'); harper().then(result => { if (result) console.log(result); }).catch(error => { console.error(error); process.exitCode = 1; });" + ); + const externalWorker = join(product, 'dist/external-worker.js'); + writeFileSync(externalWorker, "require('node:worker_threads').parentPort.postMessage('outside core');"); + const cli = (entry) => + execFileSync(process.execPath, [entry, 'version'], { encoding: 'utf8', timeout: 30000 }).trim().split(/\r?\n/); + let worker, external, databases; + try { + assert.deepStrictEqual(cli(join(runtime, 'bin/harper.js')), [metadata.version]); + assert.deepStrictEqual(cli(join(runtime, 'bin/harper')), [metadata.version]); + assert.deepStrictEqual(cli(outerCli), ['outer-cli', metadata.version]); + const { PACKAGE_ROOT, RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT, packageJson } = require( + join(runtime, 'utility/packageUtils.js') + ); + assert.strictEqual(PACKAGE_ROOT, product); + assert.strictEqual(RUNTIME_SRC_ROOT, runtime); + assert.strictEqual(packageJson.name, metadata.name); + const { systemSchema } = require(join(runtime, 'utility/systemSchema.js')); + assert.deepStrictEqual(systemSchema, JSON.parse(readFileSync(join(sourceRoot, 'json/systemSchema.json'), 'utf8'))); + assert.strictEqual(require(join(runtime, 'utility/globalSchema.js')).getSystemSchema(), systemSchema); + const config = require(join(runtime, 'utility/processManagement/servicesConfig.js')); + assert.strictEqual(config.generateMainServerConfig().script, outerCli); + assert.strictEqual(config.generateMainServerConfig().cwd, product); + assert.strictEqual(config.generateRestart().script, join(runtime, 'utility/scripts/restartHdb.js')); + assert.ok(existsSync(config.generateRestart().script)); + const { GIT_CREDENTIAL_HELPER_PATH } = require(join(runtime, 'components/Application.js')); + assert.strictEqual(GIT_CREDENTIAL_HELPER_PATH, join(runtime, 'components/gitCredentialHelper.js')); + assert.ok(existsSync(GIT_CREDENTIAL_HELPER_PATH)); + const { TRUSTED_RESOURCE_PLUGINS, loadComponent } = require(join(runtime, 'components/componentLoader.js')); + for (const name of ['fastifyRoutes', 'modelsGateway']) { + const plugin = TRUSTED_RESOURCE_PLUGINS[name]; + assert.ok(plugin.startsWith(pathToFileURL(runtime).href + '/')); + await import(plugin); + } + const appDirectory = join(product, 'default-app'); + mkdirSync(appDirectory); + const { Resources } = require(join(runtime, 'resources/Resources.js')); + await loadComponent(appDirectory, new Resources(), 'embedded-probe', { autoReload: false }); + const { statusForComponent } = require(join(runtime, 'components/status/index.js')); + assert.strictEqual(statusForComponent('default-app.fastifyRoutes').get()?.status, 'healthy'); + require(join(runtime, 'dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js')); + require(join(runtime, 'dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js')); + databases = require(join(runtime, 'resources/databases.js')); + const manager = require(join(runtime, 'server/threads/manageThreads.js')); + external = manager.startWorker('dist/external-worker', { name: 'embedded-relative-probe', autoRestart: false }); + const [externalMessage] = await once(external, 'message', { signal: AbortSignal.timeout(10000) }); + assert.strictEqual(externalMessage, 'outside core'); + worker = new Worker(join(RUNTIME_SRC_ROOT, `server/threads/threadServer${RUNTIME_FILE_EXT}`), { + workerData: { addPorts: [], addThreadIds: [], reportsLoad: true }, + }); + const messages = []; + worker.on('message', (message) => { + messages.push(message.type); + if (message.type === 'child_component_verdict') worker.postMessage({ type: 'child_admitted' }); + if (message.type === 'child_started') worker.postMessage({ type: 'shutdown', restartNumber: 2 }); + }); + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(15000) }); + assert.strictEqual(code, 0); + assert.ok(messages.includes('child_started')); + require(join(runtime, 'launchServiceScripts/launchHarperDB.js')); + const lifecycle = require(join(runtime, 'utility/lifecycle.js')); + assert.strictEqual(lifecycle.hasStarted(), true); + await lifecycle.runStartup(); + } finally { + if (worker && worker.threadId !== -1) await worker.terminate(); + if (external && external.threadId !== -1) await external.terminate(); + await databases?.closeLoadedDatabases(); + rmSync(product, { recursive: true, force: true }); + } +}; diff --git a/unitTests/server/threads/fixtures/runtime-condition.cjs b/unitTests/server/threads/fixtures/runtime-condition.cjs new file mode 100644 index 0000000000..dc3c715fd3 --- /dev/null +++ b/unitTests/server/threads/fixtures/runtime-condition.cjs @@ -0,0 +1,7 @@ +const { parentPort } = require('node:worker_threads'); + +parentPort.postMessage({ + type: 'runtime-condition', + modulePath: require.resolve('#src/server/Server'), + execArgv: process.execArgv, +}); diff --git a/unitTests/server/threads/fixtures/startup-ref.cjs b/unitTests/server/threads/fixtures/startup-ref.cjs new file mode 100644 index 0000000000..230fb54e0a --- /dev/null +++ b/unitTests/server/threads/fixtures/startup-ref.cjs @@ -0,0 +1,11 @@ +const { parentPort } = require('node:worker_threads'); +const { onStartup } = require('#src/utility/lifecycle'); +const { realExit } = require('#src/server/threads/workerProcessGuard'); + +onStartup(async () => { + parentPort.postMessage({ type: 'startup-ref', held: parentPort.hasRef() }); + // Model startup work whose completion does not itself keep the event loop alive. + await new Promise((resolve) => setTimeout(resolve, 100).unref()); + parentPort.postMessage({ type: 'startup-hook-completed' }); + realExit(0); +}); diff --git a/unitTests/server/threads/fixtures/startup-shutdown.cjs b/unitTests/server/threads/fixtures/startup-shutdown.cjs new file mode 100644 index 0000000000..3a5a1cce71 --- /dev/null +++ b/unitTests/server/threads/fixtures/startup-shutdown.cjs @@ -0,0 +1,27 @@ +const { parentPort } = require('node:worker_threads'); +const { onStartup, runStartup } = require('#src/utility/lifecycle'); + +onStartup(async () => { + parentPort.postMessage({ type: 'startup-held' }); + await new Promise((resolve) => { + const release = (message) => { + if (message.type !== 'release-startup') return; + parentPort.off('message', release); + resolve(); + }; + parentPort.on('message', release); + }); + const { trackScopeClose } = require('#src/components/scopeShutdown'); + trackScopeClose( + new Promise((resolve) => + setTimeout(() => { + parentPort.postMessage({ type: 'scope-disposed' }); + resolve(); + }, 30) + ) + ); + setImmediate(async () => { + await runStartup(); + parentPort.postMessage({ type: 'startup-drained' }); + }); +}); diff --git a/unitTests/server/threads/manageThreads-shutdown-fixture.cjs b/unitTests/server/threads/manageThreads-shutdown-fixture.cjs new file mode 100644 index 0000000000..6a5d5aba03 --- /dev/null +++ b/unitTests/server/threads/manageThreads-shutdown-fixture.cjs @@ -0,0 +1,21 @@ +'use strict'; + +// Fixture executed inside a Worker thread by manageThreads-shutdown.test.js. +// Loads the real SHUTDOWN handler from manageThreads.ts and reports back the +// exported `restartNumber` binding, to guard against the handler updating a +// globalThis shadow instead of the module binding that databases.ts and +// threadServer.ts read via live ESM imports. + +const { parentPort } = require('node:worker_threads'); +const manageThreads = require('#src/server/threads/manageThreads'); + +// SHUTDOWN unrefs parentPort; keep this fixture alive to answer the subsequent query. +setInterval(() => {}, 1000); + +parentPort.on('message', (message) => { + if (message?.type === 'query-restart-number') { + parentPort.postMessage({ type: 'restart-number', value: manageThreads.restartNumber }); + } +}); + +parentPort.postMessage({ type: 'ready' }); diff --git a/unitTests/server/threads/manageThreads-shutdown.test.js b/unitTests/server/threads/manageThreads-shutdown.test.js new file mode 100644 index 0000000000..974ed43cb0 --- /dev/null +++ b/unitTests/server/threads/manageThreads-shutdown.test.js @@ -0,0 +1,56 @@ +'use strict'; + +const { Worker } = require('node:worker_threads'); +const path = require('node:path'); +const assert = require('node:assert'); +const { ITC_EVENT_TYPES } = require('#src/utility/hdbTerms'); + +const FIXTURE = path.join(__dirname, 'manageThreads-shutdown-fixture.cjs'); + +function spawnFixture() { + const worker = new Worker(FIXTURE, { workerData: { addPorts: [], addThreadIds: [], restartNumber: 1 } }); + const ready = new Promise((resolve, reject) => { + worker.once('error', reject); + worker.once('exit', (code) => reject(new Error(`Worker exited before reporting ready (${code})`))); + const onMessage = (msg) => { + if (msg.type !== 'ready') return; + worker.off('message', onMessage); + resolve(); + }; + worker.on('message', onMessage); + }); + return { worker, ready }; +} + +async function sendAndAwait(worker, message) { + return new Promise((resolve, reject) => { + worker.once('error', reject); + worker.once('exit', (code) => reject(new Error(`Worker exited before answering (${code})`))); + const onMessage = (response) => { + if (response.type !== 'restart-number') return; + worker.off('message', onMessage); + resolve(response); + }; + worker.on('message', onMessage); + worker.postMessage(message); + }); +} + +describe('manageThreads SHUTDOWN handler', () => { + it('advances the exported restartNumber binding (not a globalThis shadow) on SHUTDOWN', async () => { + const { worker, ready } = spawnFixture(); + await ready; + try { + worker.postMessage({ type: ITC_EVENT_TYPES.SHUTDOWN, restartNumber: 7 }); + const response = await sendAndAwait(worker, { type: 'query-restart-number' }); + assert.strictEqual( + response.value, + 7, + 'restartNumber should be updated on the real module binding so restart-interrupt checks in ' + + 'databases.ts / threadServer.ts observe it' + ); + } finally { + await worker.terminate(); + } + }); +}); diff --git a/unitTests/server/threads/typestrip-runtime.test.mjs b/unitTests/server/threads/typestrip-runtime.test.mjs new file mode 100644 index 0000000000..1174f8c0ab --- /dev/null +++ b/unitTests/server/threads/typestrip-runtime.test.mjs @@ -0,0 +1,1185 @@ +import assert from 'node:assert'; +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import { Worker } from 'node:worker_threads'; +import { once, on } from 'node:events'; +import { resolve } from 'node:path'; +import { existsSync, mkdirSync, mkdtempSync, symlinkSync, rmSync } from 'node:fs'; + +const require = createRequire(import.meta.url); +const root = resolve(import.meta.dirname, '../../..'); + +function runWithIsolatedRoot(mode, code) { + let output = ''; + try { + output = execFileSync( + process.execPath, + [...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), '--input-type=module'], + { + env: process.env, + encoding: 'utf8', + timeout: 30000, + input: `const { createRequire } = await import('node:module'); + const require = createRequire(${JSON.stringify(pathToFileURL(resolve(root, 'package.json')).href)}); + const { materializePerPidRoot } = require(${JSON.stringify(resolve(root, 'unitTests/perPidRoot.js'))}); + process.env.ROOTPATH = materializePerPidRoot(); + console.log('fixture-root ' + process.pid); + ${code} + process.exit(0);`, + } + ); + return output.replace(/^fixture-root \d+\r?\n/, '').trim(); + } catch (error) { + output = error.stdout?.toString() ?? ''; + throw error; + } finally { + const pid = /^fixture-root (\d+)\r?\n/.exec(output)?.[1]; + if (pid) rmSync(resolve(root, 'unitTests/envDir', pid), { recursive: true, force: true }); + } +} + +describe('TypeStrip runtime boundaries', () => { + it('selects direct JSON codecs before serving REST and subscription messages', () => { + for (const mode of ['compiled', 'typestrip']) { + for (const bigint of [false, true]) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const runtimeUrl = (path) => pathToFileURL(resolve(root, `${prefix}${path}.${extension}`)).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { readFileSync, writeFileSync } = await import('node:fs'); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const YAML = require('yaml'); + const config = YAML.parseDocument(readFileSync(configPath, 'utf8')); + config.setIn(['serialization', 'bigInt'], ${bigint}); + writeFileSync(configPath, config.toString()); + await import(${JSON.stringify(runtimeUrl('server/threads/threadServer'))}); + const env = await import(${JSON.stringify(runtimeUrl('utility/environment/environmentManager'))}); + env.initSync(); assert.strictEqual(env.get('serialization_bigInt'), ${bigint}); + const codecs = await import(${JSON.stringify(runtimeUrl('server/serverHelpers/contentTypes'))}); + const jsonStream = await import(${JSON.stringify(runtimeUrl('server/serverHelpers/JSONStream'))}); + const custom = value => 'custom:' + value; + codecs.contentTypes.set('test/custom-json', { serialize: custom }); + const { runStartup } = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + const expected = ${bigint} ? jsonStream.stringify : JSON.stringify; + assert.strictEqual(codecs.getMessageSerializer(), expected); + for (const type of ['application/json', '*/*', '']) assert.strictEqual(codecs.contentTypes.get(type).serialize, expected); + assert.strictEqual(codecs.contentTypes.get('test/custom-json').serialize, custom); + const request = { headers: { get: () => 'application/json' } }; + const response = { headers: new Map() }; + assert.strictEqual(codecs.serialize({ value: 42 }, request, response), '{"value":42}'); + assert.strictEqual(codecs.serializeMessage({ value: 42 }, request), '{"value":42}'); + assert.strictEqual(request.serialize, expected); + const bigValue = { value: 9007199254740993n }; + if (${bigint}) { + assert.strictEqual(codecs.serialize(bigValue, request, response), '{"value":9007199254740993}'); + assert.strictEqual(codecs.getDeserializer('application/json')(Buffer.from('{"value":9007199254740993}')).value, bigValue.value); + } else { + assert.throws(() => codecs.serializeMessage(bigValue, request), error => error.message === 'Cannot serialize BigInt to JSON'); + assert.strictEqual(codecs.getMessageSerializer()({ value: 42, extra: true }, ['value'], 2), JSON.stringify({ value: 42 }, null, 2)); + assert.strictEqual(codecs.getDeserializer('application/json')(Buffer.from('{"value":9007199254740993}')).value, 9007199254740992); + } + const ndjson = codecs.contentTypes.get('application/x-ndjson').serializeStream([{ value: 42 }]); + let streamed = ''; for await (const chunk of ndjson) streamed += chunk; + assert.strictEqual(streamed, '{"value":42}\\n'); + console.log('direct configured codecs served'); + ` + ); + assert.strictEqual(output, 'direct configured codecs served', mode + ':' + bigint); + } + } + }); + + it('honors disabled and enabled response compression after serialization startup', () => { + for (const mode of ['compiled', 'typestrip']) { + for (const threshold of [0, 32]) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const runtimeUrl = (path) => pathToFileURL(resolve(root, `${prefix}${path}.${extension}`)).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { readFileSync, writeFileSync } = await import('node:fs'); + const YAML = require('yaml'); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const config = YAML.parseDocument(readFileSync(configPath, 'utf8')); + config.setIn(['http', 'compressionThreshold'], ${threshold}); + writeFileSync(configPath, config.toString()); + await import(${JSON.stringify(runtimeUrl('server/threads/threadServer'))}); + const codecs = await import(${JSON.stringify(runtimeUrl('server/serverHelpers/contentTypes'))}); + const { runStartup } = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + const value = { value: 'x'.repeat(100) }; + const request = { headers: { asObject: { accept: 'application/json', 'accept-encoding': 'br' } } }; + const response = { headers: new Map() }; + const body = await codecs.serialize(value, request, response); + if (${threshold}) { + const { brotliDecompressSync } = await import('node:zlib'); + assert.strictEqual(response.headers.get('Content-Encoding'), 'br'); + assert.ok(body instanceof Uint8Array); + assert.strictEqual(brotliDecompressSync(body).toString(), JSON.stringify(value)); + } else { + assert.strictEqual(response.headers.has('Content-Encoding'), false); + assert.strictEqual(body, JSON.stringify(value)); + } + console.log('configured response compression served'); + ` + ); + assert.strictEqual(output, 'configured response compression served', mode); + } + } + }); + + it('keeps rotation disabled after rapid cold reconfiguration', () => { + for (const mode of ['compiled', 'typestrip']) { + const runtimeUrl = pathToFileURL( + resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}utility/logging/harper_logger.${mode === 'compiled' ? 'js' : 'ts'}` + ) + ).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { readFileSync, writeFileSync, existsSync, readdirSync } = await import('node:fs'); + const { setTimeout: delay } = await import('node:timers/promises'); + const { waitFor } = require(${JSON.stringify(resolve(root, 'unitTests/waitFor.js'))}); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const YAML = require('yaml'); + const config = YAML.parseDocument(readFileSync(configPath, 'utf8')); + config.deleteIn(['logging', 'rotation']); + writeFileSync(configPath, config.toString()); + const { createLogger, updateLogger } = await import(${JSON.stringify(runtimeUrl)}); + const path = process.env.ROOTPATH + '/rapid.log'; + const archives = process.env.ROOTPATH + '/rapid-archives'; + const logger = createLogger({ path, level: 'info', stdStreams: false }); + const rotation = { interval: '0.001s', auditInterval: 10, compress: false, path: archives }; + updateLogger(logger, { path, rotation }, undefined, logger); + updateLogger(logger, { path, rotation: { ...rotation, retention: '1d' } }, undefined, logger); + updateLogger(logger, { path }, undefined, logger); + logger.info('rotation must stay disabled'); + await delay(600); + assert.deepStrictEqual(existsSync(archives) ? readdirSync(archives) : [], []); + assert.match(readFileSync(path, 'utf8'), /rotation must stay disabled/); + assert.doesNotMatch(readFileSync(path, 'utf8'), /Error initializing log rotator/); + updateLogger(logger, { path, rotation }, undefined, logger); + await waitFor(() => existsSync(archives) && readdirSync(archives).length > 0, { timeout: 3000 }); + updateLogger(logger, { path }, undefined, logger); + await delay(200); + const ended = readdirSync(archives); + await delay(200); + assert.deepStrictEqual(readdirSync(archives), ended); + logger.closeLogFile(); + console.log('rotation remained disabled'); + ` + ); + assert.strictEqual(output, 'rotation remained disabled', mode); + } + }); + + it('drains scopes without binding when shutdown interrupts HTTP startup', () => { + for (const mode of ['compiled', 'typestrip']) { + const workerPath = resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}server/threads/threadServer.${mode === 'compiled' ? 'js' : 'ts'}` + ); + const execArgv = [ + ...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), + '--require', + resolve(import.meta.dirname, 'fixtures/startup-shutdown.cjs'), + ]; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { Worker } = await import('node:worker_threads'); + const { once } = await import('node:events'); + const worker = new Worker(${JSON.stringify(workerPath)}, { execArgv: ${JSON.stringify(execArgv)}, workerData: { addPorts: [], addThreadIds: [] } }); + const messages = []; worker.on('message', message => messages.push(message)); + try { + while (!messages.some(message => message.type === 'startup-held')) await once(worker, 'message', { signal: AbortSignal.timeout(3000) }); + worker.postMessage({ type: 'shutdown', restartNumber: 2 }); + worker.postMessage({ type: 'release-startup' }); + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(3000) }); + assert.strictEqual(code, 0); + assert.ok(messages.some(message => message.type === 'scope-disposed')); + assert.ok(messages.some(message => message.type === 'startup-drained')); + assert.strictEqual(messages.some(message => message.type === 'child_started' || message.type === 'child_startup_phase'), false); + } finally { if (worker.threadId !== -1) await worker.terminate(); } + console.log('startup shutdown drained'); + ` + ); + assert.strictEqual(output, 'startup shutdown drained', mode); + } + }); + + it('drains the held component scope when shutdown interrupts component loading', () => { + for (const mode of ['compiled', 'typestrip']) { + const workerPath = resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}server/threads/threadServer.${mode === 'compiled' ? 'js' : 'ts'}` + ); + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { readFileSync, writeFileSync } = await import('node:fs'); + const YAML = require('yaml'); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const config = YAML.parseDocument(readFileSync(configPath, 'utf8')); + config.set('startupShutdownProbe', {}); + writeFileSync(configPath, config.toString()); + process.env.HARPER_BUILTIN_COMPONENTS = 'startupShutdownProbe=' + ${JSON.stringify(resolve(import.meta.dirname, 'fixtures/component-shutdown.cjs'))}; + const { Worker } = await import('node:worker_threads'); + const { once } = await import('node:events'); + const worker = new Worker(${JSON.stringify(workerPath)}, { execArgv: ${JSON.stringify(mode === 'typestrip' ? ['--conditions=typestrip'] : [])}, workerData: { addPorts: [], addThreadIds: [] } }); + const messages = []; worker.on('message', message => messages.push(message)); + const deadline = AbortSignal.timeout(5000); + try { + while (!messages.some(message => message.type === 'component-held')) await once(worker, 'message', { signal: deadline }); + worker.postMessage({ type: 'shutdown', restartNumber: 2 }); + worker.postMessage({ type: 'release-component' }); + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(3000) }); + assert.strictEqual(code, 0); + assert.ok(messages.some(message => message.type === 'component-disposed')); + assert.ok(messages.some(message => message.type === 'child_startup_phase' && message.phase === 'loading components')); + assert.strictEqual(messages.some(message => message.type === 'child_started' || (message.type === 'child_startup_phase' && message.phase !== 'loading components')), false); + } finally { if (worker.threadId !== -1) await worker.terminate(); } + console.log('component shutdown drained'); + ` + ); + assert.strictEqual(output, 'component shutdown drained', mode); + } + }); + + it('preserves environment-selected built-ins through startup', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const runtimeUrl = (path) => pathToFileURL(resolve(root, `${prefix}${path}.${extension}`)).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + process.env.HARPER_BUILTIN_COMPONENTS = 'agent=@/test/agent,operationsApi=@/test/operationsApi'; + const { TRUSTED_RESOURCE_PLUGINS } = await import(${JSON.stringify(runtimeUrl('components/componentLoader'))}); + const { runStartup } = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + assert.strictEqual(TRUSTED_RESOURCE_PLUGINS.agent, '@/test/agent'); + assert.strictEqual(TRUSTED_RESOURCE_PLUGINS.operationsApi, '@/test/operationsApi'); + console.log('built-in overrides preserved'); + ` + ); + assert.strictEqual(output, 'built-in overrides preserved', mode); + } + }); + + it('writes conditional logger warnings before startup runs', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const runtimeUrl = (path) => pathToFileURL(resolve(root, `${prefix}${path}.${extension}`)).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { readFileSync } = await import('node:fs'); + const { join } = await import('node:path'); + assert.strictEqual(typeof globalThis.module, 'undefined'); + const { createLogger, setMainLogger } = await import(${JSON.stringify(runtimeUrl('utility/logging/harper_logger'))}); + const logPath = join(process.env.ROOTPATH, 'before-startup.log'); + setMainLogger(createLogger({ path: logPath, level: 'warn', stdStreams: false })); + const { logger } = await import(${JSON.stringify(runtimeUrl('utility/logging/logger'))}); + const { hasStarted } = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + assert.strictEqual(hasStarted(), false); + logger.warn('conditional warning before startup'); + assert.match(readFileSync(logPath, 'utf8'), /conditional warning before startup/); + console.log('pre-startup warning written'); + ` + ); + assert.strictEqual(output, 'pre-startup warning written', mode); + } + }); + + it('skips empty component configuration without reporting a load error', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const runtimeUrl = (path) => pathToFileURL(resolve(root, `${prefix}${path}.${extension}`)).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { mkdirSync, writeFileSync } = await import('node:fs'); + const { join } = await import('node:path'); + const componentDirectory = join(process.env.ROOTPATH, 'empty-component'); + mkdirSync(componentDirectory); writeFileSync(join(componentDirectory,'config.yaml'),'# empty configuration\\n'); + const { loadComponent, trackBootOutcomes, bootVerdictOf } = await import(${JSON.stringify(runtimeUrl('components/componentLoader'))}); + const { Resources } = await import(${JSON.stringify(runtimeUrl('resources/Resources'))}); + trackBootOutcomes(['empty-component']); + const resources = new Resources(); + await loadComponent(componentDirectory,resources,'test-origin'); + assert.deepStrictEqual(bootVerdictOf('empty-component').failures, []); + assert.strictEqual(resources.get(''), undefined); + console.log('empty configuration skipped'); + ` + ); + assert.strictEqual(output, 'empty configuration skipped', mode); + } + }); + + it('records configured analytics after the real cold server graph starts', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const runtimeUrl = (path) => pathToFileURL(resolve(root, `${prefix}${path}.${extension}`)).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + await import(${JSON.stringify(runtimeUrl('server/threads/threadServer'))}); + const env = await import(${JSON.stringify(runtimeUrl('utility/environment/environmentManager'))}); + env.initSync(); assert.ok(env.get('analytics_aggregatePeriod') > -1); + const { runStartup } = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + const { recordAction, addAnalyticsListener, setAnalyticsEnabled } = await import(${JSON.stringify(runtimeUrl('resources/analytics/write'))}); + const reports = new Promise((resolve,reject) => { + const timeout = setTimeout(() => reject(new Error('configured analytics did not record a cold-start sample')),5000); + addAnalyticsListener(metrics => { + const metric = metrics.find(metric => metric.metric === 'typestrip-configured-sample'); + if (metric) { clearTimeout(timeout); resolve(metric); } + }); + }); + recordAction(7,'typestrip-configured-sample'); + assert.strictEqual((await reports).mean,7); + setAnalyticsEnabled(false); + const { closeLoadedDatabases } = await import(${JSON.stringify(runtimeUrl('resources/databases'))}); + await closeLoadedDatabases(); + console.log('configured analytics recorded'); + ` + ); + assert.strictEqual(output, 'configured analytics recorded', mode); + } + }); + + it('exposes the declared public globals in source and compiled facades and user workers', async () => { + const names = [ + 'contentTypes', + 'createBlob', + 'databases', + 'logger', + 'models', + 'operation', + 'Resource', + 'secrets', + 'server', + 'tables', + 'threads', + 'transaction', + ]; + for (const mode of ['compiled', 'typestrip']) { + const modulePath = pathToFileURL(resolve(root, mode === 'compiled' ? 'dist/index.js' : 'index.ts')).href; + const output = execFileSync( + process.execPath, + [...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), '--input-type=module'], + { + input: `const assert = (await import('node:assert')).default; + const facade = await import(${JSON.stringify(modulePath)}); + const names = ${JSON.stringify(names)}; + assert.deepStrictEqual(Object.keys(facade).filter(name => names.includes(name)).sort(), names.sort()); + for (const name of names) assert.strictEqual(facade[name], globalThis[name], name); + assert.strictEqual(typeof facade.Resource, 'function'); + assert.strictEqual(typeof facade.operation, 'function'); + assert.strictEqual(typeof facade.server.http, 'function'); + assert.ok(new facade.Resource('public-facade') instanceof facade.Resource); + console.log('public values preserved'); process.exit(0);`, + env: process.env, + encoding: 'utf8', + timeout: 30000, + } + ); + assert.strictEqual(output.trim(), 'public values preserved', mode); + const worker = new Worker(resolve(root, 'unitTests/bin/user-thread.js'), { + execArgv: mode === 'typestrip' ? ['--conditions=typestrip'] : [], + workerData: { noServerStart: true, addPorts: [], addThreadIds: [] }, + }); + try { + for await (const [message] of on(worker, 'message', { signal: AbortSignal.timeout(30000) })) { + if (!Object.hasOwn(message, 'hasResource')) continue; + assert.deepStrictEqual(message, { hasResource: true, hasServer: true }, mode); + break; + } + } finally { + await worker.terminate(); + } + } + }); + + it('keeps HTTP workers alive through startup hooks with unreferenced completion sources', () => { + for (const mode of ['compiled', 'typestrip']) { + const workerPath = resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}server/threads/threadServer.${mode === 'compiled' ? 'js' : 'ts'}` + ); + const execArgv = [ + ...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), + '--require', + resolve(import.meta.dirname, 'fixtures/startup-ref.cjs'), + ]; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { Worker } = await import('node:worker_threads'); + const { once } = await import('node:events'); + const worker = new Worker(${JSON.stringify(workerPath)}, { execArgv: ${JSON.stringify(execArgv)}, workerData: { addPorts: [], addThreadIds: [] } }); + const messages = []; worker.on('message', message => messages.push(message)); + try { + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(10000) }); + assert.strictEqual(code, 0); + assert.ok(messages.some(message => message.type === 'startup-ref' && message.held === true)); + assert.ok(messages.some(message => message.type === 'startup-hook-completed')); + } finally { if (worker.threadId !== -1) await worker.terminate(); } + console.log('startup hook completed');` + ); + assert.strictEqual(output, 'startup hook completed', mode); + } + }); + + it('passes the source condition to actual managed workers without NODE_OPTIONS', () => { + const manager = pathToFileURL(resolve(root, 'server/threads/manageThreads.ts')).href; + const fixture = resolve(import.meta.dirname, 'fixtures/runtime-condition.cjs'); + const output = execFileSync(process.execPath, ['--conditions=typestrip', '--input-type=module'], { + input: `const { once } = await import('node:events'); + const { startWorker } = await import(${JSON.stringify(manager)}); + const worker = startWorker(${JSON.stringify(fixture)}, { name: 'runtime-condition', autoRestart: false }); + try { + const [message] = await once(worker, 'message', { signal: AbortSignal.timeout(10000) }); + console.log(JSON.stringify(message)); + } finally { await worker.terminate(); } + process.exit(0);`, + env: { ...process.env, NODE_OPTIONS: '' }, + encoding: 'utf8', + timeout: 30000, + }); + const message = JSON.parse(output.trim()); + assert.strictEqual(message.modulePath, resolve(root, 'server/Server.ts')); + assert.ok(message.execArgv.includes('--conditions=typestrip')); + }); + + it('keeps default tagged logs on the current main sink and explicit loggers on their own sink', () => { + mkdirSync(resolve(root, 'cache'), { recursive: true }); + const directory = mkdtempSync(resolve(root, 'cache/tagged-logger-')); + try { + for (const mode of ['compiled', 'typestrip']) { + const modulePath = pathToFileURL( + resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}utility/logging/harper_logger.${mode === 'compiled' ? 'js' : 'ts'}` + ) + ).href; + const output = execFileSync( + process.execPath, + [...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), '--input-type=module'], + { + input: `const assert = (await import('node:assert')).default; + const { readFileSync, existsSync } = await import('node:fs'); + const { createLogger, setMainLogger, loggerWithTag } = await import(${JSON.stringify(modulePath)}); + const firstPath = ${JSON.stringify(resolve(directory, mode + '-first.log'))}; + const secondPath = ${JSON.stringify(resolve(directory, mode + '-second.log'))}; + const first = createLogger({path:firstPath,level:'info',stdStreams:false}); + setMainLogger(first); + const implicit = loggerWithTag('implicit'); + const explicit = loggerWithTag('explicit', false, first); + const conditional = loggerWithTag('conditional', true); + assert.strictEqual(conditional.debug, null); + implicit.notify('before replacement'); + setMainLogger(createLogger({path:secondPath,level:'trace',stdStreams:false})); + implicit.notify('after replacement'); + explicit.notify('explicit remains'); + assert.strictEqual(conditional.debug, null); + assert.ok(existsSync(secondPath), 'implicit tag did not follow new main sink'); + assert.match(readFileSync(secondPath,'utf8'), /\\[implicit\\].*after replacement/); + const firstText = readFileSync(firstPath,'utf8'); + assert.match(firstText, /\\[explicit\\].*explicit remains/); + assert.doesNotMatch(firstText, /after replacement/); + console.log('tagged logs preserved'); process.exit(0);`, + env: process.env, + encoding: 'utf8', + timeout: 30000, + } + ); + assert.strictEqual(output.trim(), 'tagged logs preserved', mode); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('initializes the legacy launcher before authenticating a session cookie', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const output = runWithIsolatedRoot( + mode, + `const assert = (await import('node:assert')).default; + await import(${JSON.stringify(pathToFileURL(resolve(root, `${prefix}launchServiceScripts/launchHarperDB.js`)).href)}); + const { hasStarted, runStartup } = await import(${JSON.stringify(pathToFileURL(resolve(root, `${prefix}utility/lifecycle.${extension}`)).href)}); + assert.strictEqual(hasStarted(), true); + await runStartup(); + const { authentication } = await import(${JSON.stringify(pathToFileURL(resolve(root, `${prefix}security/auth.${extension}`)).href)}); + const result = await authentication({headers:{asObject:{host:'example.test',cookie:'example_test-hdb-session=missing'}},ip:'203.0.113.1',method:'GET',pathname:'/',protocol:'http'}, async () => ({status:200,headers:new Headers(),body:{ok:true}})); + assert.strictEqual(result.status, 200); + const { closeLoadedDatabases } = await import(${JSON.stringify(pathToFileURL(resolve(root, `${prefix}resources/databases.${extension}`)).href)}); + await closeLoadedDatabases(); + console.log('launcher initialized');` + ); + assert.strictEqual(output.trim(), 'launcher initialized', mode); + } + }); + + it('parses CSV streams through the shared file-load helper in both modes', () => { + for (const mode of ['compiled', 'typestrip']) { + const modulePath = pathToFileURL( + resolve(root, `${mode === 'compiled' ? 'dist/' : ''}utility/common_utils.${mode === 'compiled' ? 'js' : 'ts'}`) + ).href; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { Readable } = await import('node:stream'); + const { parsePromise } = await import(${JSON.stringify(modulePath)}); + const rows = []; + await parsePromise(Readable.from([${JSON.stringify('\uFEFFid,name\n1,first\n2,second\n')}]), (reject, result) => { + if (result.errors.length) reject(new Error(JSON.stringify(result.errors))); + rows.push(...result.data); + }, value => value); + assert.deepStrictEqual(rows, [{ id: '1', name: 'first' }, { id: '2', name: 'second' }]); + console.log('CSV stream parsed');` + ); + assert.strictEqual(output, 'CSV stream parsed', mode); + } + }); + + it('imports the CLI without running a command', () => { + const cli = pathToFileURL(resolve(root, 'bin/harper.ts')).href; + const services = pathToFileURL(resolve(root, 'utility/processManagement/servicesConfig.ts')).href; + const output = execFileSync( + process.execPath, + [ + '--conditions=typestrip', + '--input-type=module', + '-e', + `const cli = await import(${JSON.stringify(cli)}); const services = await import(${JSON.stringify(services)}); const { existsSync } = await import('node:fs'); for (const config of [services.generateMainServerConfig(), services.generateRestart()]) { if (!config.script.endsWith('.ts') || !existsSync(config.script)) throw new Error('Missing source service entry'); } console.log(typeof cli.harper);`, + ], + { env: process.env, encoding: 'utf8', timeout: 30000 } + ); + assert.strictEqual(output.trim(), 'function'); + }); + + it('resolves compiled service entries inside the distribution', () => { + const services = require('#src/utility/processManagement/servicesConfig'); + for (const config of [services.generateMainServerConfig(), services.generateRestart()]) { + assert.ok(config.script.startsWith(resolve(root, 'dist') + '/')); + assert.ok(config.script.endsWith('.js')); + assert.ok(existsSync(config.script)); + } + }); + + it('chooses runtime paths from canonical module locations', () => { + mkdirSync(resolve(root, 'cache'), { recursive: true }); + const directory = mkdtempSync(resolve(root, 'cache/typestrip-links-')); + const link = resolve(directory, 'harper'); + try { + symlinkSync(root, link, process.platform === 'win32' ? 'junction' : 'dir'); + for (const mode of ['compiled', 'typestrip']) { + const modulePath = resolve(link, `${mode === 'compiled' ? 'dist/' : ''}utility/packageUtils.js`); + const output = execFileSync( + process.execPath, + [ + '--preserve-symlinks', + '-e', + `(async () => { const { RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT, loadRuntimeModule } = require(${JSON.stringify(modulePath)}); const { pathToFileURL } = require('node:url'); const { resolve } = require('node:path'); const imported = await import(pathToFileURL(resolve(${JSON.stringify(link)}, ${JSON.stringify(mode === 'compiled' ? 'dist/dataLayer/search.js' : 'dataLayer/search.ts')}))); if (loadRuntimeModule('dataLayer/search').search !== imported.search) throw new Error('Duplicate module instance'); console.log(JSON.stringify([RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT])); process.exit(0); })().catch(error => { console.error(error); process.exit(1); });`, + ], + { encoding: 'utf8', timeout: 30000 } + ); + assert.deepStrictEqual(JSON.parse(output), [ + mode === 'compiled' ? resolve(root, 'dist') : root, + mode === 'compiled' ? '.js' : '.ts', + ]); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('keeps SQL cold until first use in both runtime graphs', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const entry = pathToFileURL(resolve(root, `${prefix}server/threads/threadServer.${extension}`)).href; + const helpers = pathToFileURL(resolve(root, `${prefix}utility/packageUtils.js`)).href; + const output = execFileSync( + process.execPath, + [...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), '--input-type=module'], + { + input: `const { registerHooks } = await import('node:module'); + const resolutions = []; + registerHooks({ resolve(specifier, context, next) { const result = next(specifier, context); resolutions.push(result.url); return result; } }); + await import(${JSON.stringify(entry)}); + if (resolutions.some(url => url.includes('/alasql/') || url.includes('/mathjs/') || url.endsWith('/sqlTranslator/index.${extension}'))) throw new Error('SQL loaded during boot'); + const { loadRuntimeModule } = await import(${JSON.stringify(helpers)}); + const sql = loadRuntimeModule('sqlTranslator/index'); + if (typeof sql.evaluateSQL !== 'function' || typeof sql.convertSQLToAST !== 'function' || typeof loadRuntimeModule('dataLayer/SQLSearch').default !== 'function' || typeof loadRuntimeModule('sqlTranslator/SelectValidator').default !== 'function') throw new Error('Cold module export missing'); + if (!resolutions.some(url => url.includes('/alasql/'))) throw new Error('Cold dependency observation missing'); + console.log('cold SQL loaded'); process.exit(0);`, + env: process.env, + encoding: 'utf8', + timeout: 30000, + } + ); + assert.strictEqual(output.trim(), 'cold SQL loaded', mode); + } + }); + + it('preserves Resource inheritance through lazy source classes', () => { + const modules = [ + ['resources/ErrorResource', 'ErrorResource'], + ['security/certificateVerification/certificateVerificationSource', 'CertificateVerificationSource'], + ]; + const { Resource } = require('#src/resources/Resource'); + for (const [modulePath, name] of modules) { + const Source = require(`#src/${modulePath}`)[name]; + class Derived extends Source {} + assert.strictEqual(Object.getPrototypeOf(Source), Resource); + assert.ok(new Derived(new Error('test')) instanceof Derived); + } + const output = execFileSync(process.execPath, ['--conditions=typestrip', '--input-type=module'], { + input: `const { Resource } = await import(${JSON.stringify(pathToFileURL(resolve(root, 'resources/Resource.ts')).href)}); for (const [modulePath, name] of ${JSON.stringify(modules)}) { const Source = (await import(new URL(modulePath + '.ts', ${JSON.stringify(pathToFileURL(root + '/').href)})))[name]; class Derived extends Source {} if (Object.getPrototypeOf(Source) !== Resource || !(new Derived(new Error('test')) instanceof Derived)) throw new Error('Lost Resource inheritance'); } console.log('inherited'); process.exit(0);`, + env: process.env, + encoding: 'utf8', + timeout: 30000, + }); + assert.strictEqual(output.trim(), 'inherited'); + }); + + it('keeps worker callbacks registered before the cyclic runtime finishes loading', () => { + const router = pathToFileURL(resolve(root, 'server/threads/socketRouter.ts')).href; + const state = pathToFileURL(resolve(root, 'server/threads/threadMessageState.ts')).href; + const operations = pathToFileURL(resolve(root, 'server/serverHelpers/serverUtilities.ts')).href; + const dispatch = pathToFileURL(resolve(root, 'server/serverHelpers/operationDispatchState.ts')).href; + const output = execFileSync(process.execPath, ['--conditions=typestrip', '--input-type=module'], { + input: `const { workerHooks, listenersByType } = await import(${JSON.stringify(state)}); const schemaListener = () => {}; listenersByType.set('schema', [schemaListener]); const router = await import(${JSON.stringify(router)}); if (listenersByType.get('schema')?.[0] !== schemaListener) throw new Error('Lost event listener'); if (workerHooks.reconcile !== router.reconcileIsolatedWorkers || typeof workerHooks.monitorListener !== 'function') throw new Error('Lost startup registration'); const operations = await import(${JSON.stringify(operations)}); const { operationDispatchState } = await import(${JSON.stringify(dispatch)}); if (operationDispatchState.local?.chooseOperation !== operations.chooseOperation || operationDispatchState.local?.processLocalTransaction !== operations.processLocalTransaction) throw new Error('Lost operation dispatch'); console.log('registered'); process.exit(0);`, + env: process.env, + encoding: 'utf8', + timeout: 30000, + }); + assert.strictEqual(output.trim(), 'registered'); + }); + + it('shares the parent process incarnation with workers without loading the server', async () => { + const modulePath = require.resolve('#src/server/threads/processIncarnation'); + const { processIncarnation } = require(modulePath); + assert.match(processIncarnation, /^[a-f0-9]{16}$/); + const worker = new Worker( + `const { parentPort } = require('node:worker_threads'); parentPort.postMessage(require(${JSON.stringify(modulePath)}).processIncarnation);`, + { eval: true, workerData: { processIncarnation } } + ); + try { + const [inherited] = await once(worker, 'message'); + assert.strictEqual(inherited, processIncarnation); + } finally { + await worker.terminate(); + } + }); + + it('imports the CLI and server graph in evaluated workers without starting HTTP servers', async () => { + for (const mode of ['compiled', 'typestrip']) { + const cli = pathToFileURL( + resolve(root, `${mode === 'compiled' ? 'dist/' : ''}bin/harper.${mode === 'compiled' ? 'js' : 'ts'}`) + ).href; + const entry = pathToFileURL( + resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}server/threads/threadServer.${mode === 'compiled' ? 'js' : 'ts'}` + ) + ).href; + const worker = new Worker( + `const { parentPort } = require('node:worker_threads'); import(${JSON.stringify(cli)}).then(async cli => { await import(${JSON.stringify(entry)}); const registry = await import(${JSON.stringify(pathToFileURL(resolve(root, `${mode === 'compiled' ? 'dist/' : ''}server/serverRegistry.${mode === 'compiled' ? 'js' : 'ts'}`)).href)}); parentPort.postMessage({ type: 'import-complete', cli: typeof cli.harper, started: Object.values(registry.SERVERS).some(server => server.listening) }); }).catch(error => parentPort.postMessage({ type: 'import-complete', error: error.stack }));`, + { + eval: true, + execArgv: mode === 'typestrip' ? ['--conditions=typestrip'] : [], + workerData: { addPorts: [], addThreadIds: [], restartNumber: 1 }, + } + ); + try { + const result = await new Promise((resolve, reject) => { + const signal = AbortSignal.timeout(30000); + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + worker.on('message', (message) => { + if (message?.type === 'import-complete') resolve(message); + }); + worker.once('error', reject); + worker.once('exit', (code) => reject(new Error(`Worker exited before importing the server (${code})`))); + }); + assert.strictEqual(result.error, undefined, mode); + assert.strictEqual(result.cli, 'function', mode); + assert.strictEqual(result.started, false, mode); + } finally { + await worker.terminate(); + } + } + }); + + it('refreshes fresh-install configuration and bounds the real Node authorization cache', function () { + if (process.versions.bun) this.skip(); + for (const mode of ['compiled', 'typestrip']) { + const runtimeUrl = (path) => + pathToFileURL( + resolve(root, `${mode === 'compiled' ? 'dist/' : ''}${path}.${mode === 'compiled' ? 'js' : 'ts'}`) + ).href; + const observations = { + [runtimeUrl('security/jsLoader')]: + 'globalThis.snapshots.loader = () => ({lockdown:APPLICATIONS_LOCKDOWN, fsAllowed:ALLOWED_NODE_BUILTIN_MODULES.has("fs"), pathAllowed:ALLOWED_NODE_BUILTIN_MODULES.has("path")});', + [runtimeUrl('server/mqtt')]: 'globalThis.snapshots.mqtt = () => AUTHORIZE_LOCAL;', + [runtimeUrl('server/serverHelpers/registeredOperations')]: + 'globalThis.snapshots.timeout = () => EXECUTE_TIMEOUT_MS;', + [runtimeUrl('dataLayer/bulkLoad')]: 'globalThis.snapshots.directory = () => TEMP_DOWNLOAD_DIR;', + [runtimeUrl('server/threads/threadServer')]: 'globalThis.snapshots.debug = () => debugThreads;', + }; + const signature = + mode === 'typestrip' ? 'export function get(propName: string): any {' : 'function get(propName) {'; + const observer = + ' if (globalThis.coldRecording) globalThis.coldReads.push({key:propName,caller:new Error().stack.split("\\n")[2]}); '; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const fs = require('node:fs'); + const { registerHooks } = await import('node:module'); + const { fileURLToPath } = await import('node:url'); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + fs.unlinkSync(configPath); + delete process.env.DEV_MODE; delete process.env.HARPER_SET_CONFIG; + globalThis.snapshots = {}; globalThis.hookControls = []; + globalThis.coldReads = []; globalThis.coldRecording = true; + const observations = ${JSON.stringify(observations)}; + registerHooks({ load(url, context, next) { + const result = next(url, context); + if (url === ${JSON.stringify(runtimeUrl('utility/environment/environmentManager'))}) { + const source = (result.source || fs.readFileSync(fileURLToPath(url), 'utf8')).toString(); + assert.ok(source.includes(${JSON.stringify(signature)})); + return {...result, source:source.replace(${JSON.stringify(signature)}, ${JSON.stringify(signature + observer)})}; + } + if (observations[url]) { + globalThis.hookControls.push({url,beforeConfig:!fs.existsSync(configPath)}); + return {...result, source:(result.source || fs.readFileSync(fileURLToPath(url), 'utf8')).toString() + '\\n' + observations[url]}; + } + if (url !== ${JSON.stringify(runtimeUrl('security/auth'))}) return result; + const source = (result.source || fs.readFileSync(fileURLToPath(url), 'utf8')).toString(); + assert.strictEqual(source.split('setInterval(() => {').length - 1, 1); + globalThis.hookControls.push({url,beforeConfig:!fs.existsSync(configPath)}); + return {...result, source:source.replace('setInterval(() => {','globalThis.authCacheTimer = setInterval(() => {')}; + } }); + await import(${JSON.stringify(runtimeUrl('bin/run'))}); + globalThis.coldRecording = false; + assert.strictEqual(globalThis.hookControls.length,6); + assert.ok(globalThis.hookControls.every(control=>control.beforeConfig)); + assert.strictEqual(globalThis.authCacheTimer,undefined); + const owners = [...new Set(globalThis.coldReads.map(({key,caller}) => { + let path = caller.slice(caller.indexOf(${JSON.stringify(root + '/')}) + ${root.length + 1}).split(':')[0]; + if (path.startsWith('dist/')) path = path.slice(5); + if (path.endsWith('.js')) path = path.slice(0,-3) + '.ts'; + return path + ':' + key.toLowerCase(); + }))].sort(); + assert.deepStrictEqual(owners, ["dataLayer/bulkLoad.ts:hdb_root", "resources/DatabaseTransaction.ts:storage_debuglongtransactions", "resources/DatabaseTransaction.ts:storage_maxtransactionopentime", "resources/DatabaseTransaction.ts:storage_maxtransactionqueuetime", "resources/RecordEncoder.ts:storage_maxreadtransactionopentime", "resources/Table.ts:storage_prefetchwrites", "resources/analytics/write.ts:analytics_aggregateperiod", "resources/auditStore.ts:logging_auditretention", "resources/databases.ts:storage_pagesize", "security/jsLoader.ts:applications_allowedbuiltinmodules", "security/jsLoader.ts:applications_lockdown", "security/tokenAuthentication.ts:authentication_operationtokentimeout", "security/tokenAuthentication.ts:authentication_refreshtokentimeout", "security/user.ts:authentication_hashfunction", "server/mqtt.ts:authentication_authorizelocal", "server/serverHelpers/contentTypes.ts:http_compressionthreshold", "server/serverHelpers/contentTypes.ts:serialization_bigint", "server/serverHelpers/registeredOperations.ts:operationsapi_network_timeout", "server/storageReclamation.ts:storage_reclamation_interval", "server/storageReclamation.ts:storage_reclamation_threshold", "server/threads/manageThreads.ts:threads_heapsnapshotnearlimit", "server/threads/threadServer.ts:threads_debug", "utility/lmdb/OpenDBIObject.ts:storage_caching", "utility/password.ts:authentication_hashfunction"]); + materializePerPidRoot(); + const YAML = require('yaml'); + const config = YAML.parseDocument(fs.readFileSync(configPath,'utf8')); + config.setIn(['authentication','cacheTTL'],31337); + config.setIn(['authentication','authorizeLocal'],false); + config.setIn(['applications','allowedBuiltinModules'],['path']); + config.setIn(['applications','lockdown'],'freeze-after-load'); + config.setIn(['operationsApi','network','timeout'],31339); + config.setIn(['operationsApi','network','domainSocket'],false); + config.setIn(['threads','debug'],true); + fs.writeFileSync(configPath,config.toString()); + const env = await import(${JSON.stringify(runtimeUrl('utility/environment/environmentManager'))}); + env.initSync(); + const net = await import('node:net'); + const reservation = net.createServer(); + await new Promise(resolve=>reservation.listen(0,'127.0.0.1',resolve)); + const port = reservation.address().port; + await new Promise(resolve=>reservation.close(resolve)); + env.setProperty('threads_debug_port',port); + env.setProperty('threads_debug_host','127.0.0.1'); + process.env.DEV_MODE='true'; + const {runStartup} = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + assert.strictEqual(globalThis.authCacheTimer._idleTimeout,31337); + assert.strictEqual(globalThis.authCacheTimer.hasRef(),false); + assert.deepStrictEqual(globalThis.snapshots.loader(),{lockdown:'freeze-after-load',fsAllowed:false,pathAllowed:true}); + assert.strictEqual(globalThis.snapshots.mqtt(),false); + assert.strictEqual(globalThis.snapshots.timeout(),31339); + assert.strictEqual(globalThis.snapshots.directory(),process.env.ROOTPATH+'/tmp'); + assert.strictEqual(globalThis.snapshots.debug(),true); + const inspector = await import('node:inspector'); + const inspectorUrl = inspector.url(); + assert.strictEqual(new URL(inspectorUrl).port,String(port)); + const timer = globalThis.authCacheTimer; + await runStartup(); + assert.strictEqual(globalThis.authCacheTimer,timer); + assert.strictEqual(inspector.url(),inspectorUrl); + inspector.close(); + const databases = await import(${JSON.stringify(runtimeUrl('resources/databases'))}); + await databases.closeLoadedDatabases(); + console.log('fresh-install configuration honored'); + ` + ); + assert.strictEqual(output, 'fresh-install configuration honored', mode); + } + }); + + it('honors final main-thread inspector overrides on configured roots', function () { + if (process.versions.bun) this.skip(); + for (const mode of ['compiled', 'typestrip']) { + const runtimeUrl = (path) => + pathToFileURL( + resolve(root, `${mode === 'compiled' ? 'dist/' : ''}${path}.${mode === 'compiled' ? 'js' : 'ts'}`) + ).href; + for (const [initialDebug, finalDebug] of [ + [false, true], + [true, false], + [true, true], + ]) { + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + delete process.env.DEV_MODE; delete process.env.HARPER_SET_CONFIG; + const fs = require('node:fs'); const YAML = require('yaml'); + const net = await import('node:net'); + const reservations = [net.createServer(),net.createServer()]; + for (const server of reservations) await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + const [initialPort,finalPort] = reservations.map(server=>server.address().port); + for (const server of reservations) await new Promise(resolve=>server.close(resolve)); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const config = YAML.parseDocument(fs.readFileSync(configPath,'utf8')); + config.setIn(['threads','debug'],${initialDebug}); + config.setIn(['threads','debug_port'],initialPort); + config.setIn(['threads','debug_host'],'127.0.0.1'); + config.setIn(['operationsApi','network','domainSocket'],false); + fs.writeFileSync(configPath,config.toString()); + await import(${JSON.stringify(runtimeUrl('bin/run'))}); + const inspector = await import('node:inspector'); + assert.strictEqual(inspector.url(),undefined,'main-thread inspector waits for final configuration'); + const env = await import(${JSON.stringify(runtimeUrl('utility/environment/environmentManager'))}); + env.setProperty('threads_debug',${finalDebug}); + env.setProperty('threads_debug_port',finalPort); + env.setProperty('threads_debug_host','127.0.0.1'); + const {runStartup} = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + if (${finalDebug}) { + assert.ok(inspector.url(),'main-thread inspector honors the final debug override'); + assert.strictEqual(new URL(inspector.url()).port,String(finalPort)); + } else assert.strictEqual(inspector.url(),undefined); + inspector.close(); + const {closeLoadedDatabases} = await import(${JSON.stringify(runtimeUrl('resources/databases'))}); + await closeLoadedDatabases(); + console.log('final inspector override honored'); + ` + ); + assert.strictEqual(output, 'final inspector override honored', mode + ':' + initialDebug + ':' + finalDebug); + } + } + }); + + it('initializes a configured worker inspector once across import and startup', function () { + if (process.versions.bun) this.skip(); + for (const mode of ['compiled', 'typestrip']) { + const runtimeUrl = (path) => + pathToFileURL( + resolve(root, `${mode === 'compiled' ? 'dist/' : ''}${path}.${mode === 'compiled' ? 'js' : 'ts'}`) + ).href; + const workerCode = ` + const assert = (await import('node:assert')).default; + const {parentPort} = await import('node:worker_threads'); + parentPort.ref(); + const {registerHooks} = await import('node:module'); + const {readFileSync} = await import('node:fs'); + const {fileURLToPath} = await import('node:url'); + globalThis.inspectorAttempts = 0; + registerHooks({load(url,context,next) { + const result = next(url,context); + if (url !== ${JSON.stringify(runtimeUrl('server/threads/threadServer'))}) return result; + const source = (result.source || readFileSync(fileURLToPath(url),'utf8')).toString(); + assert.strictEqual(source.split('inspectorInitialized = true;').length - 1,1); + return {...result,source:source.replace('inspectorInitialized = true;','inspectorInitialized = true; globalThis.inspectorAttempts++;')}; + }}); + await import(${JSON.stringify(runtimeUrl('server/threads/threadServer'))}); + const inspector = await import('node:inspector'); + assert.strictEqual(globalThis.inspectorAttempts,1); + assert.strictEqual(new URL(inspector.url()).port,String(process.env.EXPECTED_INSPECTOR_PORT)); + const initialUrl = inspector.url(); + const {runStartup} = await import(${JSON.stringify(runtimeUrl('utility/lifecycle'))}); + await runStartup(); + assert.strictEqual(globalThis.inspectorAttempts,1,'startup skips the already initialized worker inspector'); + assert.strictEqual(inspector.url(),initialUrl); + inspector.close(); + const {closeLoadedDatabases} = await import(${JSON.stringify(runtimeUrl('resources/databases'))}); + await closeLoadedDatabases(); + parentPort.postMessage({type:'inspector-proof'}); + `; + const output = runWithIsolatedRoot( + mode, + ` + delete process.env.DEV_MODE; delete process.env.HARPER_SET_CONFIG; + const net = await import('node:net'); const reservation = net.createServer(); + await new Promise(resolve=>reservation.listen(0,'127.0.0.1',resolve)); + const port = reservation.address().port; + await new Promise(resolve=>reservation.close(resolve)); + const fs = require('node:fs'); const YAML = require('yaml'); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const config = YAML.parseDocument(fs.readFileSync(configPath,'utf8')); + config.setIn(['threads','debug'],true); + config.setIn(['operationsApi','network','domainSocket'],false); + fs.writeFileSync(configPath,config.toString()); + const {Worker} = await import('node:worker_threads'); + const worker = new Worker(new URL('data:text/javascript,' + encodeURIComponent(${JSON.stringify(workerCode)})),{ + execArgv:${JSON.stringify(mode === 'typestrip' ? ['--conditions=typestrip'] : [])}, + env:{...process.env,EXPECTED_INSPECTOR_PORT:String(port)}, + workerData:{noServerStart:true,workerIndex:0,workerCount:1,addPorts:[],addThreadIds:[],configOverrides:{threads_debug:true,threads_debug_startingPort:port,threads_debug_host:'127.0.0.1'}} + }); + try { + await new Promise((resolve,reject)=>{ + const signal = AbortSignal.timeout(25000); + signal.addEventListener('abort',()=>reject(signal.reason),{once:true}); + worker.on('message',message=>{if(message.type === 'inspector-proof') resolve();}); + worker.once('error',reject); + worker.once('exit',code=>reject(new Error('Worker exited before inspector proof: '+code))); + }); + } finally { await worker.terminate(); } + console.log('worker inspector initialized once'); + ` + ); + assert.strictEqual(output, 'worker inspector initialized once', mode); + } + }); +}); + +describe('TypeStrip core packaging', () => { + it('keeps lazy plugins in the caller module family when preserving symlinks', () => { + const directory = mkdtempSync(resolve(root, 'unitTests/envDir/plugin-symlink-')); + const link = resolve(directory, 'core'); + symlinkSync(root, link, 'dir'); + try { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + let output = ''; + try { + output = execFileSync( + process.execPath, + ['--preserve-symlinks', ...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), '--input-type=module'], + { + encoding: 'utf8', + timeout: 30000, + input: ` + const assert = (await import('node:assert')).default; + const { createRequire } = await import('node:module'); + const require = createRequire(${JSON.stringify(pathToFileURL(resolve(root, 'package.json')).href)}); + process.env.ROOTPATH = require('./unitTests/perPidRoot.js').materializePerPidRoot(); + console.log('fixture-root ' + process.pid); + const { TRUSTED_RESOURCE_PLUGINS } = await import(${JSON.stringify(pathToFileURL(resolve(link, prefix + 'components/componentLoader.' + extension)).href)}); + assert.strictEqual(TRUSTED_RESOURCE_PLUGINS.fastifyRoutes, ${JSON.stringify(pathToFileURL(resolve(link, prefix + 'server/fastifyRoutes.' + extension)).href)}); + await import(TRUSTED_RESOURCE_PLUGINS.fastifyRoutes); + const { closeLoadedDatabases } = await import(${JSON.stringify(pathToFileURL(resolve(link, prefix + 'resources/databases.' + extension)).href)}); + await closeLoadedDatabases(); + console.log('plugin family preserved'); process.exit(0); + `, + } + ); + assert.strictEqual(output.trim().split('\n').at(-1), 'plugin family preserved', mode); + } finally { + const pid = /fixture-root (\d+)/.exec(output)?.[1]; + if (pid) rmSync(resolve(root, 'unitTests/envDir', pid), { recursive: true, force: true }); + } + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('links every schema reader and shares the schema in both runtimes', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const readers = [ + 'security/user', + 'utility/globalSchema', + 'utility/mount_hdb', + 'upgrade/directives/5-1-0', + 'upgrade/directives/5-2-0', + 'upgrade/directives/5-3-0', + 'dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute', + 'dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch', + ]; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const schema = await import(${JSON.stringify(pathToFileURL(resolve(root, prefix + 'utility/systemSchema.js')).href)}); + for (const url of ${JSON.stringify(readers.map((path) => pathToFileURL(resolve(root, prefix + path + '.' + extension)).href))}) await import(url); + const { getSystemSchema } = await import(${JSON.stringify(pathToFileURL(resolve(root, prefix + 'utility/globalSchema.' + extension)).href)}); + assert.strictEqual(getSystemSchema(), schema.systemSchema); + const { closeLoadedDatabases } = await import(${JSON.stringify(pathToFileURL(resolve(root, prefix + 'resources/databases.' + extension)).href)}); + await closeLoadedDatabases(); + console.log('schema readers linked'); + ` + ); + assert.strictEqual(output, 'schema readers linked', mode); + } + }); + + it('preserves the public base and default extension for relative worker paths', () => { + for (const mode of ['compiled', 'typestrip']) { + const prefix = mode === 'compiled' ? 'dist/' : ''; + const extension = mode === 'compiled' ? 'js' : 'ts'; + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { writeFileSync } = await import('node:fs'); + const { join, relative } = await import('node:path'); + const { once } = await import('node:events'); + const filename = join(process.env.ROOTPATH, 'relative-worker.js'); + writeFileSync(filename, "require('node:worker_threads').parentPort.postMessage('default js worker');"); + const { startWorker } = await import(${JSON.stringify(pathToFileURL(resolve(root, prefix + 'server/threads/manageThreads.' + extension)).href)}); + const worker = startWorker(relative(${JSON.stringify(root)}, filename).slice(0, -3), { name: 'relative-path-probe', autoRestart: false }); + try { + const [message] = await once(worker, 'message', { signal: AbortSignal.timeout(10000) }); + assert.strictEqual(message, 'default js worker'); + } finally { if (worker.threadId !== -1) await worker.terminate(); } + console.log('relative worker contract preserved'); + ` + ); + assert.strictEqual(output, 'relative worker contract preserved', mode); + } + }); + + it('boots embedded core workers and invokes the product CLI only once', () => { + for (const engine of ['rocksdb', 'lmdb']) { + const output = runWithIsolatedRoot( + 'compiled', + ` + process.env.HARPER_STORAGE_ENGINE = ${JSON.stringify(engine)}; + await require(${JSON.stringify(resolve(import.meta.dirname, 'fixtures/embedded-layout.cjs'))}).probeEmbeddedRuntime(${JSON.stringify(root)}); + console.log('embedded runtime preserved'); + ` + ); + assert.strictEqual(output, 'embedded runtime preserved', engine); + } + }); +}); + +describe('TypeStrip canary admission', () => { + for (const scenario of ['admitted', 'refused', 'shutdown-during-load', 'bind-retry']) { + it(`preserves cleanup and readiness when ${scenario}`, () => { + for (const mode of ['compiled', 'typestrip']) { + const workerPath = resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}server/threads/threadServer.${mode === 'compiled' ? 'js' : 'ts'}` + ); + const output = runWithIsolatedRoot( + mode, + ` + const assert = (await import('node:assert')).default; + const { readFileSync, writeFileSync } = await import('node:fs'); + const { createServer, createConnection } = await import('node:net'); + const { Worker } = await import('node:worker_threads'); + const { once } = await import('node:events'); + const { waitFor } = require(${JSON.stringify(resolve(root, 'unitTests/waitFor.js'))}); + const scenario = ${JSON.stringify(scenario)}; + const pool = scenario === 'bind-retry'; + const reservation = createServer(); + reservation.listen(0, '127.0.0.1'); + await once(reservation, 'listening'); + const port = reservation.address().port; + if (!pool) await new Promise(resolve => reservation.close(resolve)); + const YAML = require('yaml'); + const configPath = process.env.ROOTPATH + '/harper-config.yaml'; + const config = YAML.parseDocument(readFileSync(configPath, 'utf8')); + config.set('admissionScopeProbe', {}); + writeFileSync(configPath, config.toString()); + process.env.HARPER_BUILTIN_COMPONENTS = 'admissionScopeProbe=' + ${JSON.stringify(resolve(import.meta.dirname, 'fixtures/admission-scope.cjs'))}; + const worker = new Worker(${JSON.stringify(workerPath)}, { + execArgv: ${JSON.stringify(mode === 'typestrip' ? ['--conditions=typestrip'] : [])}, + workerData: { addPorts: [], addThreadIds: [], reportsLoad: true, holdProbeLoad: true, probePort: port, probePool: pool, name: pool ? 'replication' : 'http', poolIndex: 0, workerPools: pool ? ['replication'] : [] }, + }); + const messages = []; + let workerError, exited; + worker.on('message', message => messages.push(message)); + worker.on('error', error => workerError = error); + worker.on('exit', code => exited = code); + const exit = once(worker, 'exit', { signal: AbortSignal.timeout(15000) }); + const wait = type => waitFor(() => { + if (workerError) throw workerError; + const fixtureError = messages.find(message => message.type === 'admission-fixture-error'); + if (fixtureError) throw new Error(fixtureError.message); + const message = messages.find(message => message.type === type); + if (!message && exited !== undefined) throw new Error('Worker exited before ' + type + ': ' + exited); + return message; + }, { timeout: 10000, message: () => 'Waiting for ' + type + '; got ' + messages.map(message => message.type + ':' + (message.phase ?? '')).join(', ') }); + try { + await wait('admission-scope-loaded'); + if (scenario === 'shutdown-during-load') { + worker.postMessage({ type: 'shutdown', restartNumber: 2 }); + await wait('admission-scope-disposed'); + } + worker.postMessage({ type: 'release-admission-probe' }); + await wait('child_component_verdict'); + assert.strictEqual(messages.some(message => message.type === 'admission-components-loaded'), false); + assert.strictEqual(messages.some(message => message.type === 'child_started'), false); + if (!pool && scenario !== 'shutdown-during-load') { + const socket = createConnection({ host: '127.0.0.1', port }); + try { await assert.rejects(once(socket, 'connect'), { code: 'ECONNREFUSED' }); } + finally { socket.destroy(); } + } + if (scenario === 'admitted' || pool) { + worker.postMessage({ type: 'child_admitted' }); + if (pool) await wait('admission-bind-error'); + else { + await wait('child_started'); + await wait('admission-components-loaded'); + const socket = createConnection({ host: '127.0.0.1', port }); + let text = ''; + socket.on('data', chunk => text += chunk); + try { await once(socket, 'end', { signal: AbortSignal.timeout(3000) }); assert.strictEqual(text, 'admitted'); } + finally { socket.destroy(); } + } + } + if (scenario !== 'shutdown-during-load') worker.postMessage({ type: 'shutdown', restartNumber: 2 }); + await waitFor(() => exited !== undefined, { timeout: 6000, message: 'Graceful exit must finish before the 10s backstop' }); + const [code] = await exit; + assert.strictEqual(code, 0); + assert.ok(messages.some(message => message.type === 'admission-scope-disposed')); + if (scenario !== 'admitted') assert.strictEqual(messages.some(message => message.type === 'child_started'), false); + if (scenario === 'refused' || scenario === 'shutdown-during-load') assert.strictEqual(messages.some(message => message.type === 'admission-components-loaded'), false); + } finally { + if (worker.threadId !== -1) await worker.terminate(); + if (reservation.listening) await new Promise(resolve => reservation.close(resolve)); + } + console.log('admission boundary preserved'); + ` + ); + assert.strictEqual(output, 'admission boundary preserved', mode + ':' + scenario); + } + }); + } +}); diff --git a/unitTests/server/threads/typestripSource.test.mjs b/unitTests/server/threads/typestripSource.test.mjs new file mode 100644 index 0000000000..7ccc64341d --- /dev/null +++ b/unitTests/server/threads/typestripSource.test.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert'; +import { readdirSync, readFileSync } from 'node:fs'; +import { resolve, join, relative } from 'node:path'; +import ts from 'typescript'; + +const root = resolve(import.meta.dirname, '../../..'); +const sourceDirectories = [ + 'agent', + 'bin', + 'components', + 'config', + 'dataLayer', + 'resources', + 'security', + 'server', + 'sqlEngine', + 'sqlTranslator', + 'utility', + 'validation', +]; + +function hasLocalRequire(call) { + for (let scope = call.parent; scope; scope = scope.parent) { + if (!ts.isBlock(scope) && !ts.isSourceFile(scope)) continue; + for (const statement of scope.statements) { + if (!ts.isVariableStatement(statement)) continue; + for (const declaration of statement.declarationList.declarations) { + if ( + ts.isIdentifier(declaration.name) && + declaration.name.text === 'require' && + declaration.initializer && + ts.isCallExpression(declaration.initializer) && + ts.isIdentifier(declaration.initializer.expression) && + declaration.initializer.expression.text === 'createRequire' + ) + return true; + } + } + } + return false; +} + +describe('TypeStrip source imports', () => { + it('uses an explicit runtime loader for production require calls', () => { + const unbound = []; + for (const directory of sourceDirectories) { + const parent = join(root, directory); + for (const file of readdirSync(parent, { recursive: true })) { + if (!file.endsWith('.ts')) continue; + const absolute = join(parent, file); + const source = ts.createSourceFile(absolute, readFileSync(absolute, 'utf8'), ts.ScriptTarget.Latest, true); + const visit = (node) => { + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === 'require' && + !hasLocalRequire(node) + ) { + const { line } = source.getLineAndCharacterOfPosition(node.getStart(source)); + unbound.push(`${relative(root, absolute)}:${line + 1}`); + } + ts.forEachChild(node, visit); + }; + visit(source); + } + } + assert.deepStrictEqual(unbound, [], `Bare require calls fail in native ESM: ${unbound.join(', ')}`); + }); +}); diff --git a/unitTests/testUtils.js b/unitTests/testUtils.js index bce09f58e8..d98578fafe 100644 --- a/unitTests/testUtils.js +++ b/unitTests/testUtils.js @@ -13,6 +13,7 @@ const { getDatabases } = require('#src/resources/databases'); const { handleHDBError } = require('#src/utility/errors/hdbError'); const { PRIVATEKEY_PEM_NAME } = require('#src/utility/terms/certificates'); const { materializePerPidRoot } = require('./perPidRoot.js'); +const lifecycle = require('#src/utility/lifecycle'); let envMgrInitSyncStub; @@ -89,6 +90,12 @@ function preTestPrep(testConfigObj) { // Try to change to bin changeProcessToBinDir(); env.initTestEnvironment(testConfigObj); + + // Drain startup hooks so modules that defer side effects via `onStartup(...)` + // (server-singleton wiring, listener registration, config-derived constants) + // see a wired-up state. Production calls this from `bin/harper.ts`; tests + // that go through this helper get the same effect. Idempotent. + void lifecycle.runStartup(); } /** diff --git a/unitTests/utility/lifecycle.test.js b/unitTests/utility/lifecycle.test.js new file mode 100644 index 0000000000..7716e1c4f7 --- /dev/null +++ b/unitTests/utility/lifecycle.test.js @@ -0,0 +1,181 @@ +'use strict'; + +const assert = require('node:assert'); +const { execFileSync } = require('node:child_process'); +const { resolve } = require('node:path'); + +const root = resolve(__dirname, '../..'); + +// Isolate the registry so these checks cannot reset the real runtime's startup hooks. +function run(mode, code) { + const modulePath = resolve( + root, + `${mode === 'compiled' ? 'dist/' : ''}utility/lifecycle.${mode === 'compiled' ? 'js' : 'ts'}` + ); + const output = execFileSync( + process.execPath, + [ + ...(mode === 'typestrip' ? ['--conditions=typestrip'] : []), + '-e', + `const assert = require('node:assert'); + const { onStartup, runStartup, resetStartupForTests, hasStarted } = require(${JSON.stringify(modulePath)}); + (async () => { ${code} })().then(() => console.log('completed')).catch(error => { console.error(error); process.exitCode = 1; });`, + ], + { encoding: 'utf8', timeout: 30000 } + ); + assert.strictEqual(output.trim(), 'completed'); +} + +for (const mode of ['compiled', 'typestrip']) { + describe(`startup lifecycle (${mode})`, () => { + it('awaits registered hooks in order and runs them only once', () => { + run( + mode, + ` + const calls = []; + let release; + const barrier = new Promise(resolve => { release = resolve; }); + assert.strictEqual(hasStarted(), false); + onStartup(async () => { calls.push('first'); await barrier; calls.push('finished'); }); + onStartup(() => { calls.push('second'); }); + const startup = runStartup(); + assert.strictEqual(hasStarted(), true); + assert.strictEqual(runStartup(), startup); + assert.deepStrictEqual(calls, ['first']); + release(); + await startup; + await runStartup(); + assert.deepStrictEqual(calls, ['first', 'finished', 'second']); + ` + ); + }); + + it('resets pending hooks and the cached startup promise', () => { + run( + mode, + ` + const calls = []; + onStartup(() => { calls.push('discarded'); }); + resetStartupForTests(); + onStartup(() => { calls.push('first'); }); + const first = runStartup(); + await first; + resetStartupForTests(); + assert.strictEqual(hasStarted(), false); + onStartup(() => { calls.push('second'); }); + const second = runStartup(); + assert.notStrictEqual(second, first); + await second; + assert.deepStrictEqual(calls, ['first', 'second']); + ` + ); + }); + + it('schedules late hooks independently of the completed startup promise', () => { + run( + mode, + ` + await runStartup(); + let release, complete; + let entered = false, finished = false; + const barrier = new Promise(resolve => { release = resolve; }); + const completion = new Promise(resolve => { complete = resolve; }); + onStartup(async () => { entered = true; await barrier; finished = true; complete(); }); + assert.strictEqual(entered, false); + await Promise.resolve(); + assert.strictEqual(entered, true); + await runStartup(); + assert.strictEqual(finished, false); + release(); + await completion; + assert.strictEqual(finished, true); + ` + ); + }); + + it('preserves startup failure and skips subsequent hooks', () => { + run( + mode, + ` + const error = new Error('startup failed'); + let subsequent = false; + onStartup(() => { throw error; }); + onStartup(() => { subsequent = true; }); + const startup = runStartup(); + await assert.rejects(startup, actual => actual === error); + assert.strictEqual(runStartup(), startup); + assert.strictEqual(subsequent, false); + ` + ); + }); + + it('awaits nested hooks in FIFO order and preserves reentrant promise identity', () => { + run( + mode, + ` + const calls = []; + let release, entered, reentrant; + const barrier = new Promise(resolve => { release = resolve; }); + const nestedEntered = new Promise(resolve => { entered = resolve; }); + onStartup(() => { + calls.push('first'); + reentrant = runStartup(); + onStartup(async () => { calls.push('nested'); entered(); await barrier; calls.push('finished'); }); + }); + onStartup(() => calls.push('second')); + const startup = runStartup(); + assert.strictEqual(reentrant, startup); + let settled = false; + startup.then(() => { settled = true; }); + await nestedEntered; + assert.strictEqual(settled, false); + assert.deepStrictEqual(calls, ['first', 'second', 'nested']); + release(); await startup; + assert.deepStrictEqual(calls, ['first', 'second', 'nested', 'finished']); + ` + ); + }); + + it('propagates nested failures and schedules post-failure hooks independently', () => { + run( + mode, + ` + const error = new Error('nested startup failed'); + let subsequent = false; + onStartup(() => { + onStartup(() => { throw error; }); + onStartup(() => { subsequent = true; }); + }); + const startup = runStartup(); + await assert.rejects(startup, actual => actual === error); + assert.strictEqual(runStartup(), startup); + assert.strictEqual(subsequent, false); + let complete; + const late = new Promise(resolve => { complete = resolve; }); + onStartup(complete); await late; + ` + ); + }); + + it('keeps a reset during an active drain separate from its next generation', () => { + run( + mode, + ` + let release; + const barrier = new Promise(resolve => { release = resolve; }); + const calls = []; + onStartup(async () => { calls.push('old'); await barrier; }); + onStartup(() => calls.push('discarded')); + const old = runStartup(); + resetStartupForTests(); + onStartup(() => calls.push('new')); + const current = runStartup(); + await current; + release(); await old; + assert.strictEqual(runStartup(), current); + assert.deepStrictEqual(calls, ['old', 'new']); + ` + ); + }); + }); +} diff --git a/unitTests/utility/mount_hdb.test.js b/unitTests/utility/mount_hdb.test.js index 9dd0a4a354..0a03e5577c 100644 --- a/unitTests/utility/mount_hdb.test.js +++ b/unitTests/utility/mount_hdb.test.js @@ -55,7 +55,7 @@ describe('test mount_hdb module', () => { create_table_stub = sandbox.stub(bridge, 'createTable'); mount_hdb.__set__('mkdirpSync', mk_dirp_sync_stub); mount_hdb.__set__('copySync', sandbox.stub()); - mount_hdb.__set__('systemSchema_json_1', { default: test_system_schema }); + mount_hdb.__set__('systemSchema', test_system_schema); }); after(() => { diff --git a/unitTests/utility/packageUtils.test.js b/unitTests/utility/packageUtils.test.js index 6ee50fe53c..5f4decbb92 100644 --- a/unitTests/utility/packageUtils.test.js +++ b/unitTests/utility/packageUtils.test.js @@ -2,9 +2,20 @@ const { describe, it } = require('mocha'); const assert = require('node:assert'); const { join } = require('node:path'); -const { readFileSync, realpathSync, symlinkSync, rmSync, mkdtempSync } = require('node:fs'); +const { + readFileSync, + realpathSync, + symlinkSync, + rmSync, + mkdtempSync, + mkdirSync, + copyFileSync, + writeFileSync, +} = require('node:fs'); const os = require('node:os'); const { execFileSync } = require('node:child_process'); +const { Worker } = require('node:worker_threads'); +const { once } = require('node:events'); const packageUtils = require('#src/utility/packageUtils'); @@ -61,3 +72,89 @@ describe('packageUtils', () => { } }); }); + +describe('core runtime layouts', () => { + const sourceRoot = join(__dirname, '../..'); + const expectedSchema = JSON.parse(readFileSync(join(sourceRoot, 'json/systemSchema.json'), 'utf8')); + for (const [name, relativeRoot, source] of [ + ['standalone source', '', true], + ['standalone compiled', 'dist', false], + ['embedded compiled', 'dist/core', false], + ['source without an inner manifest', 'core', true], + ]) { + it(`launches a worker and loads core assets in ${name}`, async () => { + const fixture = mkdtempSync(join(os.tmpdir(), 'harper-core-layout-')); + let worker; + try { + const runtime = join(fixture, relativeRoot); + mkdirSync(join(runtime, 'utility'), { recursive: true }); + mkdirSync(join(runtime, 'json'), { recursive: true }); + writeFileSync(join(fixture, 'package.json'), JSON.stringify({ name: 'outer-product', version: '1.0.0' })); + copyFileSync(join(sourceRoot, 'utility/packageUtils.js'), join(runtime, 'utility/packageUtils.js')); + copyFileSync(join(sourceRoot, 'utility/systemSchema.js'), join(runtime, 'utility/systemSchema.js')); + copyFileSync(join(sourceRoot, 'json/systemSchema.json'), join(runtime, 'json/systemSchema.json')); + if (source) copyFileSync(join(sourceRoot, 'utility/hdbTerms.ts'), join(runtime, 'utility/hdbTerms.ts')); + const extension = source ? '.ts' : '.js'; + writeFileSync( + join(runtime, `layout-worker${extension}`), + source + ? "import { parentPort } from 'node:worker_threads'; parentPort.postMessage('core worker');" + : "require('node:worker_threads').parentPort.postMessage('core worker');" + ); + const helpers = require(join(runtime, 'utility/packageUtils.js')); + assert.strictEqual(helpers.PACKAGE_ROOT, realpathSync(fixture)); + assert.strictEqual(helpers.packageJson.name, 'outer-product'); + const { systemSchema } = require(join(runtime, 'utility/systemSchema.js')); + assert.deepStrictEqual(systemSchema, expectedSchema); + assert.strictEqual(systemSchema, require(join(runtime, 'json/systemSchema.json'))); + worker = new Worker(join(helpers.RUNTIME_SRC_ROOT, `layout-worker${helpers.RUNTIME_FILE_EXT}`)); + const [message] = await once(worker, 'message', { signal: AbortSignal.timeout(5000) }); + assert.strictEqual(message, 'core worker'); + } finally { + if (worker && worker.threadId !== -1) await worker.terminate(); + rmSync(fixture, { recursive: true, force: true }); + } + }); + } + + for (const prefix of ['', 'core/']) { + it(`emits the current schema into dist/${prefix}json on every build`, () => { + const fixture = mkdtempSync(join(os.tmpdir(), 'harper-schema-build-')); + try { + mkdirSync(join(fixture, prefix, 'utility'), { recursive: true }); + mkdirSync(join(fixture, prefix, 'json'), { recursive: true }); + copyFileSync(join(sourceRoot, 'utility/systemSchema.js'), join(fixture, prefix, 'utility/systemSchema.js')); + copyFileSync(join(sourceRoot, 'json/systemSchema.json'), join(fixture, prefix, 'json/systemSchema.json')); + writeFileSync(join(fixture, 'package.json'), JSON.stringify({ name: 'schema-build-fixture' })); + writeFileSync( + join(fixture, 'tsconfig.json'), + JSON.stringify({ + files: [`${prefix}utility/systemSchema.js`], + compilerOptions: { + allowJs: true, + resolveJsonModule: true, + module: 'NodeNext', + moduleResolution: 'NodeNext', + rootDir: '.', + outDir: 'dist', + types: [], + }, + }) + ); + const build = () => + execFileSync(process.execPath, [require.resolve('typescript/bin/tsc'), '--project', fixture], { + encoding: 'utf8', + timeout: 30000, + }); + build(); + const emitted = join(fixture, 'dist', prefix, 'json/systemSchema.json'); + assert.deepStrictEqual(JSON.parse(readFileSync(emitted, 'utf8')), expectedSchema); + writeFileSync(emitted, '{"stale":true}'); + build(); + assert.deepStrictEqual(JSON.parse(readFileSync(emitted, 'utf8')), expectedSchema); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } + }); + } +}); diff --git a/unitTests/utility/watchPath.test.js b/unitTests/utility/watchPath.test.js index 126e4554b9..0aa2bda2a2 100644 --- a/unitTests/utility/watchPath.test.js +++ b/unitTests/utility/watchPath.test.js @@ -154,7 +154,7 @@ describe('watchPath', () => { 'config/RootConfigWatcher.ts', 'resources/blob.ts', 'security/keys.ts', - 'server/threads/manageThreads.js', + 'server/threads/manageThreads.ts', 'utility/watcherFallback.ts', ]; diff --git a/upgrade/directives/5-1-0.ts b/upgrade/directives/5-1-0.ts index 006599fc1c..30e27b2d38 100644 --- a/upgrade/directives/5-1-0.ts +++ b/upgrade/directives/5-1-0.ts @@ -1,4 +1,5 @@ -'use strict'; +import * as CreateTableObjectCjsModule from '../../dataLayer/CreateTableObject.ts'; +('use strict'); // 5.1.0 — introduces system.hdb_deployment for deployment tracking. // @@ -15,9 +16,9 @@ // failing on peer nodes. import { databases } from '../../resources/databases.ts'; -import systemSchema from '../../json/systemSchema.json'; +import { systemSchema } from '../../utility/systemSchema.js'; import * as terms from '../../utility/hdbTerms.ts'; -import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import bridge from '../../dataLayer/harperBridge/harperBridge.ts'; import hdbLogger from '../../utility/logging/harper_logger.ts'; @@ -36,8 +37,7 @@ async function createHdbDeploymentIfMissing() { hdbLogger.info(`Creating system.${DEPLOYMENT_TABLE} table for deployment tracking.`); - const CreateTableObject = - require('../../dataLayer/CreateTableObject').default || require('../../dataLayer/CreateTableObject'); + const CreateTableObject = CreateTableObjectCjsModule.default || CreateTableObjectCjsModule; const schema = (systemSchema as any)[DEPLOYMENT_TABLE]; if (!schema) { throw new Error(`systemSchema.${DEPLOYMENT_TABLE} is missing; cannot run 5.1.0 directive.`); diff --git a/upgrade/directives/5-2-0.ts b/upgrade/directives/5-2-0.ts index 31a67401fe..fce0b835c1 100644 --- a/upgrade/directives/5-2-0.ts +++ b/upgrade/directives/5-2-0.ts @@ -1,4 +1,5 @@ -'use strict'; +import * as CreateTableObjectCjsModule from '../../dataLayer/CreateTableObject.ts'; +('use strict'); // 5.2.0 — introduces system.hdb_secret for the encrypted secrets store (#715). // @@ -13,10 +14,10 @@ // dependent code means it never fires and the table is missing on upgraded installs // (see the mis-tagging history documented in 5-1-0.ts). +import { systemSchema } from '../../utility/systemSchema.js'; import { databases } from '../../resources/databases.ts'; -import systemSchema from '../../json/systemSchema.json'; import * as terms from '../../utility/hdbTerms.ts'; -import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import bridge from '../../dataLayer/harperBridge/harperBridge.ts'; import hdbLogger from '../../utility/logging/harper_logger.ts'; @@ -32,8 +33,7 @@ async function createHdbSecretIfMissing() { hdbLogger.info(`Creating system.${SECRET_TABLE} table for the secrets store.`); - const CreateTableObject = - require('../../dataLayer/CreateTableObject').default || require('../../dataLayer/CreateTableObject'); + const CreateTableObject = CreateTableObjectCjsModule.default || CreateTableObjectCjsModule; const schema = (systemSchema as any)[SECRET_TABLE]; if (!schema) { throw new Error(`systemSchema.${SECRET_TABLE} is missing; cannot run 5.2.0 directive.`); diff --git a/upgrade/directives/5-3-0.ts b/upgrade/directives/5-3-0.ts index a30ac9a80e..4b61e84ff8 100644 --- a/upgrade/directives/5-3-0.ts +++ b/upgrade/directives/5-3-0.ts @@ -1,4 +1,5 @@ -'use strict'; +import * as CreateTableObjectCjsModule from '../../dataLayer/CreateTableObject.ts'; +('use strict'); // 5.3.0 — introduces system.hdb_oidc_trust and system.hdb_oidc_token_use for OIDC trusted // publishing (#2171), system.hdb_model_decisions and system.hdb_model_outcomes for durable @@ -11,9 +12,9 @@ // happens when it does not, and DESIGN.md "System table bootstrap" for the three touchpoints. import { databases } from '../../resources/databases.ts'; -import systemSchema from '../../json/systemSchema.json'; +import { systemSchema } from '../../utility/systemSchema.js'; import * as terms from '../../utility/hdbTerms.ts'; -import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import bridge from '../../dataLayer/harperBridge/harperBridge.ts'; import hdbLogger from '../../utility/logging/harper_logger.ts'; import { declareTokenUseTable } from '../../security/authn/oidc/tokenUseTable.ts'; @@ -47,8 +48,7 @@ async function createHdbOidcTrustIfMissing() { hdbLogger.info(`Creating system.${OIDC_TRUST_TABLE} table for OIDC trusted publishing.`); - const CreateTableObject = - require('../../dataLayer/CreateTableObject').default || require('../../dataLayer/CreateTableObject'); + const CreateTableObject = CreateTableObjectCjsModule.default || CreateTableObjectCjsModule; const schema = (systemSchema as any)[OIDC_TRUST_TABLE]; if (!schema) { throw new Error(`systemSchema.${OIDC_TRUST_TABLE} is missing; cannot run 5.3.0 directive.`); @@ -99,8 +99,7 @@ async function createSystemTableIfMissing(tableName: string, purpose: string) { hdbLogger.info(`Creating system.${tableName} table for ${purpose}.`); - const CreateTableObject = - require('../../dataLayer/CreateTableObject').default || require('../../dataLayer/CreateTableObject'); + const CreateTableObject = CreateTableObjectCjsModule.default || CreateTableObjectCjsModule; const schema = (systemSchema as any)[tableName]; if (!schema) { throw new Error(`systemSchema.${tableName} is missing; cannot run 5.3.0 directive.`); diff --git a/utility/DESIGN.md b/utility/DESIGN.md index d56763d931..b903daa727 100644 --- a/utility/DESIGN.md +++ b/utility/DESIGN.md @@ -8,6 +8,10 @@ Index of every design note: [DESIGN.md](../DESIGN.md). --- +## Core runtime and product paths (`packageUtils.js`) + +Core-owned files resolve from their runtime module tree, including lazy plugin URLs and the shared compiler-emitted `systemSchema.js` JSON asset. Package metadata, public relative-worker paths and compiled product startup keep their package owner. `packageUtils.test.js` and `typestrip-runtime.test.mjs` enforce these boundaries with standalone/embedded builds, real workers and one product CLI invocation. + ## Every path handed to a native file watch must be canonicalized (`utility/watchPath.ts`) libuv's Windows fs-event callback rebuilds each event's absolute path, expands it with diff --git a/utility/common_utils.ts b/utility/common_utils.ts index 423124fdde..ffdb55a59d 100644 --- a/utility/common_utils.ts +++ b/utility/common_utils.ts @@ -1,24 +1,30 @@ -'use strict'; -import * as path from 'path'; -import * as fs from 'fs-extra'; +import { loadNativePackage } from './packageUtils.js'; +('use strict'); +import * as path from 'node:path'; +import fs from 'fs-extra'; import log from './logging/harper_logger.ts'; -import * as fsExtra from 'fs-extra'; -import * as os from 'os'; -import * as net from 'net'; +import fsExtra from 'fs-extra'; +import * as os from 'node:os'; +import * as net from 'node:net'; import RecursiveIterator from 'recursive-iterator'; import * as terms from './hdbTerms.ts'; import { PACKAGE_ROOT } from './packageUtils.js'; export { PACKAGE_ROOT }; -import * as papaParse from 'papaparse'; +import papaParse from 'papaparse'; import isNumber from 'is-number'; import minimist from 'minimist'; -import * as https from 'https'; -import * as http from 'http'; +import * as https from 'node:https'; +import * as http from 'node:http'; +// Lazy getDatabases accessor: importing databases.ts here creates circular deps +// (databases.ts → Table.ts/auditStore.ts/etc → common_utils.ts). Instead we use a +// shared registry module that has no deps of its own; databases.ts registers its +// getter there at load time, and we read it here at call time. +import { getDatabases } from './databasesRef.ts'; const ISO_DATE = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/; -import * as util from 'util'; +import * as util from 'node:util'; export const asyncSetTimeout = util.promisify(setTimeout); const EMPTY_STRING = ''; @@ -456,7 +462,7 @@ export async function isPortTaken(port: number) { * @returns string returns a thrown message if schema and or table does not exist */ export function checkGlobalSchemaTable(schemaName: string, tableName: string) { - let databases = require('../resources/databases').getDatabases(); + let databases = getDatabases(); if (!databases[schemaName]) { return hdbErrors.HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(schemaName); } @@ -540,7 +546,6 @@ export function checkSchemaTableExist(schema: string, table: string) { * @returns {string} */ export function checkSchemaExists(schema: string) { - const { getDatabases } = require('../resources/databases'); if (!getDatabases()[schema]) { return hdbErrors.HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(schema); } @@ -553,7 +558,6 @@ export function checkSchemaExists(schema: string) { * @returns {string} */ export function checkTableExists(schema: string, table: string) { - const { getDatabases } = require('../resources/databases'); if (!getDatabases()[schema][table]) { return hdbErrors.HDB_ERROR_MSGS.TABLE_NOT_FOUND(schema, table); } @@ -564,7 +568,7 @@ export function checkTableExists(schema: string, table: string) { * @returns {number} */ export function getStartOfTomorrowInSeconds() { - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); let tomorowSeconds = moment().utc().add(1, 'd').startOf('d').unix(); let nowSeconds = moment().utc().unix(); return tomorowSeconds - nowSeconds; @@ -575,7 +579,7 @@ export function getStartOfTomorrowInSeconds() { * @returns {string} */ export function getLimitKey() { - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); return moment().utc().format('DD-MM-YYYY'); } @@ -659,8 +663,7 @@ export function autoCastBooleanStrict(value: any) { * Gets a tables hash attribute from the global schema */ export function getTableHashAttribute(schema: string, table: string) { - const { getDatabases } = require('../resources/databases'); - let tableObj = getDatabases()[schema]?.[table]; + let tableObj = getDatabases()[schema]?.[table] as any; return tableObj?.primaryKey || tableObj?.hash_attribute; } @@ -670,7 +673,6 @@ export function getTableHashAttribute(schema: string, table: string) { * @returns {boolean} - returns true if schema exists */ export function doesSchemaExist(schema: string) { - const { getDatabases } = require('../resources/databases'); return getDatabases()[schema] !== undefined; } @@ -681,7 +683,6 @@ export function doesSchemaExist(schema: string) { * @returns {boolean} - returns true if table exists */ export function doesTableExist(schema: string, table: string) { - const { getDatabases } = require('../resources/databases'); return getDatabases()[schema]?.[table] !== undefined; } @@ -704,7 +705,7 @@ export function stringifyObj(value: any) { * @returns {*} */ export function ms_to_time(ms: number) { - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); const duration = moment.duration(ms); const sec = duration.seconds() > 0 ? duration.seconds() + 's' : ''; const min = duration.minutes() > 0 ? duration.minutes() + 'm ' : ''; diff --git a/utility/databasesRef.ts b/utility/databasesRef.ts new file mode 100644 index 0000000000..f850f06cca --- /dev/null +++ b/utility/databasesRef.ts @@ -0,0 +1,22 @@ +'use strict'; +/** + * Tiny registry that breaks the common_utils ↔ databases circular dependency. + * + * databases.ts and its transitive dependencies (Table.ts, auditStore.ts, etc.) + * import common_utils.ts. common_utils.ts previously used lazy `require()` calls + * to avoid the cycle; in ESM (Node 24 type-strip) `require` is unavailable. + * + * This module has no dependencies of its own. databases.ts calls + * `setDatabasesGetter` once after it initialises, and common_utils.ts calls + * `getDatabases()` lazily at runtime — after full startup the getter is always set. + */ + +let _getter: (() => any) | undefined; + +export function setDatabasesGetter(fn: () => any): void { + _getter = fn; +} + +export function getDatabases(): any { + return _getter?.() ?? {}; +} diff --git a/utility/environment/environmentManager.ts b/utility/environment/environmentManager.ts index 9111a41046..8b6969a9c8 100644 --- a/utility/environment/environmentManager.ts +++ b/utility/environment/environmentManager.ts @@ -1,8 +1,9 @@ -'use strict'; +import { PACKAGE_ROOT } from '../packageUtils.js'; +('use strict'); -import * as fs from 'fs-extra'; -import * as path from 'path'; -import * as os from 'os'; +import fs from 'fs-extra'; +import * as path from 'node:path'; +import * as os from 'node:os'; import PropertiesReader from 'properties-reader'; import log from '../logging/harper_logger.ts'; import * as commonUtils from '../common_utils.ts'; @@ -44,7 +45,7 @@ let inheritedOverridesApplied = false; * currently known base path here to help with this case. */ export function getHdbBasePath() { - return installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]; + return installProps?.[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]; } /** @@ -62,11 +63,18 @@ export function setHdbBasePath(hdbPath: string) { * @returns {*} */ export function get(propName: string): any { - const value = configUtils.getConfigValue(propName); + // Tolerate calls before this module's body has executed (ESM cycle): + // configUtils.ts / installProps may not be initialized yet, in which case + // no config has been read so the value is genuinely unknown. + let value; + try { + value = configUtils.getConfigValue(propName); + } catch (err: any) { + if (err?.name !== 'ReferenceError') throw err; + } if (value === undefined) { - return installProps[propName]; + return installProps?.[propName]; } - return value; } @@ -266,7 +274,13 @@ export function initSync(force: boolean = false) { installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY] = configHdbRoot; } } - } catch (err) { + } catch (err: any) { + // During typestrip ESM evaluation, module-load callers of initSync may + // reach this before configUtils has finished its own top-level evaluation, + // producing a ReferenceError (TDZ) on a module-scope binding. Don't exit + // the process for that — the same module-load chain will retry once + // evaluation completes, or bin/harper.ts will re-call initSync(). + if (err?.name === 'ReferenceError') return; log.error(INIT_ERR); log.error(err); console.error(err); @@ -294,8 +308,7 @@ export function initTestEnvironment(testConfigObj: any = {}) { cors_accesslist, local_studio_on, } = testConfigObj; - // __dirname is dist/utility/environment when running tests, so go up 3 levels to reach project root - const propsPath = path.join(__dirname, '../../../', 'unitTests'); + const propsPath = path.join(PACKAGE_ROOT, 'unitTests'); installProps[BOOT_PROPS_FILE_PATH] = path.join(propsPath, 'hdb_boot_properties.file'); const TEST_HDB_PATH = path.join(propsPath, 'envDir', process.pid.toString()); try { diff --git a/utility/environment/systemInformation.ts b/utility/environment/systemInformation.ts index c451e5a8d8..f845deb084 100644 --- a/utility/environment/systemInformation.ts +++ b/utility/environment/systemInformation.ts @@ -1,3 +1,4 @@ +import { loadNativePackage } from '../packageUtils.js'; import { readFile } from 'node:fs/promises'; import path from 'node:path'; import type { Systeminformation } from 'systeminformation'; @@ -5,16 +6,18 @@ import logger from '../logging/harper_logger.ts'; import * as hdbTerms from '../hdbTerms.ts'; import { getQuotaStatus } from '../../server/storageReclamation.ts'; import { lmdbGetTableSize } from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts'; -import { getThreadInfo } from '../../server/threads/manageThreads.js'; +import { getThreadInfo } from '../../server/threads/manageThreads.ts'; import * as env from './environmentManager.ts'; import { getDatabases, type Table } from '../../resources/databases.ts'; import { TableSizeObject } from '../../dataLayer/harperBridge/TableSizeObject.ts'; -import { RocksDatabase, StatsHistogramData } from '@harperfast/rocksdb-js'; - -const loadSi = () => require('systeminformation') as typeof import('systeminformation'); - -env.initSync(); +import { RocksDatabase, type StatsHistogramData } from '@harperfast/rocksdb-js'; +const loadSi = () => loadNativePackage('systeminformation') as typeof import('systeminformation'); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} //this will hold the system_information which is static to improve performance let systemInformationCache = undefined; @@ -703,7 +706,7 @@ const attributeMap: Record Promise | any> = { harperdb_processes: getHDBProcessInfo, table_size: getTableSize, metrics: getMetrics, - threads: getThreadInfo, + threads: (...args) => getThreadInfo(...args), }; /** diff --git a/utility/functions/geo.js b/utility/functions/geo.ts similarity index 85% rename from utility/functions/geo.js rename to utility/functions/geo.ts index 11df524c2d..4e1c27c46c 100644 --- a/utility/functions/geo.js +++ b/utility/functions/geo.ts @@ -1,5 +1,3 @@ -'use strict'; - /*** * geo.js * @@ -7,20 +5,20 @@ * turf.js has very robust internal validation as such we offload the validation to turf.js */ -const turfArea = require('@turf/area'); -const turfLength = require('@turf/length'); -const turfCircle = require('@turf/circle'); -const turfDifference = require('@turf/difference'); -const turfDistance = require('@turf/distance'); -const turfBooleanContains = require('@turf/boolean-contains'); -const turfBooleanEqual = require('@turf/boolean-equal'); -const turfBooleanDisjoint = require('@turf/boolean-disjoint'); -const turfHelpers = require('@turf/helpers'); -const hdbTerms = require('../hdbTerms.ts'); -const commonUtils = require('../common_utils.ts'); -const hdbLog = require('../logging/harper_logger.ts'); - -module.exports = { +import turfArea from '@turf/area'; +import turfLength from '@turf/length'; +import turfCircle from '@turf/circle'; +import turfDifference from '@turf/difference'; +import turfDistance from '@turf/distance'; +import turfBooleanContains from '@turf/boolean-contains'; +import turfBooleanEqual from '@turf/boolean-equal'; +import turfBooleanDisjoint from '@turf/boolean-disjoint'; +import * as turfHelpers from '@turf/helpers'; +import * as hdbTerms from '../hdbTerms.ts'; +import * as commonUtils from '../common_utils.ts'; +import hdbLog from '../logging/harper_logger.ts'; + +export { geoArea, geoLength, geoCircle, @@ -32,7 +30,6 @@ module.exports = { geoCrosses, geoConvert, }; - /*** * Takes one or more features and returns the area in square meters * @param geoJSON @@ -47,7 +44,7 @@ function geoArea(geoJSON) { geoJSON = commonUtils.autoCastJSON(geoJSON); } try { - return turfArea.default(geoJSON); + return turfArea(geoJSON); } catch (err) { hdbLog.trace(err, geoJSON); return NaN; @@ -70,7 +67,7 @@ function geoLength(geoJSON, units) { } try { - return turfLength.default(geoJSON, { units: units ? units : 'kilometers' }); + return turfLength(geoJSON, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, geoJSON); return NaN; @@ -98,7 +95,7 @@ function geoCircle(point, radius, units) { } try { - return turfCircle.default(point, radius, { units: units ? units : 'kilometers' }); + return turfCircle(point, radius, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, point, radius); return NaN; @@ -160,7 +157,7 @@ function geoDistance(point1, point2, units) { } try { - return turfDistance.default(point1, point2, { units: units ? units : 'kilometers' }); + return turfDistance(point1, point2, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, point1, point2); return NaN; @@ -239,7 +236,7 @@ function geoContains(geo1, geo2) { } try { - return turfBooleanContains.default(geo1, geo2); + return turfBooleanContains(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; @@ -277,7 +274,7 @@ function geoEqual(geo1, geo2) { } try { - return turfBooleanEqual.default(geo1, geo2); + return turfBooleanEqual(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; @@ -316,7 +313,7 @@ function geoCrosses(geo1, geo2) { try { //need to do ! as this checks for non-intersections of geometries - return !turfBooleanDisjoint.default(geo1, geo2); + return !turfBooleanDisjoint(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; diff --git a/utility/functions/sql/alaSQLExtension.js b/utility/functions/sql/alaSQLExtension.ts similarity index 59% rename from utility/functions/sql/alaSQLExtension.js rename to utility/functions/sql/alaSQLExtension.ts index acb59b5b59..15cb65d049 100644 --- a/utility/functions/sql/alaSQLExtension.js +++ b/utility/functions/sql/alaSQLExtension.ts @@ -1,51 +1,27 @@ -'use strict'; - /*** * alaSQLExtension.js * purpose of this module is to hold custom functions for alasql */ -const isEqual = require('lodash/isEqual'); -const uniqWith = require('lodash/uniqWith'); -const mathjs = require('mathjs'); -const jsonata = require('jsonata'); -const hdbUtils = require('../../common_utils.ts'); - -module.exports = { - /*** - * distinctArray takes in an array an dedupes its values using lodash. this works on complex as well as simple datatypes - * @param array - * @returns array - */ - distinct_array: (array) => { - if (Array.isArray(array) && array.length > 1) { - return uniqWith(array, isEqual); - } +import isEqual from 'lodash/isEqual.js'; +import uniqWith from 'lodash/uniqWith.js'; +import * as mathjs from 'mathjs'; +import jsonata from 'jsonata'; +import * as hdbUtils from '../../common_utils.ts'; +export const distinct_array = (array) => { + if (Array.isArray(array) && array.length > 1) { + return uniqWith(array, isEqual); + } - return array; - }, - searchJSON, - /*** - * median absolute deviation aggregate function based on http://mathjs.org/docs/reference/functions/mad.html - */ - mad: aggregateFunction.bind(null, mathjs.mad), - /*** - * mean aggregate function based on http://mathjs.org/docs/reference/functions/mean.html - */ - mean: aggregateFunction.bind(null, mathjs.mean), - /*** - * computes the mode of values on http://mathjs.org/docs/reference/functions/mode.html - */ - mode: aggregateFunction.bind(null, mathjs.mode), - /*** - * compute the product based on http://mathjs.org/docs/reference/functions/prod.html - */ - prod: aggregateFunction.bind(null, mathjs.prod), - /*** - * compute the median based on http://mathjs.org/docs/reference/functions/median.html - */ - median: aggregateFunction.bind(null, mathjs.median), + return array; }; +export const mad = aggregateFunction.bind(null, mathjs.mad); +export const mean = aggregateFunction.bind(null, mathjs.mean); +export const mode = aggregateFunction.bind(null, mathjs.mode); +export const prod = aggregateFunction.bind(null, mathjs.prod); +export const median = aggregateFunction.bind(null, mathjs.median); +export { searchJSON }; +export default { distinct_array, searchJSON, mad, mean, mode, prod, median }; /*** * handles the 3 pass loop for aggregates and executes the final calc with the passed in aggregator function diff --git a/utility/globalSchema.ts b/utility/globalSchema.ts index 3a21f225b9..2569e5ec9f 100644 --- a/utility/globalSchema.ts +++ b/utility/globalSchema.ts @@ -1,5 +1,5 @@ -import systemSchema from '../json/systemSchema.json'; -import { promisify } from 'util'; +import { systemSchema } from './systemSchema.js'; +import { promisify } from 'node:util'; import { getDatabases } from '../resources/databases.ts'; export const setSchemaDataToGlobalAsync = promisify(setSchemaDataToGlobal); diff --git a/utility/install/checkJWTTokensExist.js b/utility/install/checkJWTTokensExist.ts similarity index 69% rename from utility/install/checkJWTTokensExist.js rename to utility/install/checkJWTTokensExist.ts index ecc437876c..ffd48ed4db 100644 --- a/utility/install/checkJWTTokensExist.js +++ b/utility/install/checkJWTTokensExist.ts @@ -1,14 +1,20 @@ -'use strict'; +import * as env from '../../utility/environment/environmentManager.ts'; +import fs from 'fs-extra'; +import path from 'node:path'; +import * as terms from '../../utility/hdbTerms.ts'; +import crypto from 'node:crypto'; +import { v4 as uuid } from 'uuid'; -const env = require('../../utility/environment/environmentManager.ts'); -env.initSync(); -const fs = require('fs-extra'); -const path = require('path'); -const terms = require('../../utility/hdbTerms.ts'); -const crypto = require('crypto'); -const uuid = require('uuid').v4; +export default checkJWTTokenExist; -module.exports = checkJWTTokenExist; +// Preserve the CJS-callable shape consumers/tests expect — under tsc CJS emit +// this replaces the `exports` object with the function itself, so +// `require('./checkJWTTokenExist')(...)` works. No-op under typestrip ESM. +declare const module: any; +if (typeof module !== 'undefined') { + module.exports = checkJWTTokenExist; + module.exports.default = checkJWTTokenExist; +} /** * checks that the RSA keys exist for JWT generation, if not we create them */ diff --git a/utility/install/installer.ts b/utility/install/installer.ts index ebb06990db..8e414dab74 100644 --- a/utility/install/installer.ts +++ b/utility/install/installer.ts @@ -1,11 +1,12 @@ 'use strict'; -import * as os from 'os'; +import * as os from 'node:os'; import { prompts } from '../interactivePrompts.ts'; -import * as fs from 'fs-extra'; -import PropertiesReader from 'properties-reader'; +import fs from 'fs-extra'; +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; import chalk from 'chalk'; -import * as path from 'path'; +import * as path from 'node:path'; let ora; // Will be loaded dynamically as it's an ES module import * as YAML from 'yaml'; @@ -17,15 +18,18 @@ import * as hdbInfoController from '../../dataLayer/hdbInfoController.ts'; import { packageJson } from '../packageUtils.js'; import * as hdbTerms from '../hdbTerms.ts'; const { CONFIG_PARAMS } = hdbTerms; -import installValidator from '../../validation/installValidator.ts'; -import mountHdb from '../mount_hdb.ts'; +import _installValidator from '../../validation/installValidator.ts'; +const installValidator = _installValidator; +import _mountHdb from '../mount_hdb.ts'; +const mountHdb = _mountHdb; import { composeConfigFromEnv } from '../../config/harperConfigEnvVars.ts'; import * as configUtils from '../../config/configUtils.ts'; import * as userOps from '../../security/user.ts'; import * as roleOps from '../../security/role.ts'; -import checkJwtTokens from './checkJWTTokensExist.js'; +import _checkJwtTokens from './checkJWTTokensExist.ts'; +const checkJwtTokens = _checkJwtTokens; import * as globalSchema from '../globalSchema.ts'; -import { promisify } from 'util'; +import { promisify } from 'node:util'; const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal); import * as keys from '../../security/keys.ts'; import { resolveConfiguredPath } from '../../config/componentEnvPrepass.ts'; diff --git a/utility/lifecycle.ts b/utility/lifecycle.ts new file mode 100644 index 0000000000..2daf9c1c11 --- /dev/null +++ b/utility/lifecycle.ts @@ -0,0 +1,61 @@ +// Entry points await this phase after configuration initialization and before accepting requests. + +type StartupCallback = () => void | Promise; + +let callbacks: StartupCallback[] = []; +let started = false; +let completed = false; +let runningPromise: Promise | null = null; + +/** + * Register a callback to be run during the startup phase. If startup has + * completed, the callback is invoked on the next microtask. + */ +export function onStartup(cb: StartupCallback): void { + if (completed) { + Promise.resolve().then(cb); + return; + } + callbacks.push(cb); +} + +/** + * Run all registered startup callbacks in registration order. Idempotent: + * subsequent calls return the same promise as the first invocation. + */ +export function runStartup(): Promise { + if (runningPromise) return runningPromise; + let resolveStartup: () => void; + let rejectStartup: (error: unknown) => void; + const startup = (runningPromise = new Promise((resolve, reject) => { + resolveStartup = resolve; + rejectStartup = reject; + })); + const pending = callbacks; + started = true; + (async () => { + try { + while (callbacks === pending && pending.length) await pending.shift()!(); + } finally { + pending.length = 0; + if (callbacks === pending) completed = true; + } + })().then(resolveStartup!, rejectStartup!); + return startup; +} + +/** + * Reset startup state. Intended for unit tests that want to re-run startup + * (e.g. between describe blocks). Production code should never call this. + */ +export function resetStartupForTests(): void { + callbacks = []; + started = false; + completed = false; + runningPromise = null; +} + +/** True once `runStartup()` has begun. */ +export function hasStarted(): boolean { + return started; +} diff --git a/utility/lmdb/OpenDBIObject.ts b/utility/lmdb/OpenDBIObject.ts index 39f5eda517..449db4f57c 100644 --- a/utility/lmdb/OpenDBIObject.ts +++ b/utility/lmdb/OpenDBIObject.ts @@ -2,8 +2,11 @@ import * as envMngr from '../environment/environmentManager.ts'; import * as terms from '../../utility/hdbTerms.ts'; import { RecordEncoder } from '../../resources/RecordEncoder.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_CACHING = envMngr.get(terms.CONFIG_PARAMS.STORAGE_CACHING) !== false; /** diff --git a/utility/lmdb/OpenEnvironmentObject.ts b/utility/lmdb/OpenEnvironmentObject.ts index c14583108c..5c800983b8 100644 --- a/utility/lmdb/OpenEnvironmentObject.ts +++ b/utility/lmdb/OpenEnvironmentObject.ts @@ -7,8 +7,11 @@ const MAX_DBS = 10000; const MAX_READERS = 2048; import * as envMngr from '../environment/environmentManager.ts'; import * as terms from '../../utility/hdbTerms.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} export default class OpenEnvironmentObject { [key: string]: any; static MAX_DBS = MAX_DBS; diff --git a/utility/lmdb/environmentUtility.ts b/utility/lmdb/environmentUtility.ts index af62415b07..8956fa1e74 100644 --- a/utility/lmdb/environmentUtility.ts +++ b/utility/lmdb/environmentUtility.ts @@ -1,8 +1,9 @@ -'use strict'; +import { loadNativePackage } from '../packageUtils.js'; +('use strict'); import type * as lmdb from 'lmdb'; -import * as fs from 'fs-extra'; -import * as path from 'path'; +import fs from 'fs-extra'; +import * as path from 'node:path'; import * as common from './commonUtility.ts'; import log from '../logging/harper_logger.ts'; import { LMDB_ERRORS_ENUM as LMDB_ERRORS } from '../errors/commonErrors.ts'; @@ -122,7 +123,7 @@ export async function createEnvironment( let environmentPath = path.join(basePath, envName); await fs.mkdirp(isV3 ? environmentPath : basePath); let envInit = new OpenEnvironmentObject(isV3 ? environmentPath : environmentPath + MDB_FILE_EXTENSION, false); - let env = (require('lmdb') as typeof lmdb).open(envInit); + let env = (loadNativePackage('lmdb') as typeof lmdb).open(envInit); (env as any).dbis = Object.create(null); //next we create an internal dbi to track the named databases @@ -166,7 +167,7 @@ export async function openEnvironment(basePath: string, envName: string, isTxn: let standardPath = path.join(basePath, envName + MDB_FILE_EXTENSION); let readOnly = envPath != standardPath; // legacy database, only open in read only mode let envInit = new OpenEnvironmentObject(envPath, readOnly); - let env = (require('lmdb') as typeof lmdb).open(envInit); + let env = (loadNativePackage('lmdb') as typeof lmdb).open(envInit); (env as any).dbis = Object.create(null); diff --git a/utility/lmdb/writeUtility.ts b/utility/lmdb/writeUtility.ts index bd29bc6ca5..e18e08e330 100644 --- a/utility/lmdb/writeUtility.ts +++ b/utility/lmdb/writeUtility.ts @@ -13,8 +13,11 @@ import { v4 as uuidv4 } from 'uuid'; import * as lmdb from 'lmdb'; import { handleHDBError, hdbErrors } from '../errors/hdbError.ts'; import * as envMngr from '../environment/environmentManager.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms.CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const CREATED_TIME_ATTRIBUTE_NAME = hdbTerms.TIME_STAMP_NAMES_ENUM.CREATED_TIME; diff --git a/utility/logging/harper_logger.ts b/utility/logging/harper_logger.ts index 56b184ff99..eeb008142f 100644 --- a/utility/logging/harper_logger.ts +++ b/utility/logging/harper_logger.ts @@ -1,17 +1,19 @@ -'use strict'; +const segfaultHandlerPackage = 'segfault-handler'; +('use strict'); // Note - do not import/use commonUtils.js in this module, it will cause circular dependencies. -import * as fs from 'fs-extra'; -import { workerData, threadId, isMainThread } from 'worker_threads'; -import * as pathModule from 'path'; +import fs from 'fs-extra'; +import { workerData, threadId, isMainThread } from 'node:worker_threads'; +import * as pathModule from 'node:path'; import * as YAML from 'yaml'; -const PropertiesReader = require('properties-reader'); +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; import * as hdbTerms from '../hdbTerms.ts'; import assignCMDENVVariables from '../assignCmdEnvVariables.ts'; -import * as os from 'os'; -import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; +import * as os from 'node:os'; +import { PACKAGE_ROOT, loadRuntimeModule } from '../../utility/packageUtils.js'; import { _assignPackageExport } from '../../globals.js'; -import { Console } from 'console'; +import { Console } from 'node:console'; import { inspect, types } from 'node:util'; import { createRotationGuard, INVALID_MAX_SIZE_MSG, parseMaxSize, resolveRotatedLogDir } from './logRotation.ts'; import { type FileIdentity, registerLogSink } from './logGenerationCoordinator.ts'; @@ -27,7 +29,6 @@ let nativeStdWrite = process.env.IS_SCRIPTED_SERVICE : (process.stdout as any).nativeWrite || ((process.stdout as any).nativeWrite = process.stdout.write); let fileLoggers = new Map(); const { join } = pathModule; - const MAX_LOG_BUFFER = 10000; const LOG_LEVEL_HIERARCHY = { notify: 7, @@ -200,6 +201,9 @@ function resolveLogPath(configPath: string, rootPath: string) { } async function updateLogSettings() { if (!rootConfig) { + // Lazy-load to avoid a circular dependency at module evaluation time + // (RootConfigWatcher imports configUtils which imports this logger). + const { getSharedRootConfigWatcher } = await import('../../config/RootConfigWatcher.ts'); // set up the initial watcher rootConfig = getSharedRootConfigWatcher(); // wait for it to be ready @@ -420,44 +424,46 @@ class HarperLogger extends Console { if (hdbProperties === undefined) initLogSettings(); -module.exports = { - notify, - fatal, - error, - warn, - info, - debug, - trace, - logLevel, - loggerWithTag, - suppressLogging, - initLogSettings, - logCustomLevel, - closeLogFile, - createLogger, - logsAtLevel, - getLogFilePath: () => logFilePath, - forComponent: (name, isExternal) => mainLogger.forComponent(name, isExternal), - setMainLogger, - setLogLevel, - OUTPUTS, - CREDENTIAL_KEY_PATTERN, - AuthAuditLog, - // for now these functions at least notify us of when the component system is ready so - // we can start using the RootConfigWatcher - start: updateLogSettings, - startOnMainThread: updateLogSettings, - // Test-only: applies a config without the process-wide watcher `updateLogSettings` builds. - _applyLogSettingsForTests: applyLogSettings, - errorToString, - errorForLog, - inspectForLog, - isErrorLike, - disableStdio, - isStdioBrokenError, - externalLogger, - updateLogger, -}; +if (typeof module !== 'undefined') { + module.exports = { + notify, + fatal, + error, + warn, + info, + debug, + trace, + logLevel, + loggerWithTag, + suppressLogging, + initLogSettings, + logCustomLevel, + closeLogFile, + createLogger, + logsAtLevel, + getLogFilePath: () => logFilePath, + forComponent: (name, isExternal) => mainLogger.forComponent(name, isExternal), + setMainLogger, + setLogLevel, + OUTPUTS, + CREDENTIAL_KEY_PATTERN, + AuthAuditLog, + // for now these functions at least notify us of when the component system is ready so + // we can start using the RootConfigWatcher + start: updateLogSettings, + startOnMainThread: updateLogSettings, + // Test-only: applies a config without the process-wide watcher `updateLogSettings` builds. + _applyLogSettingsForTests: applyLogSettings, + errorToString, + errorForLog, + inspectForLog, + isErrorLike, + disableStdio, + isStdioBrokenError, + externalLogger, + updateLogger, + }; +} // Writes past the stdio guard installed by installStdioGuard, which would otherwise route the // text back into the file logger this is the fallback for, and swallows a broken pipe - there is @@ -568,13 +574,16 @@ export function initLogSettings(forceInit = false) { externalLogger = mainLogger.forComponent('external'); externalLogger.tag = null; // don't tag by default if (isMainThread && typeof globalThis.Bun === 'undefined') { - // Bun will crash with the segfault handler, ironically - try { - const SegfaultHandler = require('segfault-handler'); - SegfaultHandler.registerHandler(join(logRoot, 'crash.log')); - } catch { - // optional dependency, ok if we can't run it - } + // Bun will crash with the segfault handler, ironically. + // Dynamic import works in both CJS and ESM (type-strip); require() is not + // available as a global in ESM so we can't use it here. + import(segfaultHandlerPackage) + .then((mod) => { + (mod.default ?? mod).registerHandler(join(logRoot, 'crash.log')); + }) + .catch(() => { + // optional dependency, ok if unavailable + }); } } } catch (err) { @@ -667,23 +676,25 @@ function stdioLogging() { installStdioGuard(process.stderr); } -export function loggerWithTag(tag: string, conditional?: boolean, logger: any = mainLogger) { +export function loggerWithTag(tag: string, conditional?: boolean, logger?: any) { + const initialLogger = logger === undefined ? mainLogger : logger; tag = tag.replace(/ /g, '-'); // tag can't have spaces return { - notify: logWithTag(logger.notify, 'notify'), - fatal: logWithTag(logger.fatal, 'fatal'), - error: logWithTag(logger.error, 'error'), - warn: logWithTag(logger.warn, 'warn'), - info: logWithTag(logger.info, 'info'), - debug: logWithTag(logger.debug, 'debug'), - trace: logWithTag(logger.trace, 'trace'), + notify: logWithTag(initialLogger.notify, 'notify'), + fatal: logWithTag(initialLogger.fatal, 'fatal'), + error: logWithTag(initialLogger.error, 'error'), + warn: logWithTag(initialLogger.warn, 'warn'), + info: logWithTag(initialLogger.info, 'info'), + debug: logWithTag(initialLogger.debug, 'debug'), + trace: logWithTag(initialLogger.trace, 'trace'), }; function logWithTag(loggerMethod, level) { - return !conditional || logger.level <= LOG_LEVEL_HIERARCHY[level] + return !conditional || initialLogger.level <= LOG_LEVEL_HIERARCHY[level] ? function (...args) { + const target = logger === undefined ? mainLogger : logger; currentTag = tag; try { - return loggerMethod.call(logger, ...args); + return (logger === undefined ? target[level] : loggerMethod).call(target, ...args); } finally { currentTag = undefined; } @@ -915,14 +926,14 @@ function getFileLogger(path, rotation, isExternalInstance, rotationPolicy) { if (isMainThread && reconfigured) { setTimeout(() => { // Everything inside the try: a throw from a timer callback is unhandled, and neither - // require('./logRotator') (which reaches environmentManager's synchronous init) nor a + // loading logRotator.ts (which reaches environmentManager's synchronous init) nor a // rotator teardown may take the process down over log rotation (#847). try { const previousRotator = logger.rotator; logger.rotator = undefined; previousRotator?.end(); if (!rotation) return; - const { logRotator } = require('./logRotator'); + const { logRotator } = loadRuntimeModule('utility/logging/logRotator'); logger.rotator = logRotator({ logger, ...rotation, @@ -1199,7 +1210,7 @@ function getLogConfig(hdbConfigPath) { const oldHdbSettings = PropertiesReader(hdbConfigPath); return { level: oldHdbSettings.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_LEVEL_KEY), - configLogPath: pathModule.dirname(oldHdbSettings.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY)), + configLogPath: pathModule.dirname(oldHdbSettings.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY) as string), toFile: oldHdbSettings.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_TO_FILE), toStream: oldHdbSettings.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_TO_STDSTREAMS), }; @@ -2313,7 +2324,6 @@ export function AuthAuditLog( this.path = path; } // we have to load this at the end to avoid circular dependencies problems -import { getSharedRootConfigWatcher } from '../../config/RootConfigWatcher.ts'; export const getLogFilePath = () => logFilePath; export const forComponent = (name: string, isExternal?: boolean) => mainLogger.forComponent(name, isExternal); @@ -2350,4 +2360,6 @@ export default { inspectForLog, isErrorLike, updateLogger, + start: updateLogSettings, + startOnMainThread: updateLogSettings, }; diff --git a/utility/logging/logRotator.ts b/utility/logging/logRotator.ts index f499a2273c..ad2393c902 100644 --- a/utility/logging/logRotator.ts +++ b/utility/logging/logRotator.ts @@ -1,9 +1,13 @@ 'use strict'; -import { type BigIntStats, existsSync, mkdirSync, statSync, promises as fsProm } from 'fs'; -import * as path from 'path'; +import { type BigIntStats, existsSync, mkdirSync, statSync, promises as fsProm } from 'node:fs'; +import * as path from 'node:path'; import * as envMgr from '../environment/environmentManager.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} import hdbLogger from './harper_logger.ts'; import { CONFIG_PARAMS } from '../hdbTerms.ts'; import { convertToMS } from '../common_utils.ts'; diff --git a/utility/logging/logger.ts b/utility/logging/logger.ts index 2603d6142c..4414b6d034 100644 --- a/utility/logging/logger.ts +++ b/utility/logging/logger.ts @@ -1,14 +1,20 @@ +import { onStartup } from '../lifecycle.ts'; /** Like harperLogger, but conditionally exports functions based on the log level. */ import harperLogger from './harper_logger.ts'; export const logger: Logger = {}; -for (let level of ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'notify']) { - if (harperLogger.logsAtLevel(level)) { - logger[level] = harperLogger[level]; +function initializeLogger() { + for (let level of ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'notify']) { + if (harperLogger.logsAtLevel(level)) { + logger[level] = harperLogger[level]; + } } } +initializeLogger(); +if (typeof module === 'undefined') onStartup(initializeLogger); + export function loggerWithTag(tag: string): Logger { return harperLogger.loggerWithTag(tag, true) as Logger; } diff --git a/utility/logging/readLog.ts b/utility/logging/readLog.ts index 51c387aaaf..011d6ebcad 100644 --- a/utility/logging/readLog.ts +++ b/utility/logging/readLog.ts @@ -3,13 +3,13 @@ import * as hdbTerms from '../hdbTerms.ts'; import hdbLogger from './harper_logger.ts'; import validator from '../../validation/readLogValidator.ts'; -import * as path from 'path'; -import * as fs from 'fs-extra'; -import { once } from 'events'; +import * as path from 'node:path'; +import fs from 'fs-extra'; +import { once } from 'node:events'; import { getConfigPath } from '../../config/configUtils.ts'; import { handleHDBError, hdbErrors } from '../errors/hdbError.ts'; import { server } from '../../server/Server.ts'; -import { StringDecoder } from 'string_decoder'; +import { StringDecoder } from 'node:string_decoder'; const DEFAULT_READ_LOG_LIMIT = 1000; const ESTIMATED_AVERAGE_ENTRY_SIZE = 200; diff --git a/utility/logging/transactionLog.ts b/utility/logging/transactionLog.ts index 2d9bb66f40..46bbeafec3 100644 --- a/utility/logging/transactionLog.ts +++ b/utility/logging/transactionLog.ts @@ -9,7 +9,7 @@ import { readTransactionLogValidator, deleteTransactionLogsBeforeValidator, } from '../../validation/transactionLogValidator.ts'; -const harperBridge = require('../../dataLayer/harperBridge/harperBridge').default; +import harperBridge from '../../dataLayer/harperBridge/harperBridge.ts'; export async function readTransactionLog(req: any) { const validation = readTransactionLogValidator(req); diff --git a/utility/mount_hdb.ts b/utility/mount_hdb.ts index f9355ff043..608bebfeb6 100644 --- a/utility/mount_hdb.ts +++ b/utility/mount_hdb.ts @@ -1,13 +1,16 @@ 'use strict'; -const { mkdirpSync, copySync } = require('fs-extra'); -import * as path from 'path'; +import fsExtra from 'fs-extra'; +const { mkdirpSync, copySync } = fsExtra; +import * as path from 'node:path'; import * as terms from '../utility/hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import bridge from '../dataLayer/harperBridge/harperBridge.ts'; -import systemSchema from '../json/systemSchema.json'; -import * as initPaths from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +import CreateTableObject from '../dataLayer/CreateTableObject.ts'; +import * as initPaths from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; +import { systemSchema as builtinSystemSchema } from './systemSchema.js'; +const systemSchema: Record = builtinSystemSchema; export default async function mountHdb(hdbPath: string) { hdbLogger.trace('Mounting Harper'); @@ -28,9 +31,6 @@ export default async function mountHdb(hdbPath: string) { * @returns {Promise} */ async function createTables() { - const CreateTableObject = - require('../dataLayer/CreateTableObject').default || require('../dataLayer/CreateTableObject'); - let tables = Object.keys(systemSchema); for (const tableName of tables) { diff --git a/utility/npmUtilities.ts b/utility/npmUtilities.ts index 5f0a431242..01356d0d8c 100644 --- a/utility/npmUtilities.ts +++ b/utility/npmUtilities.ts @@ -1,12 +1,11 @@ 'use strict'; import Joi from 'joi'; -import * as path from 'path'; +import * as path from 'node:path'; import { handleHDBError, hdbErrors } from './errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - import * as validator from '../validation/validationWrapper.ts'; import harperLogger from './logging/harper_logger.ts'; @@ -14,7 +13,7 @@ import { CONFIG_PARAMS } from './hdbTerms.ts'; import { getConfigPath } from '../config/configUtils.ts'; import { nonInteractiveSpawn, packageManagerInstallArguments } from '../components/Application.ts'; import { withComponentPreparationLock } from '../components/componentPreparationLock.ts'; -import { isThreadRunning } from '../server/threads/manageThreads.js'; +import { isThreadRunning } from '../server/threads/manageThreads.ts'; /** * Executes npm install against specified custom function projects diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index a191a648ce..f6c2256977 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -1,4 +1,4 @@ -'use strict'; +import { loadRuntimeModule, loadNativePackage } from './packageUtils.js'; /** * This module is used before a SQL or NoSQL operation is performed in order to ensure the user's assigned role * has the permissions and lack of restrictions needed to process the operation. Only verifyPerms and verifyPermsAST @@ -31,12 +31,12 @@ import { registerGrantableOperation, unregisterGrantableOperation, } from './operationPermissions.ts'; -import * as permsTranslator from '../security/permissionsTranslator.js'; +import * as permsTranslator from '../security/permissionsTranslator.ts'; import { systemInformation } from '../utility/environment/systemInformation.ts'; import * as tokenAuthentication from '../security/tokenAuthentication.ts'; import * as auth from '../security/auth.ts'; import * as configUtils from '../config/configUtils.ts'; -import * as functionsOperations from '../components/operations.js'; +import * as functionsOperations from '../components/operations.ts'; import * as transactionLog from '../utility/logging/transactionLog.ts'; import * as npmUtilities from './npmUtilities.ts'; import * as analytics from '../resources/analytics/read.ts'; @@ -463,15 +463,6 @@ requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.SELECT, new permission(false, [ requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.INSERT, new permission(false, [INSERT_PERM], null)); requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.UPDATE, new permission(false, [UPDATE_PERM], null)); -module.exports = { - verifyPerms, - verifyPermsAST, - verifyOperationsAllowlist, - verifyBulkLoadAttributePerms, - registerOperationPermission, - unregisterOperationPermission, -}; - /** * Verifies permissions and restrictions for a SQL operation based on the user's assigned role. * @param ast - The SQL statement in Syntax Tree form. @@ -548,6 +539,8 @@ function sqlWriteScopeDenial(userObject: any, sqlVariant: string) { return tokenScopeDenial(userObject, sqlVariant); } +let StatementBucket; +let alasql; export function verifyPermsAST(ast, userObject, operation, apiOperation = terms.OPERATIONS_ENUM.SQL) { //TODO - update these validation checks to use validate.js if (commonUtils.isEmptyOrZeroLength(ast)) { @@ -572,12 +565,11 @@ export function verifyPermsAST(ast, userObject, operation, apiOperation = terms. if (scopeDenial) return scopeDenial; try { - const bucketModule = require('../sqlTranslator/sql_statement_bucket'); - const bucket = bucketModule.default || bucketModule; - const alasql = require('alasql'); + StatementBucket ??= loadRuntimeModule('sqlTranslator/sql_statement_bucket').default; + alasql ??= loadNativePackage('alasql'); const permsResponse = new PermissionResponseObject(); - let parsedAst = new bucket(ast); + let parsedAst = new StatementBucket(ast); let schemas = parsedAst.getSchemas(); let schemaTableMap = new Map(); @@ -620,7 +612,7 @@ export function verifyPermsAST(ast, userObject, operation, apiOperation = terms. //If the AST is for a SELECT, we need to check for wildcards and, if they exist, update the AST to include the // attributes that the user has READ perms for - we can skip this step for super users - if (!isSuperUser && ast instanceof alasql.yy.Select) { + if (!isSuperUser && ast instanceof (alasql.yy as any).Select) { ast = parsedAst.updateAttributeWildcardsForRolePerms(fullRolePerms); } diff --git a/utility/packageUtils.js b/utility/packageUtils.js index 75e56ad2a8..d00467526e 100644 --- a/utility/packageUtils.js +++ b/utility/packageUtils.js @@ -1,6 +1,7 @@ 'use strict'; const { join, dirname } = require('node:path'); const { existsSync, readFileSync, realpathSync } = require('node:fs'); +const { pathToFileURL } = require('node:url'); /** * A naive find-up implementation to find the root package.json, and @@ -46,4 +47,36 @@ const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')); */ const PACKAGE_ROOT = realpathSync(dirname(packageJsonPath)); -module.exports = { packageJson, PACKAGE_ROOT }; +/** + * Canonical directory containing core's runtime modules, independent of the + * metadata owner: the source root, standalone dist, or an embedded dist/core. + */ +const RUNTIME_SRC_ROOT = realpathSync(join(__dirname, '..')); + +/** + * Source layouts carry hdbTerms.ts beside this helper; compiled layouts carry + * hdbTerms.js. Use that local marker for core worker and helper filenames. + */ +const RUNTIME_FILE_EXT = existsSync(join(__dirname, 'hdbTerms.ts')) ? '.ts' : '.js'; + +function loadNativePackage(name) { + return require(name); +} + +function loadRuntimeModule(relativePath) { + return require(join(__dirname, '..', relativePath + RUNTIME_FILE_EXT)); +} + +function getRuntimeModuleURL(relativePath) { + return pathToFileURL(join(__dirname, '..', relativePath + RUNTIME_FILE_EXT)).href; +} + +module.exports = { + packageJson, + PACKAGE_ROOT, + RUNTIME_SRC_ROOT, + RUNTIME_FILE_EXT, + loadNativePackage, + loadRuntimeModule, + getRuntimeModuleURL, +}; diff --git a/utility/processManagement/processManagement.js b/utility/processManagement/processManagement.ts similarity index 83% rename from utility/processManagement/processManagement.js rename to utility/processManagement/processManagement.ts index 1816b9f37f..39801b1687 100644 --- a/utility/processManagement/processManagement.js +++ b/utility/processManagement/processManagement.ts @@ -1,18 +1,24 @@ -'use strict'; +import * as hdbTerms from '../hdbTerms.ts'; +import * as servicesConfig from './servicesConfig.ts'; +import * as envMangr from '../environment/environmentManager.ts'; +import hdbLogger from '../../utility/logging/harper_logger.ts'; +import { onMessageFromWorkers } from '../../server/threads/manageThreads.ts'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { execFile, fork } from 'node:child_process'; -const hdbTerms = require('../hdbTerms.ts'); -const servicesConfig = require('./servicesConfig.js'); -const envMangr = require('../environment/environmentManager.ts'); -const hdbLogger = require('../../utility/logging/harper_logger.ts'); -const { onMessageFromWorkers } = require('../../server/threads/manageThreads.js'); -const fs = require('fs'); -const path = require('node:path'); -const { setTimeout: delay } = require('node:timers/promises'); -const { execFile, fork } = require('node:child_process'); +const INIT_PROCESS_NAMES: any = new Set([ + 'catatonit', + 'docker-init', + 'dumb-init', + 'init', + 's6-svscan', + 'systemd', + 'tini', +]); -const INIT_PROCESS_NAMES = new Set(['catatonit', 'docker-init', 'dumb-init', 'init', 's6-svscan', 'systemd', 'tini']); - -module.exports = { +export { start, restart, kill, @@ -27,7 +33,7 @@ onMessageFromWorkers((message) => { if (message.type === 'restart') envMangr.initSync(true); }); -let childProcesses = []; +let childProcesses: any = []; const MAX_RESTARTS = 10; let shuttingDown; /** @@ -35,7 +41,7 @@ let shuttingDown; * @param procConfig * @returns void */ -function start(procConfig, noKill = false) { +function start(procConfig?: any, noKill: any = false) { const args = typeof procConfig.args === 'string' ? procConfig.args.split(' ') : procConfig.args; procConfig.silent = true; procConfig.detached = true; @@ -43,7 +49,7 @@ function start(procConfig, noKill = false) { ...procConfig.env, HARPER_PARENT_PROCESS_PID: process.pid.toString(), }; - const subprocess = procConfig.script + const subprocess: any = procConfig.script ? fork(procConfig.script, args, procConfig) : execFile(procConfig.binFile, args, procConfig); subprocess.name = procConfig.name; @@ -78,7 +84,7 @@ function start(procConfig, noKill = false) { } childProcesses.push(subprocess); } -function cleanupChildrenProcesses(exit = true) { +function cleanupChildrenProcesses(exit: any = true) { if (shuttingDown) return; shuttingDown = true; if (childProcesses.length === 0) return; @@ -93,7 +99,7 @@ function cleanupChildrenProcesses(exit = true) { * @param serviceName * @returns {Promise} */ -function restart(serviceName) { +function restart(serviceName?: any) { expectedRestartOfChildren(); for (let childProcess of childProcesses) { // kill the child process and let it (auto) restart @@ -128,7 +134,7 @@ function getHdbPid() { // return undefined } -function isInitProcess(pid) { +function isInitProcess(pid?: any) { try { const executable = path.basename(fs.readlinkSync(`/proc/${pid}/exe`)).replace(/ \(deleted\)$/, ''); if (INIT_PROCESS_NAMES.has(executable)) return true; @@ -155,7 +161,7 @@ function kill() { * @param serviceName * @returns {Promise} */ -async function startService(serviceName, noKill = false) { +async function startService(serviceName?: any, noKill: any = false) { let startConfig; serviceName = serviceName.toLowerCase(); switch (serviceName) { @@ -173,7 +179,7 @@ async function startService(serviceName, noKill = false) { * @param {string} pidFile - The path to the Harper PID file * @returns {number|null} - The PID as a number, or null if the file is not found or cannot be read */ -function readPidFile(pidFile) { +function readPidFile(pidFile?: any) { try { return Number.parseInt(fs.readFileSync(pidFile, 'utf8'), 10); } catch { @@ -186,7 +192,7 @@ function readPidFile(pidFile) { * @param {number} pid - The process ID to check * @returns {boolean} - True if the process is running, false otherwise */ -function isProcessRunning(pid) { +function isProcessRunning(pid?: any) { try { // process.kill with signal 0 tests if process exists // throws error if process doesn't exist diff --git a/utility/processManagement/servicesConfig.js b/utility/processManagement/servicesConfig.ts similarity index 62% rename from utility/processManagement/servicesConfig.js rename to utility/processManagement/servicesConfig.ts index 0e269b0380..c525df9cf9 100644 --- a/utility/processManagement/servicesConfig.js +++ b/utility/processManagement/servicesConfig.ts @@ -1,11 +1,9 @@ -'use strict'; - -const hdbTerms = require('../hdbTerms.ts'); -const path = require('path'); -const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); -const hdbUtils = require('../common_utils.ts'); -const SCRIPTS_DIR = path.join(PACKAGE_ROOT, 'utility/scripts'); -const RESTART_SCRIPT = path.join(SCRIPTS_DIR, hdbTerms.HDB_RESTART_SCRIPT); +import * as hdbTerms from '../hdbTerms.ts'; +import path from 'node:path'; +import { PACKAGE_ROOT, RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; +import * as hdbUtils from '../common_utils.ts'; +const SCRIPTS_DIR = path.join(RUNTIME_SRC_ROOT, 'utility/scripts'); +const RESTART_SCRIPT = path.join(SCRIPTS_DIR, `restartHdb${RUNTIME_FILE_EXT}`); function generateMainServerConfig() { const envVars = { @@ -17,7 +15,10 @@ function generateMainServerConfig() { return { name: hdbTerms.PROCESS_DESCRIPTORS.HDB, - script: hdbTerms.LAUNCH_SERVICE_SCRIPTS.MAIN, + script: + RUNTIME_FILE_EXT === '.ts' + ? path.join(RUNTIME_SRC_ROOT, 'bin/harper.ts') + : path.join(PACKAGE_ROOT, hdbTerms.LAUNCH_SERVICE_SCRIPTS.MAIN), exec_mode: 'fork', env: envVars, execArgv: process.execArgv, @@ -49,8 +50,4 @@ function generateAllServiceConfigs() { }; } -module.exports = { - generateAllServiceConfigs, - generateMainServerConfig, - generateRestart, -}; +export { generateAllServiceConfigs, generateMainServerConfig, generateRestart }; diff --git a/utility/scripts/restartHdb.js b/utility/scripts/restartHdb.ts similarity index 83% rename from utility/scripts/restartHdb.js rename to utility/scripts/restartHdb.ts index 141702fa61..b95743a814 100644 --- a/utility/scripts/restartHdb.js +++ b/utility/scripts/restartHdb.ts @@ -1,7 +1,6 @@ -'use strict'; - -const pm2Utils = require('../processManagement/processManagement.js'); -const hdbTerms = require('../hdbTerms.ts'); +import * as _pm2Utils from '../processManagement/processManagement.ts'; +const pm2Utils: any = _pm2Utils; +import * as hdbTerms from '../hdbTerms.ts'; /** * Gets a list of all the running Harper processes and calls reload on each one. diff --git a/utility/signalling.ts b/utility/signalling.ts index f69b7c5926..8c0fd5987d 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -1,10 +1,11 @@ -'use strict'; +import * as serverItcHandlersModule from '../server/itc/serverHandlers.ts'; +('use strict'); import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; let serverItcHandlers; -import { sendItcEvent, sendItcEventStrict } from '../server/threads/itc.js'; +import { sendItcEvent, sendItcEventStrict } from '../server/threads/itc.ts'; // Await BOTH the local handler and the cross-worker broadcast. The local handler is what // rebuilds THIS thread's cache; firing it un-awaited let the originating worker return success @@ -23,7 +24,7 @@ export async function signalSchemaChange( ) { try { hdbLogger.debug('signalSchemaChange called with message:', message); - serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + serverItcHandlers = serverItcHandlers || serverItcHandlersModule.default; const itcEventSchema = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, message); if (peersFirst) { await sendItcEvent(itcEventSchema, includeJobWorkers); @@ -63,8 +64,7 @@ export async function signalSchemaChangeToPeers(message: any): Promise { */ export function signalResourcesRegistered() { try { - serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); - serverItcHandlers.resourceHandler(); + serverItcHandlersModule.resourceHandler(); } catch (err) { hdbLogger.error(err); } diff --git a/utility/systemSchema.js b/utility/systemSchema.js new file mode 100644 index 0000000000..6b43010494 --- /dev/null +++ b/utility/systemSchema.js @@ -0,0 +1,4 @@ +'use strict'; + +// CommonJS loads JSON in native TypeStrip and lets the compiler emit it with core. +exports.systemSchema = require('../json/systemSchema.json'); diff --git a/validation/configValidator.ts b/validation/configValidator.ts index d0de356280..cee5d1137d 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -1,11 +1,11 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import Joi from 'joi'; -import * as os from 'os'; +import * as os from 'node:os'; const { boolean, string, number, array } = Joi.types(); -import { totalmem } from 'os'; -import * as path from 'path'; +import { totalmem } from 'node:os'; +import * as path from 'node:path'; import * as hdbLogger from '../utility/logging/harper_logger.ts'; import * as hdbUtils from '../utility/common_utils.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; diff --git a/validation/fileLoadValidator.ts b/validation/fileLoadValidator.ts index b4d11e6c38..cf88595679 100644 --- a/validation/fileLoadValidator.ts +++ b/validation/fileLoadValidator.ts @@ -2,12 +2,11 @@ import clone from 'clone'; import * as validator from './validationWrapper.ts'; import * as commonUtils from '../utility/common_utils.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; -import * as fs from 'fs'; +import * as fs from 'node:fs'; import joi from 'joi'; const { string } = joi.types(); import { hdbErrors, handleHDBError } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - import { commonValidators } from './common_validators.ts'; const isRequiredString = ' is required'; diff --git a/validation/installValidator.ts b/validation/installValidator.ts index 060ed26ba8..15b56d0a59 100644 --- a/validation/installValidator.ts +++ b/validation/installValidator.ts @@ -2,9 +2,9 @@ import Joi from 'joi'; const { string, number } = Joi.types(); -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as hdbTerms from '../utility/hdbTerms.ts'; -import * as path from 'path'; +import * as path from 'node:path'; import { bareHostViolation } from '../utility/nodeIdentity.ts'; import * as validator from './validationWrapper.ts'; diff --git a/validation/readLogValidator.ts b/validation/readLogValidator.ts index bee2f8227a..42b741b37d 100644 --- a/validation/readLogValidator.ts +++ b/validation/readLogValidator.ts @@ -1,9 +1,10 @@ -'use strict'; +import { loadNativePackage } from '../utility/packageUtils.js'; +('use strict'); import Joi from 'joi'; import * as validator from './validationWrapper.ts'; -import * as fs from 'fs-extra'; -import * as path from 'path'; +import fs from 'fs-extra'; +import * as path from 'node:path'; import { getConfigPath } from '../config/configUtils.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import { LOG_LEVELS } from '../utility/hdbTerms.ts'; @@ -35,7 +36,7 @@ const readLogSchema = Joi.object({ }); function validateDatetime(value, helpers) { - const moment = require('moment') as typeof import('moment'); + const moment = loadNativePackage('moment') as typeof import('moment'); if (moment(value, moment.ISO_8601).format(LOG_DATE_FORMAT) === 'Invalid date') { return helpers.message(`'${helpers.state.path[0]}' date '${value}' is invalid.`); } diff --git a/validation/role_validation.ts b/validation/role_validation.ts index 1a02ae3170..3d43549f3d 100644 --- a/validation/role_validation.ts +++ b/validation/role_validation.ts @@ -1,12 +1,12 @@ -const validate = require('validate.js'); -const validator = require('./validationWrapper'); +import validate from 'validate.js'; +import * as _validator from './validationWrapper.ts'; +const validator = _validator; import * as terms from '../utility/hdbTerms.ts'; import { validateOperations } from '../utility/operationPermissions.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - const constraintsTemplate = () => ({ role: { presence: true, diff --git a/validation/schemaMetadataValidator.ts b/validation/schemaMetadataValidator.ts index 28d6486e02..266125237b 100644 --- a/validation/schemaMetadataValidator.ts +++ b/validation/schemaMetadataValidator.ts @@ -1,6 +1,8 @@ 'use strict'; -export const schemaDescribe = require('../dataLayer/schemaDescribe'); +import * as _schemaDescribe from '../dataLayer/schemaDescribe.ts'; +const schemaDescribe = _schemaDescribe; +export { schemaDescribe }; import { hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; diff --git a/validation/searchValidator.ts b/validation/searchValidator.ts index 059cffcad1..f8f0c34dbd 100644 --- a/validation/searchValidator.ts +++ b/validation/searchValidator.ts @@ -121,7 +121,7 @@ export default function (searchObject: any, type: any) { return handleHDBError(new Error(), checkSchemaTable, HTTP_STATUS_CODES.NOT_FOUND); } - let tableSchema = getDatabases()[searchObject.schema][searchObject.table]; + let tableSchema: any = getDatabases()[searchObject.schema][searchObject.table]; let allTableAttributes = tableSchema.attributes; //this clones the get_attributes array diff --git a/validation/validationWrapper.ts b/validation/validationWrapper.ts index c554e58eec..020002946b 100644 --- a/validation/validationWrapper.ts +++ b/validation/validationWrapper.ts @@ -11,7 +11,8 @@ * These are rare enough for it not to be worth creating wrapper functions for those as well. */ -const validate = require('validate.js'); +import _validate from 'validate.js'; +const validate = _validate; //This validator is added here b/c we are still on version 0.11.1 that does not include this build in functionality. When // we do update, we can remove. The reason we have not is related to a breaking change on the "presence" validator rule @@ -74,7 +75,7 @@ export async function validateObjectAsync(object, fileConstraints) { } try { - await validate.async(object, fileConstraints, { format: 'flat' }); + await validate.async(object, fileConstraints, { format: 'flat' } as any); } catch (err) { // unroll the array and make a full error message. let msg = err.join(`,`);