From 100d832bbecdfefd055f90a398f7c697d36832f5 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Fri, 28 Aug 2026 14:37:27 +0200 Subject: [PATCH 1/4] fix: convert lint results per chunk to avoid OOM on large repos lintFilesChunkByChunk kept every chunk's ESLint.LintResult[] (each carrying full file source text) in memory for the whole repo before converting, so memory grew with repo size regardless of chunk size. Convert and discard each chunk's results immediately instead. --- src/engineImpl.ts | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/engineImpl.ts b/src/engineImpl.ts index 968db1317..bc1a9165e 100644 --- a/src/engineImpl.ts +++ b/src/engineImpl.ts @@ -30,40 +30,35 @@ export const engineImpl: Engine = async function ( const eslint = new ESLint(options) // Check if there are any glob patterns in the files array - const lintResults = files.some((file: string) => /\*|\?|\[/.test(file)) - ? await eslint.lintFiles(files) + const results = files.some((file: string) => /\*|\?|\[/.test(file)) + ? convertResults(await eslint.lintFiles(files)) : await lintFilesInChunks(eslint, files) - await debugAndCountLintIssues(eslint, lintResults) + debug(`engine: ${results.length} issues/errors found`) debug("engine: finished") - return convertResults(lintResults).map((r) => r.relativeTo(srcDirPath)) + return results.map((r) => r.relativeTo(srcDirPath)) } -async function lintFilesInChunks (eslint: ESLint, files: string[]): Promise { +async function lintFilesInChunks (eslint: ESLint, files: string[]): Promise { const maxTotalSizePerChunk = 5 * 1024 * 1024; // size in bytes (5MB) const chunksOfFiles = await chunkFilesByTotalSize(files, maxTotalSizePerChunk) return lintFilesChunkByChunk(eslint, chunksOfFiles) } -async function lintFilesChunkByChunk (eslint: ESLint, chunksOfFiles: string[][]): Promise { +async function lintFilesChunkByChunk (eslint: ESLint, chunksOfFiles: string[][]): Promise { debug("engine: linting chunks started") - const lintResults = [] + const results: ToolResult[] = [] + let filesLinted = 0 for (const chunkOfFiles of chunksOfFiles) { const chunkResults = await eslint.lintFiles(chunkOfFiles) - lintResults.push.apply(lintResults, chunkResults) + filesLinted += chunkResults.length + results.push(...convertResults(chunkResults)) } - debug("engine: linting chunks finished") + debug(`engine: linting chunks finished, ${filesLinted} files linted`) - return lintResults -} - -async function debugAndCountLintIssues (eslint: ESLint, lintResults: ESLint.LintResult[]): Promise { - if (!DEBUG) return - - const nIssues = lintResults.reduce((acc, r) => acc + r.messages.length, 0) - debug(`engine: ${lintResults.length} files linted and ${nIssues} issues found`) + return results } async function chunkFilesByTotalSize(files: string[], maxChunkSize: number): Promise { From 66525fad35896dc3afe30c01c9956b40897b5dae Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Fri, 28 Aug 2026 14:45:04 +0200 Subject: [PATCH 2/4] fix: expand glob patterns before chunking so default full-repo scans are memory-bounded too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codacy review correctly flagged that the glob branch bypassed chunking entirely — since the default files-to-analyze list ("**/*.ts", etc.) is globs, most repos with no explicit codacyrc.files never hit the chunked path at all, leaving the OOM risk unresolved for the common case. Expand globs up front (excluding node_modules/dist/build/.git) and always run the chunked, per-chunk-converted lint path. --- src/engineImpl.ts | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/engineImpl.ts b/src/engineImpl.ts index bc1a9165e..d6d9d9ec7 100644 --- a/src/engineImpl.ts +++ b/src/engineImpl.ts @@ -1,6 +1,7 @@ import { Codacyrc, Engine, ToolResult } from "codacy-seed" import { ESLint } from "eslint" import fs from "fs" +import { glob } from "glob" import { createEslintConfig } from "./configCreator" import { convertResults } from "./convertResults" @@ -8,6 +9,13 @@ import { DEBUG, debug } from "./logging" import { toolName } from "./toolMetadata" import fsPromises from "fs/promises"; +const GLOB_EXPANSION_IGNORE = [ + "**/node_modules/**", + "**/dist/**", + "**/build/**", + "**/.git/**" +] + export const engineImpl: Engine = async function ( codacyrc?: Codacyrc ): Promise { @@ -18,21 +26,21 @@ export const engineImpl: Engine = async function ( } const srcDirPath = "/src" - const [options, files] = await createEslintConfig( + const [options, filesOrGlobs] = await createEslintConfig( srcDirPath, codacyrc ) - debug(`engine: list of ${files.length} files (or globs) to process in "${srcDirPath}" and options used`) - debug(files) + debug(`engine: list of ${filesOrGlobs.length} files (or globs) to process in "${srcDirPath}" and options used`) + debug(filesOrGlobs) debug(options) const eslint = new ESLint(options) - // Check if there are any glob patterns in the files array - const results = files.some((file: string) => /\*|\?|\[/.test(file)) - ? convertResults(await eslint.lintFiles(files)) - : await lintFilesInChunks(eslint, files) + // Expand any glob patterns up front so every run (default full-repo globs + // included) goes through the same chunked, memory-bounded lint path. + const files = await expandFiles(srcDirPath, filesOrGlobs) + const results = await lintFilesInChunks(eslint, files) debug(`engine: ${results.length} issues/errors found`) @@ -40,6 +48,21 @@ export const engineImpl: Engine = async function ( return results.map((r) => r.relativeTo(srcDirPath)) } +async function expandFiles (srcDirPath: string, filesOrGlobs: string[]): Promise { + const hasGlobs = filesOrGlobs.some((file: string) => /\*|\?|\[/.test(file)) + if (!hasGlobs) return filesOrGlobs + + const expanded = await glob(filesOrGlobs, { + cwd: srcDirPath, + nodir: true, + dot: true, + ignore: GLOB_EXPANSION_IGNORE + }) + + debug(`engine: expanded globs to ${expanded.length} files`) + return expanded +} + async function lintFilesInChunks (eslint: ESLint, files: string[]): Promise { const maxTotalSizePerChunk = 5 * 1024 * 1024; // size in bytes (5MB) const chunksOfFiles = await chunkFilesByTotalSize(files, maxTotalSizePerChunk) From 8a99d6eec54f04af6aa999b0b53d273d665722da Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Fri, 28 Aug 2026 14:47:33 +0200 Subject: [PATCH 3/4] test: cover expandFiles and chunkFilesByTotalSize Addresses reviewer request for test coverage on the new chunking path. --- src/engineImpl.ts | 4 +-- src/test/engineImpl.spec.ts | 64 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/test/engineImpl.spec.ts diff --git a/src/engineImpl.ts b/src/engineImpl.ts index d6d9d9ec7..32532f173 100644 --- a/src/engineImpl.ts +++ b/src/engineImpl.ts @@ -48,7 +48,7 @@ export const engineImpl: Engine = async function ( return results.map((r) => r.relativeTo(srcDirPath)) } -async function expandFiles (srcDirPath: string, filesOrGlobs: string[]): Promise { +export async function expandFiles (srcDirPath: string, filesOrGlobs: string[]): Promise { const hasGlobs = filesOrGlobs.some((file: string) => /\*|\?|\[/.test(file)) if (!hasGlobs) return filesOrGlobs @@ -84,7 +84,7 @@ async function lintFilesChunkByChunk (eslint: ESLint, chunksOfFiles: string[][]) return results } -async function chunkFilesByTotalSize(files: string[], maxChunkSize: number): Promise { +export async function chunkFilesByTotalSize(files: string[], maxChunkSize: number): Promise { const chunks: string[][] = []; let currentChunk: string[] = []; let currentChunkSize = 0; diff --git a/src/test/engineImpl.spec.ts b/src/test/engineImpl.spec.ts new file mode 100644 index 000000000..a7437c75b --- /dev/null +++ b/src/test/engineImpl.spec.ts @@ -0,0 +1,64 @@ +import { expect } from "chai" +import fs from "fs" +import os from "os" +import path from "path" + +import { chunkFilesByTotalSize, expandFiles } from "../engineImpl" + +describe("expandFiles", () => { + it("returns explicit file lists untouched", async () => { + const files = ["src/foo.ts", "src/bar.ts"] + const result = await expandFiles("/src", files) + expect(result).to.deep.equal(files) + }) + + it("expands glob patterns into real files, excluding node_modules", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "engineImpl-")) + try { + fs.mkdirSync(path.join(tmpDir, "node_modules"), { recursive: true }) + fs.writeFileSync(path.join(tmpDir, "a.ts"), "") + fs.writeFileSync(path.join(tmpDir, "b.ts"), "") + fs.writeFileSync(path.join(tmpDir, "node_modules", "ignored.ts"), "") + + const result = await expandFiles(tmpDir, ["**/*.ts"]) + + expect(result.sort()).to.deep.equal(["a.ts", "b.ts"]) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) +}) + +describe("chunkFilesByTotalSize", () => { + it("groups files into chunks not exceeding the max chunk size", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "engineImpl-")) + try { + const fileA = path.join(tmpDir, "a.ts") + const fileB = path.join(tmpDir, "b.ts") + const fileC = path.join(tmpDir, "c.ts") + fs.writeFileSync(fileA, "a".repeat(3)) + fs.writeFileSync(fileB, "b".repeat(3)) + fs.writeFileSync(fileC, "c".repeat(3)) + + const chunks = await chunkFilesByTotalSize([fileA, fileB, fileC], 5) + + expect(chunks).to.deep.equal([[fileA], [fileB], [fileC]]) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + it("always puts at least one file in a chunk even if it alone exceeds the max size", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "engineImpl-")) + try { + const bigFile = path.join(tmpDir, "big.ts") + fs.writeFileSync(bigFile, "x".repeat(10)) + + const chunks = await chunkFilesByTotalSize([bigFile], 5) + + expect(chunks).to.deep.equal([[bigFile]]) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) +}) From 2fca54e2798855aa1ede73ea856a23ab4f06c9eb Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Fri, 28 Aug 2026 14:51:27 +0200 Subject: [PATCH 4/4] revert: drop glob-expansion, keep only per-chunk-conversion fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The glob-expansion path added extra risk (mixed explicit+glob file lists could drop explicitly-requested files under dist/build-named folders) for a case that doesn't occur in normal usage — Codacy always passes an explicit file list, filtering is done upstream. Reverting to just the per-chunk result conversion, which is the actual OOM fix. --- src/engineImpl.ts | 39 +++++----------------- src/test/engineImpl.spec.ts | 64 ------------------------------------- 2 files changed, 8 insertions(+), 95 deletions(-) delete mode 100644 src/test/engineImpl.spec.ts diff --git a/src/engineImpl.ts b/src/engineImpl.ts index 32532f173..bc1a9165e 100644 --- a/src/engineImpl.ts +++ b/src/engineImpl.ts @@ -1,7 +1,6 @@ import { Codacyrc, Engine, ToolResult } from "codacy-seed" import { ESLint } from "eslint" import fs from "fs" -import { glob } from "glob" import { createEslintConfig } from "./configCreator" import { convertResults } from "./convertResults" @@ -9,13 +8,6 @@ import { DEBUG, debug } from "./logging" import { toolName } from "./toolMetadata" import fsPromises from "fs/promises"; -const GLOB_EXPANSION_IGNORE = [ - "**/node_modules/**", - "**/dist/**", - "**/build/**", - "**/.git/**" -] - export const engineImpl: Engine = async function ( codacyrc?: Codacyrc ): Promise { @@ -26,21 +18,21 @@ export const engineImpl: Engine = async function ( } const srcDirPath = "/src" - const [options, filesOrGlobs] = await createEslintConfig( + const [options, files] = await createEslintConfig( srcDirPath, codacyrc ) - debug(`engine: list of ${filesOrGlobs.length} files (or globs) to process in "${srcDirPath}" and options used`) - debug(filesOrGlobs) + debug(`engine: list of ${files.length} files (or globs) to process in "${srcDirPath}" and options used`) + debug(files) debug(options) const eslint = new ESLint(options) - // Expand any glob patterns up front so every run (default full-repo globs - // included) goes through the same chunked, memory-bounded lint path. - const files = await expandFiles(srcDirPath, filesOrGlobs) - const results = await lintFilesInChunks(eslint, files) + // Check if there are any glob patterns in the files array + const results = files.some((file: string) => /\*|\?|\[/.test(file)) + ? convertResults(await eslint.lintFiles(files)) + : await lintFilesInChunks(eslint, files) debug(`engine: ${results.length} issues/errors found`) @@ -48,21 +40,6 @@ export const engineImpl: Engine = async function ( return results.map((r) => r.relativeTo(srcDirPath)) } -export async function expandFiles (srcDirPath: string, filesOrGlobs: string[]): Promise { - const hasGlobs = filesOrGlobs.some((file: string) => /\*|\?|\[/.test(file)) - if (!hasGlobs) return filesOrGlobs - - const expanded = await glob(filesOrGlobs, { - cwd: srcDirPath, - nodir: true, - dot: true, - ignore: GLOB_EXPANSION_IGNORE - }) - - debug(`engine: expanded globs to ${expanded.length} files`) - return expanded -} - async function lintFilesInChunks (eslint: ESLint, files: string[]): Promise { const maxTotalSizePerChunk = 5 * 1024 * 1024; // size in bytes (5MB) const chunksOfFiles = await chunkFilesByTotalSize(files, maxTotalSizePerChunk) @@ -84,7 +61,7 @@ async function lintFilesChunkByChunk (eslint: ESLint, chunksOfFiles: string[][]) return results } -export async function chunkFilesByTotalSize(files: string[], maxChunkSize: number): Promise { +async function chunkFilesByTotalSize(files: string[], maxChunkSize: number): Promise { const chunks: string[][] = []; let currentChunk: string[] = []; let currentChunkSize = 0; diff --git a/src/test/engineImpl.spec.ts b/src/test/engineImpl.spec.ts deleted file mode 100644 index a7437c75b..000000000 --- a/src/test/engineImpl.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { expect } from "chai" -import fs from "fs" -import os from "os" -import path from "path" - -import { chunkFilesByTotalSize, expandFiles } from "../engineImpl" - -describe("expandFiles", () => { - it("returns explicit file lists untouched", async () => { - const files = ["src/foo.ts", "src/bar.ts"] - const result = await expandFiles("/src", files) - expect(result).to.deep.equal(files) - }) - - it("expands glob patterns into real files, excluding node_modules", async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "engineImpl-")) - try { - fs.mkdirSync(path.join(tmpDir, "node_modules"), { recursive: true }) - fs.writeFileSync(path.join(tmpDir, "a.ts"), "") - fs.writeFileSync(path.join(tmpDir, "b.ts"), "") - fs.writeFileSync(path.join(tmpDir, "node_modules", "ignored.ts"), "") - - const result = await expandFiles(tmpDir, ["**/*.ts"]) - - expect(result.sort()).to.deep.equal(["a.ts", "b.ts"]) - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }) - } - }) -}) - -describe("chunkFilesByTotalSize", () => { - it("groups files into chunks not exceeding the max chunk size", async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "engineImpl-")) - try { - const fileA = path.join(tmpDir, "a.ts") - const fileB = path.join(tmpDir, "b.ts") - const fileC = path.join(tmpDir, "c.ts") - fs.writeFileSync(fileA, "a".repeat(3)) - fs.writeFileSync(fileB, "b".repeat(3)) - fs.writeFileSync(fileC, "c".repeat(3)) - - const chunks = await chunkFilesByTotalSize([fileA, fileB, fileC], 5) - - expect(chunks).to.deep.equal([[fileA], [fileB], [fileC]]) - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }) - } - }) - - it("always puts at least one file in a chunk even if it alone exceeds the max size", async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "engineImpl-")) - try { - const bigFile = path.join(tmpDir, "big.ts") - fs.writeFileSync(bigFile, "x".repeat(10)) - - const chunks = await chunkFilesByTotalSize([bigFile], 5) - - expect(chunks).to.deep.equal([[bigFile]]) - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }) - } - }) -})