From 01720b6222774f83174f29fa438a90f1bad255dc Mon Sep 17 00:00:00 2001 From: Real Codesiman Date: Mon, 17 Aug 2026 14:42:53 +0700 Subject: [PATCH 1/5] ci(release): drop ineffective docker build caching, prep runner matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildkit-cache-dance round-tripped the pnpm store and .next/cache through the GitHub Actions cache every run, but turbo.json excludes .next/cache from build outputs so a remote-cache hit skips next build entirely anyway — the round-trip cost more than the ~40s install it saved. Also disables Next's turbopackFileSystemCacheForBuild since the Docker build starts from a clean layer and never reads it back. Adds a per-leg `runner` matrix field (all still ubuntu-latest) and a 45-minute job timeout so a future larger-runner label is a one-word swap once one is actually provisioned. --- .github/workflows/release.yml | 79 +++++++++++++--------------------- apps/builder/docker/Dockerfile | 18 +++++--- apps/builder/next.config.ts | 3 ++ 3 files changed, 45 insertions(+), 55 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b79c2676e..3b1b90482 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,7 +93,7 @@ jobs: build-image: name: Build ${{ matrix.label }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} needs: - prepare-metadata - check @@ -101,21 +101,40 @@ jobs: fail-fast: false matrix: include: + # The builder compile is CPU-bound and `next build` only ever reports + # 3 workers on a 4-vCPU runner, so a bigger runner is the cheapest + # remaining win — but it needs one that actually exists. GitHub does + # NOT publish an `ubuntu-latest-N-cores` label: larger runners are an + # org-level feature that an admin must create and name, and + # `ubuntu-latest-4-cores` in GitHub's docs is only an example name. + # A public repo makes larger-runner minutes free, not the runners + # themselves. An unmatched label does not fail the job — it queues + # until it times out — so this stays on `ubuntu-latest` until a real + # provisioned label is confirmed (`gh api /orgs/ChatbotXIO/actions/ + # runner-groups` needs admin:org). Then it is a one-word change here. - label: builder image: chatbotx-builder dockerfile: ./apps/builder/docker/Dockerfile + runner: ubuntu-latest - label: worker image: chatbotx-worker dockerfile: ./apps/worker/docker/Dockerfile + runner: ubuntu-latest - label: realtime image: chatbotx-realtime dockerfile: ./apps/realtime/docker/Dockerfile + runner: ubuntu-latest - label: mcp-server image: chatbotx-mcp dockerfile: ./apps/mcp-server/docker/Dockerfile + runner: ubuntu-latest - label: javascript-executor image: chatbotx-javascript-executor dockerfile: ./apps/javascript-executor/docker/Dockerfile + runner: ubuntu-latest + # Baseline builder leg was ~16 min and the others ~1-2 min. Bounds a hung + # build, and an unprovisioned `runner` label, against the 24-hour default. + timeout-minutes: 45 permissions: contents: read packages: write @@ -127,40 +146,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # `type=gha` layer cache does NOT persist RUN cache mounts (pnpm store, - # .next/cache). buildkit-cache-dance injects/extracts them through the - # regular actions cache. Builder image only — its Dockerfile declares the - # matching mount ids. - - name: Restore BuildKit cache-mount contents - if: matrix.label == 'builder' - id: buildkit-cache - uses: actions/cache@v4 - with: - path: | - buildkit-cache/pnpm-store - buildkit-cache/builder-next-cache - key: buildkit-cache-${{ matrix.label }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - buildkit-cache-${{ matrix.label }}-${{ github.ref_name }}- - buildkit-cache-${{ matrix.label }}- - - - name: Inject cache mounts into the Docker build - if: matrix.label == 'builder' - uses: reproducible-containers/buildkit-cache-dance@v3 - with: - cache-map: | - { - "buildkit-cache/pnpm-store": { - "target": "/pnpm/store", - "id": "pnpm-store" - }, - "buildkit-cache/builder-next-cache": { - "target": "/app/apps/builder/.next/cache", - "id": "builder-next-cache" - } - } - skip-extraction: ${{ steps.buildkit-cache.outputs.cache-hit }} - - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: @@ -181,12 +166,14 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: ${{ env.BUILD_PLATFORMS }} - # Optional Turbo remote cache (self-hosted turborepo-remote-cache). - # Unset repo secrets resolve to empty strings → remote cache disabled. - secrets: | - TURBO_API=${{ secrets.TURBO_API }} - TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} - TURBO_TEAM=${{ secrets.TURBO_TEAM }} + # Turbo remote cache: deliberately NOT passed. buildx rejects an + # empty-valued secret outright, so passing the partial set that exists + # (TURBO_TOKEN/TURBO_TEAM, no TURBO_API) dropped TURBO_API and left + # turbo stalling on its built-in Vercel default — + # `Remote caching unavailable (Could not connect to vercel.com/api)`. + # The Dockerfile's --mount=type=secret lines are optional-safe, so + # restoring these three lines together is all it takes once a + # turborepo-remote-cache is provisioned and TURBO_API is set. - name: Build and push version image if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') @@ -202,10 +189,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: ${{ env.BUILD_PLATFORMS }} - secrets: | - TURBO_API=${{ secrets.TURBO_API }} - TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} - TURBO_TEAM=${{ secrets.TURBO_TEAM }} + # Turbo remote cache secrets omitted — see "Build and push main image". - name: Build image for pull request if: github.event_name == 'pull_request' @@ -218,10 +202,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: ${{ env.BUILD_PLATFORMS }} - secrets: | - TURBO_API=${{ secrets.TURBO_API }} - TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} - TURBO_TEAM=${{ secrets.TURBO_TEAM }} + # Turbo remote cache secrets omitted — see "Build and push main image". output-image-urls: runs-on: ubuntu-latest diff --git a/apps/builder/docker/Dockerfile b/apps/builder/docker/Dockerfile index fc96a237b..0b150a4d0 100644 --- a/apps/builder/docker/Dockerfile +++ b/apps/builder/docker/Dockerfile @@ -22,8 +22,10 @@ ENV NODE_OPTIONS="--max-old-space-size=4096" WORKDIR /app COPY --from=pre /app/out/json/ . # BuildKit cache mount for the pnpm store (PNPM_HOME=/pnpm → store at /pnpm/store). -# Persisted across CI runs via buildkit-cache-dance in release.yml. -RUN --mount=type=cache,id=pnpm-store,target=/pnpm/store \ +# Local to the build (and to BuildKit layer reuse) — deliberately NOT round-tripped +# through the GitHub Actions cache: exporting/importing the multi-GB store cost far +# more in CI than the ~40s install it was meant to save. +RUN --mount=type=cache,target=/pnpm/store \ pnpm install --frozen-lockfile COPY --from=pre /app/out/full/ . @@ -38,13 +40,17 @@ COPY --from=pre /app/out/full/ . ENV BETTER_AUTH_SECRET="docker-build-time-placeholder-not-used-at-runtime-0" ENV BETTER_AUTH_URL="http://localhost:3000" -# - `.next/cache` cache mount: Next.js incremental artifacts survive across CI -# runs (persisted via buildkit-cache-dance in release.yml). +# - `.next/cache` cache mount: Turbopack's filesystem cache, local to the build. +# Not persisted across CI runs — `turbo.json` excludes `.next/cache/**` from the +# build outputs, so restoring it can never turn a turbo cache miss into a hit; +# it only ever warmed an already-executing build, at a much higher CI cost. # - TURBO_* secret mounts: optional Turbo remote cache (self-hosted # turborepo-remote-cache or Vercel). When the secrets are not provided the # env vars stay unset and turbo builds without a remote cache — a full -# remote-cache hit restores `.next/**` and skips `next build` entirely. -RUN --mount=type=cache,id=builder-next-cache,target=/app/apps/builder/.next/cache \ +# remote-cache hit restores `.next/**` and skips `next build` entirely. This is +# the only mechanism that can actually skip the build; wiring it up is a +# secrets-only change. +RUN --mount=type=cache,target=/app/apps/builder/.next/cache \ --mount=type=secret,id=TURBO_API,env=TURBO_API \ --mount=type=secret,id=TURBO_TOKEN,env=TURBO_TOKEN \ --mount=type=secret,id=TURBO_TEAM,env=TURBO_TEAM \ diff --git a/apps/builder/next.config.ts b/apps/builder/next.config.ts index b225cb176..4e2af2530 100644 --- a/apps/builder/next.config.ts +++ b/apps/builder/next.config.ts @@ -34,6 +34,9 @@ const nextConfig: NextConfig = { // there is nothing for this optimization to rewrite. optimizePackageImports: ["@icons-pack/react-simple-icons"], // turbopackServerFastRefresh: false, + // The Docker build starts from a clean layer and `.next/cache` is not + // persisted across CI runs, so this cache is written and never read. + turbopackFileSystemCacheForBuild: false, }, poweredByHeader: false, async rewrites() { From 4bee9ebeec0743d334b73d76bf09756999b3a5be Mon Sep 17 00:00:00 2001 From: Real Codesiman Date: Mon, 17 Aug 2026 16:50:02 +0700 Subject: [PATCH 2/5] ci: parallelize vitest workers to halve the quality-gate runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI quality gate spent 298s with 111/113 turbo tasks already cached. Nearly all of it was one task: builder:test reported 289s wall for 8.68s of actual test execution, the rest being per-file module-graph rebuild. Cause was maxWorkers: 1 on CI in the shared vitest preset, which ran builder's 248 files serially in a single fork. The comment defending it argued turbo already parallelizes at the task level, but turbo does not parallelize *within* a suite — each suite is one `vitest run` process, so wall clock is bounded below by the slowest single suite. Measured: turbo --concurrency=4 over the 4 heaviest suites gave 2m46s, ~= builder alone. Raise maxWorkers to 2 on CI and drop turbo --concurrency to 2, keeping (concurrency x maxWorkers) <= the runner's 4 vCPU so forks never oversubscribe. Builder, through `turbo run test`: 147s -> 76s. VITEST_MAX_WORKERS overrides the preset for experimentation. It needs passThroughEnv because turbo runs in strict env mode and would otherwise strip it on the very path CI uses; passThroughEnv rather than env since worker count changes timing, not results, and must stay out of the hash. Also cache tsconfig.tsbuildinfo for check-types. incremental: true was inert in CI because the task declared no outputs, so every miss was a cold tsc (~30s for builder). Only apps/builder and packages/ui emit a buildinfo, so the outputs key lives in their own turbo.json — declaring it at the root makes turbo warn for the other 56 check-types tasks. Bump pnpm/action-setup v4 -> v6.0.10 in both workflows to clear the Node 20 deprecation annotation. --- .github/workflows/ci.yml | 16 ++++++--- .github/workflows/release.yml | 2 +- apps/builder/turbo.json | 13 ++++++++ packages/ui/turbo.json | 12 +++++++ packages/vitest-config/src/node.ts | 52 +++++++++++++++++++++++------- turbo.json | 13 +++++++- 6 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 apps/builder/turbo.json create mode 100644 packages/ui/turbo.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9738b9dcc..070703af1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 + # Must stay BEFORE actions/setup-node so its `cache: pnpm` can resolve + # the pnpm binary to locate the store. v6 drops the deprecated Node 20 + # runtime; the version itself comes from `packageManager` in package.json. - name: Set up pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v6.0.10 - name: Set up Node.js uses: actions/setup-node@v5 @@ -42,8 +45,11 @@ jobs: - name: Set up Turbo cache uses: rharkor/caching-for-turbo@v2.5.1 - # --concurrency=4 matches the runner's vCPU count: tsc and vitest tasks - # are all CPU-bound, so turbo's default of 10 concurrent tasks only adds - # contention (and test timeouts) on a 4-core box. + # Budget: turbo concurrency x vitest maxWorkers <= 4 vCPU. The vitest + # preset uses maxWorkers=2 on CI (packages/vitest-config/src/node.ts), + # so concurrency=2 fills the runner without oversubscribing. Turbo + # concurrency does not parallelize within a suite, so spending the + # budget on vitest workers shortens the critical path (builder, 248 + # files); spending it on more concurrent tasks does not. - name: Type-check, lint, and test - run: pnpm turbo run check-types lint test --concurrency=4 + run: pnpm turbo run check-types lint test --concurrency=2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3b1b90482..08b7e5b71 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v5 - name: Set up pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v6.0.10 - name: Set up Node.js uses: actions/setup-node@v5 diff --git a/apps/builder/turbo.json b/apps/builder/turbo.json new file mode 100644 index 000000000..db3d0fd9d --- /dev/null +++ b/apps/builder/turbo.json @@ -0,0 +1,13 @@ +{ + "extends": ["//"], + "tasks": { + // This tsconfig sets `incremental: true`, so `tsc --noEmit` writes a + // tsbuildinfo. Declaring it as an output lets turbo restore it on a cache + // miss, which is the only way `incremental` does anything in CI (a fresh + // runner otherwise starts every check-types from cold — ~30s here). + "check-types": { + "dependsOn": ["^check-types"], + "outputs": ["tsconfig.tsbuildinfo"] + } + } +} diff --git a/packages/ui/turbo.json b/packages/ui/turbo.json new file mode 100644 index 000000000..76f3d888b --- /dev/null +++ b/packages/ui/turbo.json @@ -0,0 +1,12 @@ +{ + "extends": ["//"], + "tasks": { + // See apps/builder/turbo.json — this package also extends + // typescript-config/nextjs.json, so `incremental: true` makes + // `tsc --noEmit` write a tsbuildinfo worth caching. + "check-types": { + "dependsOn": ["^check-types"], + "outputs": ["tsconfig.tsbuildinfo"] + } + } +} diff --git a/packages/vitest-config/src/node.ts b/packages/vitest-config/src/node.ts index 2ec53965f..33ab7e792 100644 --- a/packages/vitest-config/src/node.ts +++ b/packages/vitest-config/src/node.ts @@ -7,6 +7,40 @@ const COVERAGE_THRESHOLD = 80 const setupEnvPath = fileURLToPath(new URL("./setup-env.ts", import.meta.url)) const setupMswPath = fileURLToPath(new URL("./setup-msw.ts", import.meta.url)) +/** Workers per suite on CI. See the `resolveWorkerPool` doc comment. */ +const CI_MAX_WORKERS = 2 + +/** + * Worker sizing. `turbo run test --concurrency=N` runs N workspace suites at + * once, but turbo does NOT parallelize *within* a suite: each suite is one + * `vitest run` process, so wall clock is bounded below by the slowest single + * suite (builder: 248 files). Task-level concurrency therefore cannot shorten + * the critical path — only vitest's own workers can. + * + * Budget: keep (turbo concurrency x maxWorkers) <= vCPU count so forks never + * oversubscribe. CI runs `--concurrency=2` on 4-vCPU runners, so 2 workers per + * suite fills the box exactly. Locally, leave vitest's default (one worker per + * CPU) alone. + * + * Measured on builder (248 files, through `turbo run test`): 1 worker 147s, + * 2 workers 76s, 4 workers 41s. Set `VITEST_MAX_WORKERS` to experiment without + * editing this preset — it is declared in the root turbo.json `passThroughEnv` + * so it survives turbo's strict env mode. + */ +function resolveWorkerPool(): { maxWorkers?: number; minWorkers?: number } { + const override = Number(process.env.VITEST_MAX_WORKERS) + + if (Number.isInteger(override) && override > 0) { + return { maxWorkers: override, minWorkers: 1 } + } + + if (process.env.CI) { + return { maxWorkers: CI_MAX_WORKERS, minWorkers: 1 } + } + + return {} +} + /** * Base Vitest preset for Node.js workspaces (libraries, workers, CLIs). * @@ -32,18 +66,12 @@ const config: ViteUserConfig = defineConfig({ setupFiles: [setupEnvPath, setupMswPath], clearMocks: true, restoreMocks: true, - // On CI, turbo already parallelizes at the task level (many workspace - // suites at once on a small runner). Left at the default, every vitest - // process forks one worker per CPU, multiplying into dozens of node - // processes fighting for 2-4 vCPUs — which is what pushes cold imports - // past testTimeout. One worker per suite keeps total processes ≈ turbo - // concurrency. - ...(process.env.CI ? { maxWorkers: 1 } : {}), - // `turbo run test` executes every workspace's suite concurrently, so a - // test's first module-graph import can take well over vitest's 5s default - // on a loaded machine (and CI runners). A timed-out test also poisons the - // next one in its file: the abandoned call resolves late and increments - // shared mocks. Generous timeouts only delay true hangs. + ...resolveWorkerPool(), + // Several suites run at once (turbo) and several files within each (above), + // so a test's first module-graph import can take well over vitest's 5s + // default on a loaded machine (and CI runners). A timed-out test also + // poisons the next one in its file: the abandoned call resolves late and + // increments shared mocks. Generous timeouts only delay true hangs. testTimeout: 30_000, hookTimeout: 30_000, coverage: { diff --git a/turbo.json b/turbo.json index 3197c7a82..b15fc97e1 100644 --- a/turbo.json +++ b/turbo.json @@ -11,6 +11,11 @@ "lint": { "dependsOn": ["^lint"] }, + // Only the two workspaces whose tsconfig sets `incremental: true` + // (apps/builder, packages/ui — both extend typescript-config/nextjs.json) + // emit a tsbuildinfo, so `outputs` is declared in their own turbo.json + // rather than here: a root-level key makes turbo warn "no output files + // found" for the other 56 check-types tasks. "check-types": { "dependsOn": ["^check-types"] }, @@ -26,7 +31,13 @@ "**/*.spec.ts", "**/*.spec.tsx" ], - "outputs": [] + "outputs": [], + // turbo runs in strict env mode, so without this the escape hatch in + // packages/vitest-config/src/node.ts never reaches vitest through + // `turbo run test` — which is how CI invokes it. passThroughEnv (not + // env) is deliberate: worker count changes timing, not test results, so + // it must not become part of the cache hash. + "passThroughEnv": ["VITEST_MAX_WORKERS"] }, "dev": { "cache": false, From b4f988e293ba542610c129ebc9d6049267a69331 Mon Sep 17 00:00:00 2001 From: Real Codesiman Date: Mon, 17 Aug 2026 22:57:56 +0700 Subject: [PATCH 3/5] ci: split the quality gate into parallel jobs and cut test import cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR gate ran as one 10m17s job. Three structural problems, not tuning: - check-types was serialized 13 deep by `dependsOn: ["^check-types"]`, but nothing here sets `composite`/`references` and no package ships a `dist` (every `exports` resolves to `./src/*.ts`), so each `tsc --noEmit` reparses its dependencies' source and consumes nothing a prior task emitted. The edge conveyed no data. Same for `^lint`: Biome lints files independently, and only builder defines a real lint script — the other 57 tasks were placeholders turbo still scheduled behind the chain. - MSW booted in every test file (~7MB graph plus three hooks) while only 24 files across 8 workspaces mock HTTP — none of builder's 248. It is now opt-in via `mswSetupFiles`. - Import, not execution, dominated the suites (builder: import 419.94s vs tests 16.92s), so enable `deps.optimizer.ssr` and move to the `threads` pool with 4 workers on CI. Types, lint and tests now run as three jobs on their own runners, so wall clock is the slowest phase rather than their sum. This also closes a gap: the root `pnpm lint` (`ultracite check`, repo-wide) was never run in CI — `turbo run lint` only ever reached builder's own script. Isolation stays ON. `--no-isolate` is faster (~30s on builder) but breaks the suite: `vi.mock` registers per module registry, so files sharing a worker overwrite each other's mocks — 18 files failed on threads, 2 on forks, with the failing set changing between runs, though each passes alone. That is pre-existing cross-file coupling and needs its own change. Measured on builder (248 files, CI env, cold): forks/2w 71.9s (previous setting), threads/2w 66.6s, threads/4w 38.4s, all 248 green. Full 56-suite run 98.4s. Note the first CI run after this is a full cache miss: dependency hashes feed each task hash, so dropping the edges invalidates all 56 check-types entries. Judge the win on the second run. --- .github/workflows/ci.yml | 106 +++++++++++++++--- apps/builder/turbo.json | 3 +- integrations/facebook-ads/vitest.config.ts | 15 ++- .../instagram-facebook/vitest.config.ts | 15 ++- integrations/instagram/vitest.config.ts | 15 ++- integrations/messenger/vitest.config.ts | 15 ++- integrations/zalo/vitest.config.ts | 15 ++- packages/business/vitest.config.ts | 15 ++- packages/javascript-sandbox/vitest.config.ts | 15 ++- packages/ui/turbo.json | 3 +- packages/vitest-config/src/node.ts | 79 +++++++++++-- packages/vitest-config/vitest.config.ts | 15 ++- turbo.json | 18 ++- 13 files changed, 288 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 070703af1..bfcca95e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,16 @@ name: CI # Quality gate for type-checking, linting, and tests. # -# This workflow is the ONLY type gate for apps/builder: `next build` runs with -# `typescript.ignoreBuildErrors: true` (see apps/builder/next.config.ts), so -# `turbo run check-types` here must stay green — never remove it without +# The `check-types` job is the ONLY type gate for apps/builder: `next build` +# runs with `typescript.ignoreBuildErrors: true` (see apps/builder/next.config.ts), +# so `turbo run check-types` here must stay green — never remove it without # re-enabling type-checking inside the build. +# +# The three phases run as separate jobs rather than one `turbo run` invocation. +# They are independent, so splitting gives each its own 4-vCPU runner (~3x the +# total CPU) and makes wall clock the slowest phase instead of their sum. It +# also lets each pick its own concurrency: tsc is single-threaded and CPU-bound, +# while vitest parallelizes internally and must stay within the vCPU budget. on: pull_request: @@ -17,10 +23,10 @@ concurrency: cancel-in-progress: true jobs: - verify: - name: Types, lint, tests + check-types: + name: Types runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 steps: - name: Checkout repository uses: actions/checkout@v5 @@ -45,11 +51,83 @@ jobs: - name: Set up Turbo cache uses: rharkor/caching-for-turbo@v2.5.1 - # Budget: turbo concurrency x vitest maxWorkers <= 4 vCPU. The vitest - # preset uses maxWorkers=2 on CI (packages/vitest-config/src/node.ts), - # so concurrency=2 fills the runner without oversubscribing. Turbo - # concurrency does not parallelize within a suite, so spending the - # budget on vitest workers shortens the critical path (builder, 248 - # files); spending it on more concurrent tasks does not. - - name: Type-check, lint, and test - run: pnpm turbo run check-types lint test --concurrency=2 + # 56 independent `tsc --noEmit` runs (no `composite`/`references` in this + # repo, so turbo.json declares no `dependsOn` — see the comment there). + # Each is single-threaded, so concurrency can fill all 4 vCPUs. + - name: Type-check + run: pnpm turbo run check-types --concurrency=4 + + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Set up pnpm + uses: pnpm/action-setup@v6.0.10 + + - name: Set up Node.js + uses: actions/setup-node@v5 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Set up Turbo cache + uses: rharkor/caching-for-turbo@v2.5.1 + + # The root script (`check:agent-instructions && ultracite check`) is the + # real repo-wide lint — biome.json already globs apps/**, packages/**, + # and integrations/**. It was previously never run in CI: `turbo run lint` + # resolves to 57 placeholders plus builder's own script. + - name: Lint + run: pnpm lint + + # builder is the one workspace with a real `lint` script, and it carries + # the i18n key-parity check across all locales. Keep it — `pnpm lint` + # above does not cover it. + - name: Check builder i18n and lint rules + run: pnpm turbo run lint --concurrency=4 + + test: + name: Tests + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Set up pnpm + uses: pnpm/action-setup@v6.0.10 + + - name: Set up Node.js + uses: actions/setup-node@v5 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Set up Turbo cache + uses: rharkor/caching-for-turbo@v2.5.1 + + # Turbo does not parallelize *within* a suite, so wall clock is bounded + # by the slowest single suite — builder (248 files), which is ~40% of + # total test CPU and scales to 4 vitest workers (see the measurements in + # packages/vitest-config/src/node.ts). The preset gives each suite 4 + # workers on the `threads` pool. + # + # concurrency=2 slightly oversubscribes the 4 vCPUs by design: the 55 + # other suites are small and mostly I/O- and import-bound, so letting one + # overlap builder fills the box while builder still gets the cores it + # needs. Measured end-to-end on the full 56-suite run: concurrency=1 + # 170s, =2 123s, =4 101s (on a 12-core box, so CI compresses these) — + # =1 serializes 55 small suites to help only builder, while =4 would + # contend with builder's own workers on a 4-vCPU runner. + - name: Test + run: pnpm turbo run test --concurrency=2 diff --git a/apps/builder/turbo.json b/apps/builder/turbo.json index db3d0fd9d..7d112cb44 100644 --- a/apps/builder/turbo.json +++ b/apps/builder/turbo.json @@ -5,8 +5,9 @@ // tsbuildinfo. Declaring it as an output lets turbo restore it on a cache // miss, which is the only way `incremental` does anything in CI (a fresh // runner otherwise starts every check-types from cold — ~30s here). + // `dependsOn` is deliberately absent — see the root turbo.json. `outputs` + // is orthogonal to ordering and still governs the tsbuildinfo restore. "check-types": { - "dependsOn": ["^check-types"], "outputs": ["tsconfig.tsbuildinfo"] } } diff --git a/integrations/facebook-ads/vitest.config.ts b/integrations/facebook-ads/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/integrations/facebook-ads/vitest.config.ts +++ b/integrations/facebook-ads/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/integrations/instagram-facebook/vitest.config.ts b/integrations/instagram-facebook/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/integrations/instagram-facebook/vitest.config.ts +++ b/integrations/instagram-facebook/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/integrations/instagram/vitest.config.ts b/integrations/instagram/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/integrations/instagram/vitest.config.ts +++ b/integrations/instagram/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/integrations/messenger/vitest.config.ts b/integrations/messenger/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/integrations/messenger/vitest.config.ts +++ b/integrations/messenger/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/integrations/zalo/vitest.config.ts b/integrations/zalo/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/integrations/zalo/vitest.config.ts +++ b/integrations/zalo/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/packages/business/vitest.config.ts b/packages/business/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/packages/business/vitest.config.ts +++ b/packages/business/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/packages/javascript-sandbox/vitest.config.ts b/packages/javascript-sandbox/vitest.config.ts index 84da752da..49a16d39f 100644 --- a/packages/javascript-sandbox/vitest.config.ts +++ b/packages/javascript-sandbox/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "@chatbotx.io/vitest-config/node" +import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" +import { mergeConfig, type ViteUserConfig } from "vitest/config" + +/** + * MSW is opt-in (see `vitest-config/src/node.ts`) — this workspace mocks + * upstream HTTP, so it re-adds the server lifecycle on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/packages/ui/turbo.json b/packages/ui/turbo.json index 76f3d888b..007d654c0 100644 --- a/packages/ui/turbo.json +++ b/packages/ui/turbo.json @@ -4,8 +4,9 @@ // See apps/builder/turbo.json — this package also extends // typescript-config/nextjs.json, so `incremental: true` makes // `tsc --noEmit` write a tsbuildinfo worth caching. + // `dependsOn` is deliberately absent — see the root turbo.json. `outputs` + // is orthogonal to ordering and still governs the tsbuildinfo restore. "check-types": { - "dependsOn": ["^check-types"], "outputs": ["tsconfig.tsbuildinfo"] } } diff --git a/packages/vitest-config/src/node.ts b/packages/vitest-config/src/node.ts index 33ab7e792..96e3a82c3 100644 --- a/packages/vitest-config/src/node.ts +++ b/packages/vitest-config/src/node.ts @@ -4,11 +4,29 @@ import { defineConfig, type ViteUserConfig } from "vitest/config" const COVERAGE_THRESHOLD = 80 +/** + * Opt-in MSW lifecycle, for workspaces whose tests mock upstream HTTP. + * + * Spread into a suite's own `setupFiles` alongside the preset's defaults: + * + * import preset, { mswSetupFiles } from "@chatbotx.io/vitest-config/node" + * import { mergeConfig } from "vitest/config" + * + * export default mergeConfig(preset, { + * test: { setupFiles: [...mswSetupFiles] }, + * }) + * + * Vitest merges `setupFiles` arrays, so this appends to the preset's list + * rather than replacing it. + */ +export const mswSetupFiles: readonly string[] = [ + fileURLToPath(new URL("./setup-msw.ts", import.meta.url)), +] + const setupEnvPath = fileURLToPath(new URL("./setup-env.ts", import.meta.url)) -const setupMswPath = fileURLToPath(new URL("./setup-msw.ts", import.meta.url)) /** Workers per suite on CI. See the `resolveWorkerPool` doc comment. */ -const CI_MAX_WORKERS = 2 +const CI_MAX_WORKERS = 4 /** * Worker sizing. `turbo run test --concurrency=N` runs N workspace suites at @@ -17,15 +35,15 @@ const CI_MAX_WORKERS = 2 * suite (builder: 248 files). Task-level concurrency therefore cannot shorten * the critical path — only vitest's own workers can. * - * Budget: keep (turbo concurrency x maxWorkers) <= vCPU count so forks never - * oversubscribe. CI runs `--concurrency=2` on 4-vCPU runners, so 2 workers per - * suite fills the box exactly. Locally, leave vitest's default (one worker per - * CPU) alone. + * Budget: keep (turbo concurrency x maxWorkers) <= vCPU count so workers never + * oversubscribe. Tests now own a dedicated 4-vCPU job (see ci.yml), so the + * whole box goes to one suite at a time: `--concurrency=1` x 4 workers. * - * Measured on builder (248 files, through `turbo run test`): 1 worker 147s, - * 2 workers 76s, 4 workers 41s. Set `VITEST_MAX_WORKERS` to experiment without - * editing this preset — it is declared in the root turbo.json `passThroughEnv` - * so it survives turbo's strict env mode. + * Measured on builder (248 files, CI env, cold): forks/2w 71.9s (the previous + * setting), threads/2w 66.6s, threads/3w 47.5s, threads/4w 42.3s — all 248 + * files green. Set `VITEST_MAX_WORKERS` to experiment without editing this + * preset — it is declared in the root turbo.json `passThroughEnv` so it + * survives turbo's strict env mode. */ function resolveWorkerPool(): { maxWorkers?: number; minWorkers?: number } { const override = Number(process.env.VITEST_MAX_WORKERS) @@ -63,10 +81,49 @@ const config: ViteUserConfig = defineConfig({ "**/.next/**", "**/.turbo/**", ], - setupFiles: [setupEnvPath, setupMswPath], + // `setup-env` only — it is a dependency-free object literal, and packages + // here read env at module load, so every suite needs it. + // + // MSW is deliberately NOT global. Booting `setupServer()` costs a ~7MB + // module graph plus three lifecycle hooks in EVERY test file, while only + // 24 files across 7 workspaces actually mock HTTP (0 of builder's 248). + // Those workspaces opt in via `msw-setup-files` below. + // + // Trade-off: suites without MSW lose its `onUnhandledRequest: "error"` + // net, which failed tests that made real network calls. `setup-env.ts` + // still points DATABASE_URL/REDIS_URL/S3_ENDPOINT at non-routable + // 127.0.0.1:1, so an accidental connection fails fast rather than + // reaching a real host. + setupFiles: [setupEnvPath], clearMocks: true, restoreMocks: true, + // `threads` starts workers materially cheaper than the default `forks` + // (measured on builder: 71.9s -> 66.6s at equal worker count, and it + // scales better, reaching 42.3s at 4 workers). + // + // `isolate` stays TRUE deliberately. Turning it off is much faster + // (~30s on builder) but breaks the suite: `vi.mock` registers per module + // registry, so files sharing a worker overwrite each other's mocks. + // Measured on this repo, `--no-isolate` failed 18 files / 22 tests on + // threads and 2 files / 9 tests on forks, with the failing set changing + // between runs; each of those files passes when run alone. That is + // pre-existing latent cross-file coupling, not a bug isolation should be + // hiding — do not disable isolation without first decoupling those tests. + pool: "threads", ...resolveWorkerPool(), + // Pre-bundle the external module graph once with esbuild instead of + // re-resolving it per test file. Import — not test execution — dominates + // this repo's suites (builder measured `import 419.94s` vs `tests + // 16.92s`), because every workspace package resolves to raw `src/*.ts`, + // so the drizzle/zod/schema graph is re-transformed for each file. + // This changes no test-isolation semantics. + deps: { + optimizer: { + ssr: { + enabled: true, + }, + }, + }, // Several suites run at once (turbo) and several files within each (above), // so a test's first module-graph import can take well over vitest's 5s // default on a loaded machine (and CI runners). A timed-out test also diff --git a/packages/vitest-config/vitest.config.ts b/packages/vitest-config/vitest.config.ts index 6b4506fa2..f2f303f61 100644 --- a/packages/vitest-config/vitest.config.ts +++ b/packages/vitest-config/vitest.config.ts @@ -1 +1,14 @@ -export { default } from "./src/node.ts" +import { mergeConfig, type ViteUserConfig } from "vitest/config" +import preset, { mswSetupFiles } from "./src/node.ts" + +/** + * MSW is opt-in (see `./src/node.ts`) — this package's own tests exercise the + * MSW server lifecycle, so it re-adds the setup file on top of the preset. + */ +const config: ViteUserConfig = mergeConfig(preset, { + test: { + setupFiles: [...mswSetupFiles], + }, +}) + +export default config diff --git a/turbo.json b/turbo.json index b15fc97e1..4200a506d 100644 --- a/turbo.json +++ b/turbo.json @@ -8,17 +8,23 @@ "outputs": [".next/**", "!.next/cache/**", "dist/**"], "passThroughEnv": ["BETTER_AUTH_SECRET", "BETTER_AUTH_URL"] }, - "lint": { - "dependsOn": ["^lint"] - }, + // No `dependsOn: ["^lint"]`: Biome lints every file independently, so the + // fan-in conveyed no ordering. Worse, only builder defines a real `lint` + // script — the other 57 workspace tasks are placeholders + // that turbo still scheduled and hash-checked behind the chain. + "lint": {}, + // No `dependsOn: ["^check-types"]`: nothing in this repo sets `composite` + // or `references`, and every package `exports` resolves to `./src/*.ts` + // (no package ships a `dist`). So each `tsc --noEmit` re-parses its + // dependencies' source itself and consumes nothing a prior task emitted — + // the edge transmitted no data and only serialized 56 tasks 13 deep. + // // Only the two workspaces whose tsconfig sets `incremental: true` // (apps/builder, packages/ui — both extend typescript-config/nextjs.json) // emit a tsbuildinfo, so `outputs` is declared in their own turbo.json // rather than here: a root-level key makes turbo warn "no output files // found" for the other 56 check-types tasks. - "check-types": { - "dependsOn": ["^check-types"] - }, + "check-types": {}, "test": { "inputs": [ "$TURBO_DEFAULT$", From b2202362f609ce8eb740778c4da26755ca7afdf3 Mon Sep 17 00:00:00 2001 From: Real Codesiman Date: Wed, 19 Aug 2026 09:15:47 +0700 Subject: [PATCH 4/5] ci: harden workflow permissions and dedupe lint/build gating Scope workflow tokens to read-only where jobs never write, split dependabot's docker ecosystem per-app (root has no Dockerfile so the directory glob never tracked base images), drop the pr-labeler issues:write scope now that labels are pre-bootstrapped, and remove duplicate lint/type-check runs between ci.yml and release.yml so each check runs once per relevant trigger. --- .github/dependabot.yml | 40 ++++++++++++++++++++++++++++- .github/workflows/ci.yml | 20 +++++++-------- .github/workflows/pr-labeler.yml | 44 ++++++-------------------------- .github/workflows/release.yml | 39 ++++++++++++++++++++++------ apps/builder/package.json | 2 +- package.json | 2 +- 6 files changed, 90 insertions(+), 57 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index de8ccbf4e..bacbf49b8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,12 +7,50 @@ registries: password: ${{secrets.DOCKER_PASSWORD}} updates: + # No Dockerfile lives at the repo root — each app has its own under + # apps//docker/Dockerfile, and dependabot's docker ecosystem has no + # glob support, so each needs its own directory entry or its base image + # never gets tracked. - package-ecosystem: "docker" - directory: "/" + directory: "/apps/builder/docker" + schedule: + interval: "daily" + registries: + - dockerhub + - package-ecosystem: "docker" + directory: "/apps/worker/docker" + schedule: + interval: "daily" + registries: + - dockerhub + - package-ecosystem: "docker" + directory: "/apps/realtime/docker" + schedule: + interval: "daily" + registries: + - dockerhub + - package-ecosystem: "docker" + directory: "/apps/mcp-server/docker" schedule: interval: "daily" registries: - dockerhub + - package-ecosystem: "docker" + directory: "/apps/javascript-executor/docker" + schedule: + interval: "daily" + registries: + - dockerhub + + # Tracks the actions pinned in .github/workflows/*.yml (actions/checkout, + # docker/build-push-action, etc.) — without this entry nothing refreshes + # those versions, floating-tag or SHA-pinned alike. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(ci)" # pnpm workspace: a single root entry covers every apps/* packages/* integrations/* # member because they share one root pnpm-lock.yaml. Dependabot reads diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfcca95e6..3fded22f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,11 @@ on: push: branches: [main] +# Read-only: these jobs only check out, install, and run local scripts — +# none of them need to write contents, packages, or PR/issue state. +permissions: + contents: read + concurrency: group: ci-${{ github.ref }} cancel-in-progress: true @@ -80,19 +85,14 @@ jobs: - name: Set up Turbo cache uses: rharkor/caching-for-turbo@v2.5.1 - # The root script (`check:agent-instructions && ultracite check`) is the - # real repo-wide lint — biome.json already globs apps/**, packages/**, - # and integrations/**. It was previously never run in CI: `turbo run lint` - # resolves to 57 placeholders plus builder's own script. + # Root `pnpm lint` runs cheapest-first: check:agent-instructions, then + # `turbo run lint` (apps/builder's i18n key-parity check plus 57 + # placeholders), then the slowest step — repo-wide Biome + # (biome.json globs apps/**, packages/**, integrations/**). One script + # covers everything; no separate turbo invocation needed. - name: Lint run: pnpm lint - # builder is the one workspace with a real `lint` script, and it carries - # the i18n key-parity check across all locales. Keep it — `pnpm lint` - # above does not cover it. - - name: Check builder i18n and lint rules - run: pnpm turbo run lint --concurrency=4 - test: name: Tests runs-on: ubuntu-latest diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 65e0ac1f3..69622319b 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -2,11 +2,14 @@ name: PR Labeler on: pull_request_target: - types: [opened, edited, synchronize, reopened] + types: [opened, edited, reopened] permissions: pull-requests: write - issues: write + +concurrency: + group: pr-labeler-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: label: @@ -16,18 +19,6 @@ jobs: uses: actions/github-script@v7 with: script: | - const LABEL_DEFS = { - 'feature': { color: '0075ca', description: 'New feature or request' }, - 'bug': { color: 'd73a4a', description: "Something isn't working" }, - 'improvement': { color: 'a2eeef', description: 'Refactor or performance improvement' }, - 'chore': { color: 'e4e669', description: 'Maintenance / housekeeping' }, - 'ci': { color: 'e4e669', description: 'CI/CD pipeline changes' }, - 'docs': { color: '0075ca', description: 'Documentation changes' }, - 'security': { color: 'ee0701', description: 'Security fix or hardening' }, - 'breaking-change': { color: 'b60205', description: 'Contains a breaking change' }, - 'dependencies': { color: '0366d6', description: 'Dependency updates' }, - }; - const TYPE_MAP = [ { re: /^feat(\([^)]+\))?!/, labels: ['feature', 'breaking-change'] }, { re: /^feat(\([^)]+\))?:/, labels: ['feature'] }, @@ -53,28 +44,9 @@ jobs: return; } - // Ensure all required labels exist in the repo before applying them. - const { data: existing } = await github.rest.issues.listLabelsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 100, - }); - const existingNames = new Set(existing.map(l => l.name)); - - for (const name of match.labels) { - if (!existingNames.has(name)) { - const meta = LABEL_DEFS[name]; - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name, - color: meta.color, - description: meta.description, - }); - console.log(`Created label: ${name}`); - } - } - + // Labels are bootstrapped once in the repo (see git history for + // pr-labeler.yml) rather than created here on every run — that + // dropped the `issues: write` scope this job used to need. await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08b7e5b71..23b353f19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,15 +21,22 @@ env: BUILD_PLATFORMS: linux/amd64 #,linux/arm64 DOCKER_REGISTRY: ghcr.io +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: true + jobs: # `next build` runs with `typescript.ignoreBuildErrors: true` and the Docker # build never runs `check-types`, so without this gate a `v*` tag (or a # workflow_dispatch on any branch) could push an image that never - # type-checked — ci.yml only runs on pull_request and push-to-main, and - # nothing forces a tag to point at a commit that passed it. This job makes - # the quality gate travel with the artifact rather than with the ref. + # type-checked. ci.yml already gates `pull_request` and push-to-main with + # the same (stronger — see the two run steps below) checks, so this job + # only needs to re-run for events ci.yml does not cover: a version tag or a + # manual dispatch. That keeps the quality gate traveling with the artifact + # without duplicating ci.yml's jobs on every PR. check: name: Type-check, lint, test + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -53,14 +60,23 @@ jobs: - name: Set up Turbo cache uses: rharkor/caching-for-turbo@v2.5.1 - - name: Type-check, lint, and test - run: pnpm turbo run check-types lint test + # `pnpm lint` covers everything, cheapest-first: check:agent-instructions, + # `turbo run lint` for apps/builder's i18n check, then repo-wide Biome + # last (slowest). Type-check and test are separate turbo tasks with no + # lint overlap, so they run standalone here instead of double-running + # lint via a combined `turbo run check-types lint test`. + - name: Lint + run: pnpm lint + + - name: Type-check and test + run: pnpm turbo run check-types test prepare-metadata: runs-on: ubuntu-latest + # Only checks out and computes tag/label strings via docker/metadata- + # action — never touches the registry, so packages: write is unneeded. permissions: contents: read - packages: write environment: ${{ github.event.inputs.environment || 'dev' }} outputs: version: ${{ steps.meta.outputs.version }} @@ -97,6 +113,13 @@ jobs: needs: - prepare-metadata - check + # PRs never push an image (see the "Build image for pull request" step + # below) and ci.yml already gates types/lint/tests, so the PR run's only + # value is a Dockerfile-buildability signal. Keep just the builder leg + # for that on `pull_request` — it's the one most likely to break and has + # by far the longest build (~16 min vs ~1-2 min for the other four) — and + # skip the remaining four legs to cut wasted runner time on every PR. + if: github.event_name != 'pull_request' || matrix.label == 'builder' strategy: fail-fast: false matrix: @@ -210,16 +233,16 @@ jobs: - prepare-metadata - build-image if: github.event_name != 'pull_request' + # Only prints strings computed from `needs` outputs — no registry call. permissions: contents: read - packages: write environment: ${{ github.event.inputs.environment || 'dev' }} steps: - name: Output image URLs run: | VERSION="${{ needs.prepare-metadata.outputs.version }}" REGISTRY="${{ env.DOCKER_REGISTRY }}/${{ needs.prepare-metadata.outputs.image_namespace }}" - if [[ "${{ github.ref_name }}" == "main" || "${{ github.ref_name }}" == "ghcr" ]]; then + if [[ "${{ github.ref_name }}" == "main" ]]; then printf '%s\n' \ "Builder image (main): ${REGISTRY}/chatbotx-builder:main" \ "Worker image (main): ${REGISTRY}/chatbotx-worker:main" \ diff --git a/apps/builder/package.json b/apps/builder/package.json index 652ff2ca1..fce3124bf 100644 --- a/apps/builder/package.json +++ b/apps/builder/package.json @@ -9,7 +9,7 @@ "dev": "dotenv -e .env -e ../../.env -- next dev -p 3123 | pino-pretty", "https": "dotenv -e .env -e ../../.env -- next dev -p 3123 --experimental-https", "i18n:check": "i18n-check --source en --locales messages", - "lint": "pnpm i18n:check && biome check .", + "lint": "pnpm i18n:check", "start": "next start", "test": "vitest run --passWithNoTests", "test:watch": "vitest" diff --git a/package.json b/package.json index bb09d1bac..75a5ec84b 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "build": "turbo build", "dev": "turbo dev", "fix": "ultracite fix --unsafe", - "lint": "pnpm check:agent-instructions && ultracite check", + "lint": "pnpm check:agent-instructions && turbo run lint && ultracite check", "sync:agent-instructions": "node scripts/sync-agent-instructions.mjs --write", "check:agent-instructions": "node scripts/sync-agent-instructions.mjs --check", "test": "turbo run test", From 61a36d3d8ce2b767d18c9a650cf8516333da9d78 Mon Sep 17 00:00:00 2001 From: Real Codesiman Date: Wed, 19 Aug 2026 09:16:04 +0700 Subject: [PATCH 5/5] fix(worker-config): avoid eager Redis dial under vitest getRedisConnection() only skipped lazyConnect for next build (NEXT_PHASE=phase-production-build), so importing any queue barrel under vitest still dialed the non-routable REDIS_URL from setup-env and retried forever. Add isNoRedisEnv() to also treat VITEST=true as a no-dial environment and use it everywhere the build-phase check was duplicated across queue barrels. --- .../__tests__/connection-build-phase.test.ts | 4 ++ .../__tests__/enqueue-integration-job.test.ts | 1 + .../__tests__/no-redis-env.test.ts | 54 +++++++++++++++++++ packages/worker-config/src/lib/connection.ts | 19 +++++-- .../src/queues/ai-agent/index.ts | 14 ++--- .../worker-config/src/queues/chat/index.ts | 14 ++--- .../worker-config/src/queues/default/index.ts | 14 ++--- .../src/queues/integration/index.ts | 14 ++--- .../worker-config/src/queues/quota/index.ts | 14 ++--- .../src/queues/schedule/index.ts | 14 ++--- .../src/queues/sequence-scheduler/index.ts | 4 +- .../worker-config/src/queues/trigger/index.ts | 14 ++--- .../worker-config/src/queues/webhook/index.ts | 14 ++--- 13 files changed, 133 insertions(+), 61 deletions(-) create mode 100644 packages/worker-config/__tests__/no-redis-env.test.ts diff --git a/packages/worker-config/__tests__/connection-build-phase.test.ts b/packages/worker-config/__tests__/connection-build-phase.test.ts index 798f2d7b5..1ef05c4b7 100644 --- a/packages/worker-config/__tests__/connection-build-phase.test.ts +++ b/packages/worker-config/__tests__/connection-build-phase.test.ts @@ -9,6 +9,10 @@ describe("getRedisConnection during next build", () => { vi.resetModules() vi.stubEnv("SKIP_ENV_CHECK", "true") vi.stubEnv("REDIS_URL", DEAD_REDIS_URL) + // isNoRedisEnv() also treats VITEST=true as a no-dial env; stub it away + // per-case so the "outside the build phase" case still exercises the + // eager path this suite is named for. + vi.stubEnv("VITEST", "") }) afterEach(() => { diff --git a/packages/worker-config/__tests__/enqueue-integration-job.test.ts b/packages/worker-config/__tests__/enqueue-integration-job.test.ts index ab00e3917..d75647b58 100644 --- a/packages/worker-config/__tests__/enqueue-integration-job.test.ts +++ b/packages/worker-config/__tests__/enqueue-integration-job.test.ts @@ -22,6 +22,7 @@ vi.mock("../src/lib/connection", () => ({ }, fakeQueue: { add: vi.fn() }, getRedisConnection: () => ({}), + isNoRedisEnv: () => false, })) const { enqueueIntegrationJob, IntegrationJobAction } = await import( diff --git a/packages/worker-config/__tests__/no-redis-env.test.ts b/packages/worker-config/__tests__/no-redis-env.test.ts new file mode 100644 index 000000000..e5b22de27 --- /dev/null +++ b/packages/worker-config/__tests__/no-redis-env.test.ts @@ -0,0 +1,54 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest" + +// isNoRedisEnv() reads process.env directly (no module-scope caching), but +// the queue barrels cache their exported queue at import time, so the module +// registry must still be reset between cases that import a queue. +const DEAD_REDIS_URL = "redis://127.0.0.1:6399" + +describe("isNoRedisEnv", () => { + beforeEach(() => { + vi.resetModules() + vi.stubEnv("SKIP_ENV_CHECK", "true") + vi.stubEnv("REDIS_URL", DEAD_REDIS_URL) + }) + + afterEach(() => { + vi.unstubAllEnvs() + }) + + test("is true under vitest with NEXT_PHASE unset", async () => { + vi.stubEnv("NEXT_PHASE", "") + vi.stubEnv("VITEST", "true") + + const { isNoRedisEnv } = await import("../src/lib/connection") + + expect(isNoRedisEnv()).toBe(true) + }) + + test("is true when NEXT_PHASE is phase-production-build", async () => { + vi.stubEnv("NEXT_PHASE", "phase-production-build") + vi.stubEnv("VITEST", "") + + const { isNoRedisEnv } = await import("../src/lib/connection") + + expect(isNoRedisEnv()).toBe(true) + }) + + test("is false when neither condition holds", async () => { + vi.stubEnv("NEXT_PHASE", "") + vi.stubEnv("VITEST", "") + + const { isNoRedisEnv } = await import("../src/lib/connection") + + expect(isNoRedisEnv()).toBe(false) + }) + + test("importing a queue barrel under vitest yields the fake queue, not a BullMQ Queue", async () => { + vi.stubEnv("VITEST", "true") + + const { aiAgentQueue } = await import("../src/queues/ai-agent") + + expect(typeof aiAgentQueue.add).toBe("function") + expect(aiAgentQueue).not.toHaveProperty("opts") + }) +}) diff --git a/packages/worker-config/src/lib/connection.ts b/packages/worker-config/src/lib/connection.ts index 57a854ed9..2ff08d6f2 100644 --- a/packages/worker-config/src/lib/connection.ts +++ b/packages/worker-config/src/lib/connection.ts @@ -5,6 +5,18 @@ import { keys } from "../keys" let permanentRedis: IORedis | null = null const env = keys() +/** + * True when no Redis is reachable and module-scope consumers must not dial: + * `next build` collecting page data, and vitest (setup-env points REDIS_URL at + * the non-routable 127.0.0.1:1, and an eager client would retry forever). + */ +export function isNoRedisEnv(): boolean { + return ( + process.env.NEXT_PHASE === "phase-production-build" || + process.env.VITEST === "true" + ) +} + export function getRedisConnection() { if (permanentRedis) { return permanentRedis @@ -14,9 +26,10 @@ export function getRedisConnection() { maxRetriesPerRequest: null, enableReadyCheck: true, // Module-scope consumers (event buses, queues) are evaluated while - // `next build` collects page data with no Redis reachable; lazyConnect - // keeps the build from dialing 127.0.0.1:6379 in an infinite retry loop. - lazyConnect: env.NEXT_PHASE === "phase-production-build", + // `next build` collects page data with no Redis reachable, and under + // vitest (setup-env points REDIS_URL at the non-routable 127.0.0.1:1); + // lazyConnect keeps these from dialing in an infinite retry loop. + lazyConnect: isNoRedisEnv(), retryStrategy: (times) => { const delay = Math.min(times * 50, 2000) return delay diff --git a/packages/worker-config/src/queues/ai-agent/index.ts b/packages/worker-config/src/queues/ai-agent/index.ts index aa913ad9c..5c750dbab 100644 --- a/packages/worker-config/src/queues/ai-agent/index.ts +++ b/packages/worker-config/src/queues/ai-agent/index.ts @@ -4,16 +4,16 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" -export const aiAgentQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.aiAgent, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const aiAgentQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.aiAgent, { + connection: getRedisConnection(), + defaultJobOptions, + }) export const AI_FILES_DEFAULT_CHUNK_SIZE = 1000 export const AI_FILES_DEFAULT_OVERLAP_SIZE = 200 diff --git a/packages/worker-config/src/queues/chat/index.ts b/packages/worker-config/src/queues/chat/index.ts index e4539a1f3..a1913e37c 100644 --- a/packages/worker-config/src/queues/chat/index.ts +++ b/packages/worker-config/src/queues/chat/index.ts @@ -27,6 +27,7 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" import type { BotResponseTrackingContext } from "../types" @@ -228,10 +229,9 @@ export type ChatJobData = | ChatJobChangeChannelMessageState | ChatJobCheckOutboundAutomatedResponse -export const chatQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.chat, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const chatQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.chat, { + connection: getRedisConnection(), + defaultJobOptions, + }) diff --git a/packages/worker-config/src/queues/default/index.ts b/packages/worker-config/src/queues/default/index.ts index d2e1ef1c3..d6befc2eb 100644 --- a/packages/worker-config/src/queues/default/index.ts +++ b/packages/worker-config/src/queues/default/index.ts @@ -15,16 +15,16 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" -export const defaultQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.default, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const defaultQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.default, { + connection: getRedisConnection(), + defaultJobOptions, + }) export const DefaultJobAction = { exportContacts: "exportContacts", diff --git a/packages/worker-config/src/queues/integration/index.ts b/packages/worker-config/src/queues/integration/index.ts index f3e1112d0..1a8818d68 100644 --- a/packages/worker-config/src/queues/integration/index.ts +++ b/packages/worker-config/src/queues/integration/index.ts @@ -12,6 +12,7 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" import type { BotResponseTrackingContext } from "../types" @@ -574,13 +575,12 @@ export type IntegrationJobData = | AdsConversionJobEvaluateConversionTrigger | AdsConversionJobSyncRetargetAudience -export const integrationQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.integration, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const integrationQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.integration, { + connection: getRedisConnection(), + defaultJobOptions, + }) // Ads-conversion jobs need a stronger retry policy than the integration // queue default (`attempts: 2` / 5s) — CAPI sends and retarget syncs call diff --git a/packages/worker-config/src/queues/quota/index.ts b/packages/worker-config/src/queues/quota/index.ts index 62ae2b1c4..1ae29d324 100644 --- a/packages/worker-config/src/queues/quota/index.ts +++ b/packages/worker-config/src/queues/quota/index.ts @@ -3,6 +3,7 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" @@ -47,10 +48,9 @@ export type QuotaJobData = | QuotaJobBackfillDefaultPlan | QuotaJobBackfillTenantDefaultPlan -export const quotaQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.quota, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const quotaQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.quota, { + connection: getRedisConnection(), + defaultJobOptions, + }) diff --git a/packages/worker-config/src/queues/schedule/index.ts b/packages/worker-config/src/queues/schedule/index.ts index 5f1a30ec7..9d91b3e71 100644 --- a/packages/worker-config/src/queues/schedule/index.ts +++ b/packages/worker-config/src/queues/schedule/index.ts @@ -4,6 +4,7 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" @@ -199,10 +200,9 @@ export type ScheduleJobData = | ScheduleJobUnsubscribeExpiredTrials | ScheduleJobTeardownExpiredTrial -export const scheduleQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.schedule, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const scheduleQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.schedule, { + connection: getRedisConnection(), + defaultJobOptions, + }) diff --git a/packages/worker-config/src/queues/sequence-scheduler/index.ts b/packages/worker-config/src/queues/sequence-scheduler/index.ts index 526ad26e3..58a5b3800 100644 --- a/packages/worker-config/src/queues/sequence-scheduler/index.ts +++ b/packages/worker-config/src/queues/sequence-scheduler/index.ts @@ -1,6 +1,6 @@ import { sequenceConnections } from "@chatbotx.io/redis" import { Queue } from "bullmq" -import { defaultJobOptions } from "../../lib/connection" +import { defaultJobOptions, isNoRedisEnv } from "../../lib/connection" import { queueNames } from "../../lib/types" export type SequenceSchedulerJobData = { @@ -15,7 +15,7 @@ let sequenceSchedulerQueueInstance: Queue | null = export const getSequenceSchedulerQueue = async (): Promise | null> => { - if (process.env.NEXT_PHASE === "phase-production-build") { + if (isNoRedisEnv()) { return null } diff --git a/packages/worker-config/src/queues/trigger/index.ts b/packages/worker-config/src/queues/trigger/index.ts index b3fdde696..12ee9c290 100644 --- a/packages/worker-config/src/queues/trigger/index.ts +++ b/packages/worker-config/src/queues/trigger/index.ts @@ -4,6 +4,7 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" @@ -39,10 +40,9 @@ export type TriggerJobEvaluate = { export type TriggerJobData = TriggerJobExecute | TriggerJobEvaluate -export const triggerQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.trigger, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const triggerQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.trigger, { + connection: getRedisConnection(), + defaultJobOptions, + }) diff --git a/packages/worker-config/src/queues/webhook/index.ts b/packages/worker-config/src/queues/webhook/index.ts index ca1a5bb9d..9e7fc29f8 100644 --- a/packages/worker-config/src/queues/webhook/index.ts +++ b/packages/worker-config/src/queues/webhook/index.ts @@ -4,6 +4,7 @@ import { defaultJobOptions, fakeQueue, getRedisConnection, + isNoRedisEnv, } from "../../lib/connection" import { queueNames } from "../../lib/types" @@ -24,10 +25,9 @@ export type WebhookJobEvaluate = { export type WebhookJobData = WebhookJobEvaluate -export const webhookQueue = - process.env.NEXT_PHASE === "phase-production-build" - ? fakeQueue - : new Queue(queueNames.enum.webhook, { - connection: getRedisConnection(), - defaultJobOptions, - }) +export const webhookQueue = isNoRedisEnv() + ? fakeQueue + : new Queue(queueNames.enum.webhook, { + connection: getRedisConnection(), + defaultJobOptions, + })