fix(workers): keep results of each browser apart in multi-browser run-workers (#5759)

* fix(workers): keep results of each browser apart in multi-browser run-workers (#5411)

In `run-workers <n> <run> --by suite` with several browsers, the same test
had the same uid in every browser, so the main process merged their results,
and every worker wrote its partial report.xml into the shared run folder.

- Tag every message from a multi-run worker with the run's output folder
  and make the test uid unique per run
- junitReporter: write reports only in the main process, one per run folder
- --last-failed: ignore the run suffix when matching saved uids

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(workers): find the Testomatio HTML report wherever the reporter writes it

Newer @testomatio/reporter versions write the report to <output>/report and
ignore TESTOMATIO_HTML_REPORT_FOLDER.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Jaromir Obr <jobr@spork.tech>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: 3 people

docs/plugins/junitReporter.md

@@ -33,7 +33,7 @@ The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions
 Possible config options:
 
 *   `outputName`: file name for the report. Default: `report.xml`.
-*   `output`: directory where the report is stored, relative to the project root. Default: the `output` directory.
+*   `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored.
 *   `testGroupName`: value of the `name` attribute on the root `<testsuites>` element. Default: `CodeceptJS`.
 *   `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as `<properties>`. Default: true.
 *   `attachSteps`: add the step/substep log as `<system-out>`. Default: true.

lib/codecept.js

@@ -279,7 +279,7 @@ class Codecept {
    */
   filterLastFailedTests(mocha) {
     if (!this.opts.lastFailed) return
-    const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid))
+    const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid?.split('@')[0]))
     const filterSuite = suite => {
       suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid))
       suite.suites.forEach(filterSuite)

lib/plugin/junitReporter.js

@@ -1,6 +1,7 @@
 import fs from 'fs'
 import path from 'path'
 import os from 'os'
+import { isMainThread } from 'worker_threads'
 import { mkdirp } from 'mkdirp'
 import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom'
 
@@ -45,7 +46,7 @@ const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u0
  * Possible config options:
  *
  * * `outputName`: file name for the report. Default: `report.xml`.
- * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory.
+ * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored.
  * * `testGroupName`: value of the `name` attribute on the root `<testsuites>` element. Default: `CodeceptJS`.
  * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as `<properties>`. Default: true.
  * * `attachSteps`: add the step/substep log as `<system-out>`. Default: true.
@@ -92,19 +93,35 @@ export default function (config = {}) {
       meta: {},
       steps: [],
       duration: (runnable && runnable.duration) || 0,
+      outputDir: hook.outputDir,
     })
   })
 
   const writeReport = result => {
     if (written) return
     if (!result || !Array.isArray(result.tests)) return
+    if (!isMainThread) return
     written = true
 
+    const runDirs = [...new Set(result.tests.map(test => test.outputDir).filter(Boolean))]
+    if (runDirs.length) {
+      for (const dir of runDirs) {
+        const runResult = { tests: result.tests.filter(test => test.outputDir === dir), stats: result.stats }
+        const runHookFailures = hookFailures.filter(hf => hf.outputDir === dir)
+        saveXml(dir, buildXml(runResult, config, runHookFailures))
+      }
+      return
+    }
+
     const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd()
+    saveXml(dir, buildXml(result, config, hookFailures))
+  }
+
+  const saveXml = (dir, xml) => {
     mkdirp.sync(dir)
     const file = path.join(dir, config.outputName)
 
-    fs.writeFileSync(file, buildXml(result, config, hookFailures))
+    fs.writeFileSync(file, xml)
     output.plugin('junitReporter', `JUnit report saved to ${file}`)
   }
 

lib/workers.js

@@ -104,9 +104,22 @@ const createWorker = (workerObject, isPoolMode = false, skipTests = []) => {
   })
 
   WorkerStorage.addWorker(worker)
+  worker.runOutputDir = workerObject.runOutputDir
   return worker
 }
 
+const tagWithRun = (message, runOutputDir) => {
+  if (!message.event || !message.data || typeof message.data !== 'object') return
+  const runName = path.basename(runOutputDir)
+  const tag = obj => {
+    if (!obj || typeof obj !== 'object' || obj.outputDir) return
+    obj.outputDir = runOutputDir
+    if (obj.uid) obj.uid = `${obj.uid}@${runName}`
+  }
+  tag(message.data)
+  if (Array.isArray(message.data.tests)) message.data.tests.forEach(tag)
+}
+
 const simplifyObject = object => {
   return Object.keys(object)
     .filter(k => k.indexOf('_') !== 0)
@@ -169,6 +182,7 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns
       workerObj.addTests(tests)
       workerObj.setTestRoot(testRoot)
       workerObj.addOptions(options)
+      workerObj.runOutputDir = path.resolve(testRoot, finalConfig.output)
       testWorkerArray.push(workerObj)
     })
     workers.push(...testWorkerArray)
@@ -659,6 +673,7 @@ class Workers extends EventEmitter {
 
     worker.on('message', message => {
       lastActivity = Date.now() // Update activity timestamp
+      if (worker.runOutputDir) tagWithRun(message, worker.runOutputDir)
 
       // Track current test
       if (message.event === event.test.started && message.data) {

test/data/sandbox/codecept.workers-junit-multiple.conf.js

@@ -0,0 +1,24 @@
+export const config = {
+  tests: './workers-multiple-browsers/*.js',
+  timeout: 10000,
+  output: './output/junit-multiple',
+  helpers: {
+    FakeDriver: {
+      require: '../fake_driver',
+      browser: 'dummy',
+    },
+  },
+  include: {},
+  mocha: {},
+  multiple: {
+    dev: {
+      browsers: ['chromium', 'webkit'],
+    },
+  },
+  plugins: {
+    junitReporter: {
+      enabled: true,
+    },
+  },
+  name: 'sandbox',
+}

test/data/sandbox/codecept.workers-testomatio-multiple.conf.js

@@ -0,0 +1,25 @@
+export const config = {
+  tests: './workers-multiple-browsers/*.js',
+  timeout: 10000,
+  output: './output/testomatio-multiple',
+  helpers: {
+    FakeDriver: {
+      require: '../fake_driver',
+      browser: 'dummy',
+    },
+  },
+  include: {},
+  mocha: {},
+  multiple: {
+    dev: {
+      browsers: ['chromium', 'webkit'],
+    },
+  },
+  plugins: {
+    testomatio: {
+      enabled: true,
+      require: '@testomatio/reporter/codecept',
+    },
+  },
+  name: 'sandbox',
+}

test/data/sandbox/configs/last-failed/codecept.conf.js

@@ -5,5 +5,10 @@ export const config = {
   include: {},
   bootstrap: null,
   mocha: {},
+  multiple: {
+    dev: {
+      browsers: ['chromium', 'webkit'],
+    },
+  },
   name: 'last-failed',
 }

test/data/sandbox/workers-multiple-browsers/suite_a.worker.js

@@ -0,0 +1,9 @@
+Feature('MultipleBrowsersSuiteA')
+
+Scenario('scenario 1', ({ I }) => {
+  I.printBrowser()
+})
+
+Scenario('scenario 2', ({ I }) => {
+  I.printBrowser()
+})

test/data/sandbox/workers-multiple-browsers/suite_b.worker.js

@@ -0,0 +1,9 @@
+Feature('MultipleBrowsersSuiteB')
+
+Scenario('scenario 1', ({ I }) => {
+  I.printBrowser()
+})
+
+Scenario('scenario 2', ({ I }) => {
+  I.printBrowser()
+})

test/runner/last_failed_test.js

@@ -119,6 +119,17 @@ describe('--last-failed', function () {
     expect(stdout).not.toContain('multiplies numbers')
   })
 
+  it('should rerun only failed tests in workers with multiple browsers', async () => {
+    const fullRun = await run('run-workers 2 dev')
+    expect(fullRun.stdout).toContain('4 passed, 6 failed')
+
+    const { code, stdout } = await run('run-workers 2 dev --last-failed')
+    expect(code).toEqual(1)
+    expect(stdout).toContain('0 passed, 6 failed')
+    expect(stdout).not.toContain('adds numbers')
+    expect(stdout).not.toContain('concats strings')
+  })
+
   it('should rerun only failed tests in workers by pool', async () => {
     await run('run')
     const { stdout } = await run('run-workers 2 --by pool --last-failed')