fix(workers): keep results of each browser apart in multi-browser run-workers

docs/plugins/junitReporter.md

@@ -33,7 +33,7 @@ The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions
 Possible config options:
 
 *   `outputName`: file name for the report. Default: `report.xml`.
-*   `output`: directory where the report is stored, relative to the project root. Default: the `output` directory.
+*   `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored.
 *   `testGroupName`: value of the `name` attribute on the root `<testsuites>` element. Default: `CodeceptJS`.
 *   `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as `<properties>`. Default: true.
 *   `attachSteps`: add the step/substep log as `<system-out>`. Default: true.

lib/codecept.js

@@ -279,7 +279,7 @@ class Codecept {
    */
   filterLastFailedTests(mocha) {
     if (!this.opts.lastFailed) return
-    const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid))
+    const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid?.split('@')[0]))
     const filterSuite = suite => {
       suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid))
       suite.suites.forEach(filterSuite)

lib/plugin/junitReporter.js

@@ -1,6 +1,7 @@
 import fs from 'fs'
 import path from 'path'
 import os from 'os'
+import { isMainThread } from 'worker_threads'
 import { mkdirp } from 'mkdirp'
 import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom'
 
@@ -45,7 +46,7 @@ const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u0
  * Possible config options:
  *
  * * `outputName`: file name for the report. Default: `report.xml`.
- * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory.
+ * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored.
  * * `testGroupName`: value of the `name` attribute on the root `<testsuites>` element. Default: `CodeceptJS`.
  * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as `<properties>`. Default: true.
  * * `attachSteps`: add the step/substep log as `<system-out>`. Default: true.
@@ -100,19 +101,35 @@ export default function (config = {}) {
       meta: {},
       steps: [],
       duration: (runnable && runnable.duration) || 0,
+      outputDir: hook.outputDir,
     })
   })
 
   const writeReport = result => {
     if (written) return
     if (!result || !Array.isArray(result.tests)) return
+    if (!isMainThread) return
     written = true
 
+    const runDirs = [...new Set(result.tests.map(test => test.outputDir).filter(Boolean))]
+    if (runDirs.length) {
+      for (const dir of runDirs) {
+        const runResult = { tests: result.tests.filter(test => test.outputDir === dir), stats: result.stats }
+        const runHookFailures = hookFailures.filter(hf => hf.outputDir === dir)
+        saveXml(dir, buildXml(runResult, config, runHookFailures))
+      }
+      return
+    }
+
     const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd()
+    saveXml(dir, buildXml(result, config, hookFailures))
+  }
+
+  const saveXml = (dir, xml) => {
     mkdirp.sync(dir)
     const file = path.join(dir, config.outputName)
 
-    fs.writeFileSync(file, buildXml(result, config, hookFailures))
+    fs.writeFileSync(file, xml)
     output.plugin('junitReporter', `JUnit report saved to ${file}`)
   }
 

lib/workers.js

@@ -104,9 +104,22 @@ const createWorker = (workerObject, isPoolMode = false, skipTests = []) => {
   })
 
   WorkerStorage.addWorker(worker)
+  worker.runOutputDir = workerObject.runOutputDir
   return worker
 }
 
+const tagWithRun = (message, runOutputDir) => {
+  if (!message.event || !message.data || typeof message.data !== 'object') return
+  const runName = path.basename(runOutputDir)
+  const tag = obj => {
+    if (!obj || typeof obj !== 'object' || obj.outputDir) return
+    obj.outputDir = runOutputDir
+    if (obj.uid) obj.uid = `${obj.uid}@${runName}`
+  }
+  tag(message.data)
+  if (Array.isArray(message.data.tests)) message.data.tests.forEach(tag)
+}
+
 const simplifyObject = object => {
   return Object.keys(object)
     .filter(k => k.indexOf('_') !== 0)
@@ -169,6 +182,7 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns
       workerObj.addTests(tests)
       workerObj.setTestRoot(testRoot)
       workerObj.addOptions(options)
+      workerObj.runOutputDir = path.resolve(testRoot, finalConfig.output)
       testWorkerArray.push(workerObj)
     })
     workers.push(...testWorkerArray)
@@ -659,6 +673,7 @@ class Workers extends EventEmitter {
 
     worker.on('message', message => {
       lastActivity = Date.now() // Update activity timestamp
+      if (worker.runOutputDir) tagWithRun(message, worker.runOutputDir)
 
       // Track current test
       if (message.event === event.test.started && message.data) {

test/data/sandbox/codecept.workers-junit-multiple.conf.js

@@ -0,0 +1,24 @@
+export const config = {
+  tests: './workers-multiple-browsers/*.js',
+  timeout: 10000,
+  output: './output/junit-multiple',
+  helpers: {
+    FakeDriver: {
+      require: '../fake_driver',
+      browser: 'dummy',
+    },
+  },
+  include: {},
+  mocha: {},
+  multiple: {
+    dev: {
+      browsers: ['chromium', 'webkit'],
+    },
+  },
+  plugins: {
+    junitReporter: {
+      enabled: true,
+    },
+  },
+  name: 'sandbox',
+}

test/data/sandbox/codecept.workers-testomatio-multiple.conf.js

@@ -0,0 +1,25 @@
+export const config = {
+  tests: './workers-multiple-browsers/*.js',
+  timeout: 10000,
+  output: './output/testomatio-multiple',
+  helpers: {
+    FakeDriver: {
+      require: '../fake_driver',
+      browser: 'dummy',
+    },
+  },
+  include: {},
+  mocha: {},
+  multiple: {
+    dev: {
+      browsers: ['chromium', 'webkit'],
+    },
+  },
+  plugins: {
+    testomatio: {
+      enabled: true,
+      require: '@testomatio/reporter/codecept',
+    },
+  },
+  name: 'sandbox',
+}

test/data/sandbox/configs/last-failed/codecept.conf.js

@@ -5,5 +5,10 @@ export const config = {
   include: {},
   bootstrap: null,
   mocha: {},
+  multiple: {
+    dev: {
+      browsers: ['chromium', 'webkit'],
+    },
+  },
   name: 'last-failed',
 }

test/data/sandbox/workers-multiple-browsers/suite_a.worker.js

@@ -0,0 +1,9 @@
+Feature('MultipleBrowsersSuiteA')
+
+Scenario('scenario 1', ({ I }) => {
+  I.printBrowser()
+})
+
+Scenario('scenario 2', ({ I }) => {
+  I.printBrowser()
+})

test/data/sandbox/workers-multiple-browsers/suite_b.worker.js

@@ -0,0 +1,9 @@
+Feature('MultipleBrowsersSuiteB')
+
+Scenario('scenario 1', ({ I }) => {
+  I.printBrowser()
+})
+
+Scenario('scenario 2', ({ I }) => {
+  I.printBrowser()
+})

test/runner/last_failed_test.js

@@ -119,6 +119,17 @@ describe('--last-failed', function () {
     expect(stdout).not.toContain('multiplies numbers')
   })
 
+  it('should rerun only failed tests in workers with multiple browsers', async () => {
+    const fullRun = await run('run-workers 2 dev')
+    expect(fullRun.stdout).toContain('4 passed, 6 failed')
+
+    const { code, stdout } = await run('run-workers 2 dev --last-failed')
+    expect(code).toEqual(1)
+    expect(stdout).toContain('0 passed, 6 failed')
+    expect(stdout).not.toContain('adds numbers')
+    expect(stdout).not.toContain('concats strings')
+  })
+
   it('should rerun only failed tests in workers by pool', async () => {
     await run('run')
     const { stdout } = await run('run-workers 2 --by pool --last-failed')

test/runner/run_workers_test.js

@@ -3,6 +3,7 @@ chai.should()
 import { expect } from 'expect'
 import path from 'path'
 import fs from 'fs'
+import os from 'os'
 import semver from 'semver'
 import { exec } from 'child_process'
 import { fileURLToPath } from 'url'
@@ -573,6 +574,60 @@ describe('CodeceptJS Workers Runner', function () {
     })
   })
 
+  it('should write one JUnit report per browser with all suites of that browser in multi-run --by suite', function (done) {
+    if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
+    const outputDir = path.join(codecept_dir, 'output', 'junit-multiple')
+    fs.rmSync(outputDir, { recursive: true, force: true })
+
+    exec(`${codecept_run_glob('codecept.workers-junit-multiple.conf.js')} 2 dev --by suite`, (err, stdout) => {
+      ;(async () => {
+        expect(stdout).toContain('Worker 4: 2 tests')
+        expect(stdout).toMatch(/OK\s+\|\s+8 passed/)
+        expect(err).toEqual(null)
+
+        expect(fs.existsSync(path.join(outputDir, 'report.xml'))).toEqual(false)
+        for (const run of ['dev_chromium1', 'dev_webkit1']) {
+          const reportFile = path.join(outputDir, run, 'report.xml')
+          expect(fs.existsSync(reportFile)).toEqual(true)
+          const parsed = await new xml2js.Parser().parseStringPromise(fs.readFileSync(reportFile, 'utf8'))
+          expect(parsed.testsuites.$.tests).toEqual('4')
+          const testcases = parsed.testsuites.testsuite.flatMap(s => s.testcase.map(tc => `${s.$.name}: ${tc.$.name}`)).sort()
+          expect(testcases).toEqual(['MultipleBrowsersSuiteA: scenario 1', 'MultipleBrowsersSuiteA: scenario 2', 'MultipleBrowsersSuiteB: scenario 1', 'MultipleBrowsersSuiteB: scenario 2'])
+        }
+      })().then(done, done)
+    })
+  })
+
+  it('should keep results of all browsers and suites in the Testomatio HTML report in multi-run --by suite', function (done) {
+    if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
+    const outputDir = path.join(codecept_dir, 'output', 'testomatio-multiple')
+    fs.rmSync(outputDir, { recursive: true, force: true })
+    const htmlDir = path.join(outputDir, 'html')
+    const nodePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cjs-node-path-'))
+    fs.symlinkSync(path.join(__dirname, '../..'), path.join(nodePath, 'codeceptjs'), 'dir')
+    const env = { ...process.env, NODE_PATH: nodePath, TESTOMATIO_HTML_REPORT_SAVE: '1', TESTOMATIO_HTML_REPORT_FOLDER: htmlDir }
+    delete env.TESTOMATIO
+
+    exec(`${codecept_run_glob('codecept.workers-testomatio-multiple.conf.js')} 2 dev --by suite`, { env }, (err, stdout) => {
+      fs.rmSync(nodePath, { recursive: true, force: true })
+      expect(stdout).toMatch(/OK\s+\|\s+8 passed/)
+      expect(err).toEqual(null)
+
+      const reports = fs.readdirSync(outputDir, { recursive: true }).filter(file => file.endsWith('testomatio-report.html'))
+      expect(reports).toHaveLength(1)
+      const html = fs.readFileSync(path.join(outputDir, reports[0]), 'utf8')
+      const entries = new Set([...html.matchAll(/"rid":"[^"]*@(dev_\w+)"[^{}]*?"title":"([^"]*)","suite_title":"([^"]*)"/g)].map(([, run, title, suite]) => `${run} ${suite}: ${title}`))
+      const expected = []
+      for (const run of ['dev_chromium1', 'dev_webkit1']) {
+        for (const suite of ['MultipleBrowsersSuiteA', 'MultipleBrowsersSuiteB']) {
+          expected.push(`${run} ${suite}: scenario 1`, `${run} ${suite}: scenario 2`)
+        }
+      }
+      expect([...entries].sort()).toEqual(expected)
+      done()
+    })
+  })
+
   it('should handle large worker count without inflating statistics', function (done) {
     if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version')
     // Test with more workers than tests to ensure no inflation

test/unit/junitReporter_test.js

@@ -253,4 +253,40 @@ describe('JUnit Reporter Plugin', () => {
     expect(suiteEl.testcase[0].$.name).to.contain('"after all" hook: AfterSuite')
     expect(suiteEl.testcase[0].failure[0].$.message).to.contain('AfterSuite failed')
   })
+
+  it('writes one report per run output folder when tests carry outputDir', async () => {
+    const chromiumDir = path.join(tmpDir, 'dev_chromium1')
+    const webkitDir = path.join(tmpDir, 'dev_webkit1')
+    const result = makeResult()
+    const [passed, failed, skipped] = result.tests
+    result.tests = [
+      { ...passed, uid: 'pass-1@dev_chromium1', outputDir: chromiumDir },
+      { ...failed, uid: 'fail-1@dev_chromium1', outputDir: chromiumDir },
+      { ...passed, uid: 'pass-1@dev_webkit1', outputDir: webkitDir },
+      { ...skipped, uid: 'skip-1@dev_webkit1', outputDir: webkitDir },
+    ]
+
+    junitReporter({})
+    event.dispatcher.emit(event.hook.failed, {
+      hookName: 'AfterSuite',
+      title: '"after all" hook: AfterSuite for "logs in successfully"',
+      error: { message: 'AfterSuite failed', stack: 'Error: AfterSuite failed' },
+      outputDir: webkitDir,
+    })
+    event.dispatcher.emit(event.workers.result, result)
+
+    assert.isFalse(fs.existsSync(path.join(tmpDir, 'report.xml')), 'no combined report should be written')
+
+    const chromium = await parseReport(chromiumDir)
+    expect(chromium.testsuites.$.tests).to.equal('2')
+    expect(chromium.testsuites.$.failures).to.equal('1')
+    expect(chromium.testsuites.$.skipped).to.equal('0')
+
+    const webkit = await parseReport(webkitDir)
+    expect(webkit.testsuites.$.tests).to.equal('3')
+    expect(webkit.testsuites.$.failures).to.equal('1')
+    expect(webkit.testsuites.$.skipped).to.equal('1')
+    const names = webkit.testsuites.testsuite.flatMap(s => s.testcase.map(tc => tc.$.name))
+    expect(names.some(n => n.includes('AfterSuite'))).to.equal(true)
+  })
 })