keep result.json on empty runs, record tests of suites with failed BeforeSuite

Author: DenysKuchma

lib/codecept.js

@@ -351,7 +351,7 @@ class Codecept {
         event.emit(event.all.after, this)
         // Wait for any recorder tasks added by event.all.after handlers
         await recorder.promise()
-        if (!this.opts.child && !store.dryRun) container.result().save()
+        if (!this.opts.child && !store.dryRun && container.result().tests.length) container.result().save()
         // Set exit code based on test failures
         if (failures) {
           process.exitCode = 1

lib/listener/result.js

@@ -9,4 +9,8 @@ export default function () {
   event.dispatcher.on(event.test.before, test => {
     container.result().addTest(test)
   })
+
+  event.dispatcher.on(event.test.failed, test => {
+    container.result().addTest(test)
+  })
 }

lib/workers.js

@@ -915,7 +915,7 @@ class Workers extends EventEmitter {
 
     this.emit(event.all.result, Container.result())
     event.dispatcher.emit(event.workers.result, Container.result())
-    Container.result().save()
+    if (Container.result().tests.length) Container.result().save()
     this.emit('end') // internal event
   }
 

test/data/sandbox/configs/last-failed-hooks/codecept.conf.js

@@ -0,0 +1,9 @@
+export const config = {
+  tests: './*_test.js',
+  output: './output',
+  helpers: {},
+  include: {},
+  bootstrap: null,
+  mocha: {},
+  name: 'last-failed-hooks',
+}

test/data/sandbox/configs/last-failed-hooks/hooks_test.js

@@ -0,0 +1,15 @@
+import assert from 'assert'
+
+Feature('Hooks')
+
+BeforeSuite(() => {
+  assert.equal(process.env.LAST_FAILED_HOOKS_OK, '1')
+})
+
+Scenario('prepares data', () => {
+  console.log('executed: prepares data')
+})
+
+Scenario('serves requests @smoke', () => {
+  console.log('executed: serves requests')
+})

test/runner/last_failed_test.js

@@ -13,13 +13,24 @@ const config = `--config ${codecept_dir}/codecept.conf.js`
 const outputDir = path.join(codecept_dir, 'output')
 const resultFile = path.join(outputDir, 'result.json')
 
+const hooks_dir = path.join(__dirname, '/../data/sandbox/configs/last-failed-hooks')
+const hooksConfig = `--config ${hooks_dir}/codecept.conf.js`
+const hooksResultFile = path.join(hooks_dir, 'output', 'result.json')
+
 const run = (command, env = {}) =>
   new Promise(resolve => {
     exec(`${runner} ${command} ${config}`, { env: { ...process.env, LAST_FAILED_FIXED: '', ...env } }, (err, stdout) => {
       resolve({ code: err ? err.code : 0, stdout })
     })
   })
 
+const runHooks = (command, env = {}) =>
+  new Promise(resolve => {
+    exec(`${runner} ${command} ${hooksConfig}`, { env: { ...process.env, LAST_FAILED_HOOKS_OK: '', ...env } }, (err, stdout) => {
+      resolve({ code: err ? err.code : 0, stdout })
+    })
+  })
+
 const reportedTests = () => {
   const { tests } = JSON.parse(fs.readFileSync(resultFile, 'utf8'))
   return Object.fromEntries(tests.map(test => [test.title, test.state]))
@@ -37,6 +48,7 @@ describe('--last-failed', function () {
 
   beforeEach(() => {
     fs.rmSync(outputDir, { recursive: true, force: true })
+    fs.rmSync(path.join(hooks_dir, 'output'), { recursive: true, force: true })
   })
 
   it('should save a report with test states after a run', async () => {
@@ -137,4 +149,51 @@ describe('--last-failed', function () {
       expect(stdout).not.toContain('passed')
     }
   })
+
+  it('should not wipe result.json when a run executes no tests', async () => {
+    await run('run')
+    expect(reportedTests()['multiplies numbers']).toEqual('failed')
+
+    const { stdout } = await run('run --grep "no such test"')
+    expect(executedTests(stdout)).toEqual([])
+
+    expect(reportedTests()['multiplies numbers']).toEqual('failed')
+    const { stdout: rerun } = await run('run --last-failed')
+    expect(executedTests(rerun)).toEqual(['divides numbers', 'multiplies numbers', 'uppercases strings'])
+  })
+
+  it('should not wipe result.json in workers when a run executes no tests', async () => {
+    await run('run-workers 2')
+    expect(reportedTests()['multiplies numbers']).toEqual('failed')
+
+    const { stdout } = await run('run-workers 2 --grep "no such test"')
+    expect(executedTests(stdout)).toEqual([])
+
+    expect(reportedTests()['multiplies numbers']).toEqual('failed')
+  })
+
+  it('should rerun tests of a suite with a failed BeforeSuite', async () => {
+    const { code } = await runHooks('run')
+    expect(code).toEqual(1)
+    const { tests } = JSON.parse(fs.readFileSync(hooksResultFile, 'utf8'))
+    expect(Object.fromEntries(tests.map(test => [test.title, test.state]))).toEqual({
+      'prepares data': 'failed',
+      'serves requests @smoke': 'failed',
+    })
+
+    const { stdout } = await runHooks('run --last-failed')
+    expect(executedTests(stdout)).toEqual(['prepares data', 'serves requests'])
+  })
+
+  it('should rerun tests of a suite with a failed BeforeSuite in workers', async () => {
+    await runHooks('run-workers 2')
+    const { tests } = JSON.parse(fs.readFileSync(hooksResultFile, 'utf8'))
+    expect(Object.fromEntries(tests.map(test => [test.title, test.state]))).toEqual({
+      'prepares data': 'failed',
+      'serves requests @smoke': 'failed',
+    })
+
+    const { stdout } = await runHooks('run-workers 2 --last-failed')
+    expect(executedTests(stdout)).toEqual(['prepares data', 'serves requests'])
+  })
 })