diff --git a/CHANGELOG.md b/CHANGELOG.md index a6f5e4d1..593b8e26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo - Changed the base event inheritance to a new base class structure - Refactored some internals of the library to be slightly more performant (Including removing redundant nil checks) +### Changed +- Change to use worst Test Step result as the Test Case result +([#317](https://github.com/cucumber/cucumber-ruby-core/pull/317)) + ## [16.2.0] - 2026-02-06 ### Changed - Added the test result type 'ambiguous' diff --git a/lib/cucumber/core/test/result/boolean_methods.rb b/lib/cucumber/core/test/result/boolean_methods.rb index 78c9c865..8fa0f9da 100644 --- a/lib/cucumber/core/test/result/boolean_methods.rb +++ b/lib/cucumber/core/test/result/boolean_methods.rb @@ -10,7 +10,7 @@ module Result # Simple module that when included generates all the boolean methods for each category of result # The single exception to this is the class method `self.ok?` which is defined for each result individually module BooleanMethods - TYPES = %i[failed ambiguous flaky skipped undefined pending passed unknown].freeze + TYPES = %i[failed ambiguous flaky undefined pending skipped passed unknown].freeze TYPES.each do |result| define_method("#{result}?") do diff --git a/lib/cucumber/core/test/result/summary.rb b/lib/cucumber/core/test/result/summary.rb index 80116191..574073de 100644 --- a/lib/cucumber/core/test/result/summary.rb +++ b/lib/cucumber/core/test/result/summary.rb @@ -16,7 +16,7 @@ module Result # => 1 # class Summary - TYPES = %i[failed ambiguous flaky skipped undefined pending passed unknown].freeze + TYPES = %i[failed ambiguous flaky undefined pending skipped passed unknown].freeze attr_reader :exceptions, :durations diff --git a/lib/cucumber/core/test/runner.rb b/lib/cucumber/core/test/runner.rb index cf329c40..49bf98db 100644 --- a/lib/cucumber/core/test/runner.rb +++ b/lib/cucumber/core/test/runner.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'cucumber/messages/helpers/test_step_result_comparator' + require_relative 'timer' module Cucumber @@ -45,6 +47,8 @@ def done end class RunningTestCase + include Cucumber::Messages::Helpers::TestStepResultComparator + def initialize @timer = Timer.new.start @status = Status::Unknown.new(Result::Unknown.new) @@ -59,27 +63,27 @@ def result end def failed(step_result) - @status = Status::Failing.new(step_result) + not_passing(step_result) self end def ambiguous(step_result) - @status = Status::Ambiguous.new(step_result) + failed(step_result) self end def passed(step_result) - @status = Status::Passing.new(step_result) + @status = Status::Passing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status] self end def pending(_message, step_result) - @status = Status::Pending.new(step_result) + failed(step_result) self end def skipped(step_result) - @status = Status::Skipping.new(step_result) + failed(step_result) self end @@ -96,6 +100,13 @@ def duration(_step_duration, _step_result) self end + private + + def not_passing(step_result) + @status = Status::NotPassing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status] + self + end + attr_reader :status private :status @@ -118,6 +129,10 @@ def execute(test_step, monitor, &) def result raise NoMethodError, 'Override me' end + + def step_result_message + step_result.to_message + end end class Unknown < Base @@ -132,26 +147,14 @@ def result(duration) end end - class Failing < Base + class NotPassing < Base def execute(test_step, monitor) result = test_step.skip(monitor.result) - if result.undefined? - result = result.with_message(%(Undefined step: "#{test_step.text}")) - result = result.with_appended_backtrace(test_step) - end - result - end - - def result(duration) - step_result.with_duration(duration) + result = result.with_message(%(Undefined step: "#{test_step.text}")) if result.undefined? + result = result.with_appended_backtrace(test_step) unless test_step.hook? + result.describe_to(monitor, result) end - end - - Pending = Class.new(Failing) - - Ambiguous = Class.new(Failing) - class Skipping < Failing def result(duration) step_result.with_duration(duration) end diff --git a/spec/cucumber/core/test/runner_spec.rb b/spec/cucumber/core/test/runner_spec.rb index d156ced8..3cbae6f5 100644 --- a/spec/cucumber/core/test/runner_spec.rb +++ b/spec/cucumber/core/test/runner_spec.rb @@ -222,14 +222,14 @@ let(:test_steps) { [failing_step, passing_step] } it 'emits the test_step_finished event with a failed result' do - expect(event_bus).to receive(:test_step_finished).with(failing_step, anything) do |_reported_test_case, result| + expect(event_bus).to receive(:test_step_finished).with(failing_step, anything) do |_reported_test_step, result| expect(result).to be_failed end test_case.describe_to(runner) end it 'emits a test_step_finished event with a skipped result' do - expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_case, result| + expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_step, result| expect(result).to be_skipped end test_case.describe_to(runner) @@ -250,6 +250,158 @@ test_case.describe_to(runner) end end + + context 'with an initial undefined step' do + let(:test_steps) { [undefined_step, passing_step] } + + it 'emits a test_step_finished event when executing an undefined step' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |reported_test_step, _result| + expect(reported_test_step).to be_a(Cucumber::Core::Test::Step) + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_step, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'skips, rather than executing the second step' do + expect(passing_step).not_to receive(:execute) + + allow(passing_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Skipped.new) + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event when executing a skipped step' do + expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |reported_test_step, _result| + expect(reported_test_step).to be_a(Cucumber::Core::Test::Step) + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a skipped result' do + expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_step, result| + expect(result).to be_skipped + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an undefined result' do + expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an exception object' do + expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + context 'with an undefined step and a subsequent ambiguous step' do + let(:test_steps) { [undefined_step, ambiguous_step] } + + it 'emits a test_step_finished event when executing an undefined step' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |reported_test_step, _result| + expect(reported_test_step).to be_a(Cucumber::Core::Test::Step) + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_step, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'skips, rather than executing the second step' do + expect(passing_step).not_to receive(:execute) + + allow(passing_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Skipped.new) + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event when executing a skipped step' do + expect(event_bus).to receive(:test_step_finished).with(ambiguous_step, anything) do |reported_test_step, _result| + expect(reported_test_step).to be_a(Cucumber::Core::Test::Step) + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with an ambiguous result' do + expect(event_bus).to receive(:test_step_finished).with(ambiguous_step, anything) do |_reported_test_step, result| + expect(result).to be_ambiguous + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an ambiguous result' do + expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_ambiguous + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an exception object' do + expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + end + + context 'with a failing after hook' do + let(:test_steps) { [undefined_step, failing_hook] } + + it 'emits a test_step_finished event when executing an undefined step' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |reported_test_step, _result| + expect(reported_test_step).to be_a(Cucumber::Core::Test::Step) + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_step, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a failing result' do + expect(event_bus).to receive(:test_step_finished).with(failing_hook, anything) do |_reported_test_step, result| + expect(result).to be_failed + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with a failing result' do + expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_failed + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an exception object' do + expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'calls skip, rather than execute on test step of the hook' do + expect(failing_hook).not_to receive(:execute) + + allow(failing_hook).to receive(:skip).and_return(Cucumber::Core::Test::Result::Failed.new(Cucumber::Core::Test::Result::UnknownDuration.new, instance_double(StandardError, backtrace: []))) + test_case.describe_to(runner) + end + end + end end context 'with multiple test cases' do diff --git a/spec/support/shared_context/test_step_types.rb b/spec/support/shared_context/test_step_types.rb index 51cd040e..551956a4 100644 --- a/spec/support/shared_context/test_step_types.rb +++ b/spec/support/shared_context/test_step_types.rb @@ -8,4 +8,6 @@ let(:pending_step) { Cucumber::Core::Test::Step.new(3, 'Pending Step', double).with_action { raise Cucumber::Core::Test::Result::Pending, 'TODO' } } let(:skipping_step) { Cucumber::Core::Test::Step.new(4, 'Skipped Step', double).with_action { raise Cucumber::Core::Test::Result::Skipped } } let(:undefined_step) { Cucumber::Core::Test::Step.new(5, 'Undefined Step', double) } + let(:ambiguous_step) { Cucumber::Core::Test::Step.new(6, 'Ambiguous Step', double).with_unskippable_action { raise Cucumber::Core::Test::Result::Ambiguous } } + let(:failing_hook) { Cucumber::Core::Test::Step.new(7, 'Failing Step', double).with_unskippable_action { raise StandardError, 'Error' } } end