diff --git a/bin/app_cfg.rb b/bin/app_cfg.rb index 6b0c3287..06bd5973 100644 --- a/bin/app_cfg.rb +++ b/bin/app_cfg.rb @@ -55,6 +55,9 @@ def initialize() :include_test_case => '', :exclude_test_case => '', + # Default to trusting delta-build staleness tracking (no forced rerun) + :force_test_rerun => false, + # Default to task categry other than build/plugin tasks :build_tasks? => false, @@ -95,6 +98,10 @@ def set_exclude_test_case(matcher) @app_cfg[:exclude_test_case] = matcher end + def set_force_test_rerun(enable) + @app_cfg[:force_test_rerun] = enable + end + def set_build_tasks(enable) @app_cfg[:build_tasks?] = enable end diff --git a/bin/cli.rb b/bin/cli.rb index 20b234b1..2d292854 100644 --- a/bin/cli.rb +++ b/bin/cli.rb @@ -312,6 +312,8 @@ def upgrade(path) :desc => "Filter for individual unit test names" method_option :exclude_test_case, :type => :string, :default => '', :lazy_default => CLI_MISSING_PARAMETER_DEFAULT, :desc => "Prevent matched unit test names from running" + method_option :force_test_rerun, :type => :boolean, :default => false, + :desc => "Run test executables and report fresh results even if unchanged" method_option :ruby_replacement, :type => :boolean, :default => false, :desc => DOC_RUBY_REPLACEMENT_FLAG # Include for consistency with other commands (override --verbosity) method_option :debug, :type => :boolean, :default => false, :hide => true @@ -334,6 +336,11 @@ def upgrade(path) • `--test-case` and its inverse `--exclude-test-case` set test case name matchers to run only a subset of the unit test suite. See docs for full details. + • `--force-test-rerun` runs test executables and reports fresh results + even when nothing changed since the last build. A test build with + `:unity` ↳ `:shuffle_tests` enabled does this automatically, since a + shuffled run order is only meaningful if the executable actually runs. + • `If --log and --logfile are both specified, --logfile will set the log file path. If --no-log and --logfile are both specified, no logging will occur. diff --git a/bin/cli_handler.rb b/bin/cli_handler.rb index 59e25c05..67cc7182 100644 --- a/bin/cli_handler.rb +++ b/bin/cli_handler.rb @@ -204,6 +204,12 @@ def build(env:, app_cfg:, options:{}, tasks:) default_tasks: default_tasks ) + @helper.process_force_test_rerun( + force_test_rerun: options[:force_test_rerun], + tasks: tasks, + default_tasks: default_tasks + ) + logging_path = @helper.process_logging_path( config ) log_filepath = @helper.process_log_filepath( logging_path, options[:log], options[:logfile] ) @@ -215,6 +221,7 @@ def build(env:, app_cfg:, options:{}, tasks:) app_cfg.set_log_filepath( log_filepath ) app_cfg.set_include_test_case( options[:test_case] ) app_cfg.set_exclude_test_case( options[:exclude_test_case] ) + app_cfg.set_force_test_rerun( options[:force_test_rerun] ) # Set graceful_exit from command line & configuration options app_cfg.set_tests_graceful_fail( diff --git a/bin/cli_helper.rb b/bin/cli_helper.rb index 98dffd3d..10fd8cb0 100644 --- a/bin/cli_helper.rb +++ b/bin/cli_helper.rb @@ -228,6 +228,16 @@ def process_testcase_filters(config:, include:, exclude:, tasks:, default_tasks: end + def process_force_test_rerun(force_test_rerun:, tasks:, default_tasks:) + # Do nothing if the flag wasn't set + return if !force_test_rerun + + unless test_task?( tasks: (tasks.empty? ? default_tasks : tasks ) ) + raise CeedlingException.new( "The force test rerun option is only applicable to test tasks. No test tasks were specified." ) + end + end + + def process_graceful_fail(config:, cmdline_graceful_fail:, tasks:, default_tasks:) # Precedence # 1. Command line option diff --git a/lib/ceedling/config/configurator.rb b/lib/ceedling/config/configurator.rb index e3af3d89..7d01c027 100644 --- a/lib/ceedling/config/configurator.rb +++ b/lib/ceedling/config/configurator.rb @@ -17,7 +17,7 @@ class Configurator attr_reader :project_config_hash, :programmatic_plugins, :rake_plugins - attr_accessor :project_logging, :sanity_checks, :include_test_case, :exclude_test_case + attr_accessor :project_logging, :sanity_checks, :include_test_case, :exclude_test_case, :force_test_rerun constructor :configurator_setup, :configurator_builder, :configurator_plugins, :config_walkinator, :yaml_wrapper, :system_wrapper, :loginator, :reportinator, :ruby_expandinator diff --git a/lib/ceedling/setupinator.rb b/lib/ceedling/setupinator.rb index dff6b7e3..140c2199 100644 --- a/lib/ceedling/setupinator.rb +++ b/lib/ceedling/setupinator.rb @@ -50,6 +50,9 @@ def do_setup( app_cfg ) @configurator.include_test_case = app_cfg[:include_test_case] @configurator.exclude_test_case = app_cfg[:exclude_test_case] + # Set special purpose forced test re-execution (from command line) + @configurator.force_test_rerun = app_cfg[:force_test_rerun] + # Verbosity handling @configurator.set_verbosity( config_hash ) diff --git a/lib/ceedling/test_invoker/test_build_executor.rb b/lib/ceedling/test_invoker/test_build_executor.rb index 87887e54..ebb2d68e 100644 --- a/lib/ceedling/test_invoker/test_build_executor.rb +++ b/lib/ceedling/test_invoker/test_build_executor.rb @@ -908,18 +908,27 @@ def stage_build_executables(state) # An executable that didn't need relinking (see stage 16) still has valid # cached results on disk from whenever it was last built -- Generator#generate_test_results # reports those instead of actually (re)running it, per `skipped:` below. + # + # `:force_test_rerun` and `:unity ↳ :shuffle_tests` both override that skip: + # shuffled test-case order is decided at runtime inside the executable itself + # (Unity's generated `main()` reshuffles on every invocation), not at compile + # or link time, so an executable that's otherwise unchanged still needs to + # actually run again for shuffling to have any effect at all. def stage_execute(state) skipped = 0 + force_rerun = @configurator.force_test_rerun || @configurator.unity_shuffle_tests @batchinator.exec(workload: :test, things: state.testables) do |_, testable| begin + run_now = testable.executable_rebuilt || force_rerun + # Clear out any stale prior result (e.g. a lingering `.fail` from a test # that now passes) immediately before an actual (re)run -- not upfront # for every test regardless of whether it's about to run, which would # destroy the still-valid cached result of a test left unchanged. - clean_test_results( testable.paths[:results], File.basename( testable.name ) ) if testable.executable_rebuilt + clean_test_results( testable.paths[:results], File.basename( testable.name ) ) if run_now - unless testable.executable_rebuilt + unless run_now msg = @reportinator.generate_module_progress( operation: 'Skipping test execution for', module_name: testable.name, @@ -935,7 +944,7 @@ def stage_execute(state) test_filepath: testable.filepath, executable: testable.executable, result: testable.results_pass, - skipped: !testable.executable_rebuilt + skipped: !run_now } run_fixture_now( **arg_hash ) diff --git a/spec/system/delta_builds_spec.rb b/spec/system/delta_builds_spec.rb index cc1c55f2..82d143a0 100644 --- a/spec/system/delta_builds_spec.rb +++ b/spec/system/delta_builds_spec.rb @@ -278,6 +278,59 @@ def probe_source_file!(relative_path, function_name) end end + # :unity ↳ :shuffle_tests decides test-case execution order at runtime, inside + # the compiled executable's own main() -- not at compile or link time. So an + # executable delta builds correctly judge unchanged (same source, same flags, + # same generated runner) still needs to actually run again for shuffling to do + # anything at all; otherwise the previous run's cached result is reported + # forever and the shuffled order is never realized. --force-test-rerun is the + # same override, available on demand independent of shuffling. + describe "Delta builds: forcing test re-execution despite no changes (temp_sensor)" do + before do + @c.with_context do + output = @c.ceedling_appcmd_exec("example temp_sensor") + expect(output).to match(/created/) + end + end + + it "--force-test-rerun reruns test executables and reports fresh results on an otherwise-unchanged rebuild" do + @c.with_context do + Dir.chdir "temp_sensor" do + @c.ceedling_build_exec("test:all") + + rebuild = @c.ceedling_build_exec("test:all", "--force-test-rerun") + + expect(rebuild).to_not match(/^Compiling /) + expect(rebuild).to_not match(/^Linking /) + expect(rebuild).to match(/^Running /) + + expect(rebuild).to match(/TESTED:\s+86/) + expect(rebuild).to match(/PASSED:\s+86/) + end + end + end + + it ":unity ↳ :shuffle_tests automatically reruns test executables on an otherwise-unchanged rebuild" do + @c.with_context do + Dir.chdir "temp_sensor" do + settings = { :unity => { :shuffle_tests => true } } + @c.merge_project_yml_for_test(settings) + + @c.ceedling_build_exec("test:all") + + rebuild = @c.ceedling_build_exec("test:all") + + expect(rebuild).to_not match(/^Compiling /) + expect(rebuild).to_not match(/^Linking /) + expect(rebuild).to match(/^Running /) + + expect(rebuild).to match(/TESTED:\s+86/) + expect(rebuild).to match(/PASSED:\s+86/) + end + end + end + end + # Partials preprocessing (stages 6-8) gates its directives-only/preserve-macros/ # full-expansion passes on their own DependencyTracker targets, one per partial # header and one per partial source -- separate from (and in addition to) the diff --git a/spec/units/bin/cli_helper_spec.rb b/spec/units/bin/cli_helper_spec.rb index e1f9448f..6e748f64 100644 --- a/spec/units/bin/cli_helper_spec.rb +++ b/spec/units/bin/cli_helper_spec.rb @@ -19,20 +19,28 @@ require 'cli_helper' require 'ceedling/ruby_expandinator' +# CliHelper#test_task? references RakeTaskRegistry::TAG_TEST, but only +# bin/cli_handler.rb requires this file in the real bootstrap (loaded before +# cli_helper.rb's methods are ever called) -- required directly here since this +# spec exercises CliHelper in isolation. +require 'ceedling/rake_app/rake_task_registry' -# Scoped narrowly to #set_ruby_replacement, the new --ruby-replacement CLI flag -# wiring. Broader CliHelper coverage is a pre-existing gap outside this feature's -# scope (no spec file existed for this class before this feature). +# Scoped narrowly to #set_ruby_replacement and #process_force_test_rerun, the +# --ruby-replacement and --force-test-rerun CLI flags' wiring. Broader CliHelper +# coverage (including the pre-existing #process_testcase_filters and +# #process_graceful_fail this new method is modeled on) is a pre-existing gap +# outside either feature's scope. describe CliHelper do before(:each) do - @ruby_expandinator = RubyExpandinator.new + @ruby_expandinator = RubyExpandinator.new + @rake_task_registry = double('rake_task_registry').as_null_object @cli_helper = described_class.new({ :file_wrapper => double('file_wrapper').as_null_object, :actions_wrapper => double('actions_wrapper').as_null_object, :config_walkinator => double('config_walkinator').as_null_object, :path_validator => double('path_validator').as_null_object, - :rake_task_registry => double('rake_task_registry').as_null_object, + :rake_task_registry => @rake_task_registry, :loginator => double('loginator').as_null_object, :reportinator => double('reportinator').as_null_object, :system_wrapper => double('system_wrapper').as_null_object, @@ -66,4 +74,38 @@ expect(@ruby_expandinator.enabled?).to eq(true) end end + + describe '#process_force_test_rerun' do + it 'does nothing when the flag is false, without even consulting the task registry' do + expect(@rake_task_registry).to_not receive(:task_is?) + + expect { + @cli_helper.process_force_test_rerun( force_test_rerun: false, tasks: [], default_tasks: [] ) + }.to_not raise_error + end + + it 'raises when the flag is true and no test task is present in tasks or default_tasks' do + allow(@rake_task_registry).to receive(:task_is?).and_return( false ) + + expect { + @cli_helper.process_force_test_rerun( force_test_rerun: true, tasks: ['release'], default_tasks: ['test:all'] ) + }.to raise_error( CeedlingException, /only applicable to test tasks/ ) + end + + it 'does not raise when the flag is true and a test task is present in tasks' do + allow(@rake_task_registry).to receive(:task_is?).with( 'test:all', RakeTaskRegistry::TAG_TEST ).and_return( true ) + + expect { + @cli_helper.process_force_test_rerun( force_test_rerun: true, tasks: ['test:all'], default_tasks: [] ) + }.to_not raise_error + end + + it 'does not raise when the flag is true, tasks is empty, and a test task is present in default_tasks' do + allow(@rake_task_registry).to receive(:task_is?).with( 'test:all', RakeTaskRegistry::TAG_TEST ).and_return( true ) + + expect { + @cli_helper.process_force_test_rerun( force_test_rerun: true, tasks: [], default_tasks: ['test:all'] ) + }.to_not raise_error + end + end end diff --git a/spec/units/test_build_executor_spec.rb b/spec/units/test_build_executor_spec.rb index 00d4b792..707b6a23 100644 --- a/spec/units/test_build_executor_spec.rb +++ b/spec/units/test_build_executor_spec.rb @@ -45,6 +45,8 @@ allow(@configurator).to receive(:project_use_mocks).and_return( false ) allow(@configurator).to receive(:project_use_exceptions).and_return( false ) allow(@configurator).to receive(:collection_all_support).and_return( [] ) + allow(@configurator).to receive(:force_test_rerun).and_return( false ) + allow(@configurator).to receive(:unity_shuffle_tests).and_return( false ) allow(@file_path_utils).to receive(:form_test_build_list_filepath).and_return( 'build/list' ) allow(@file_path_utils).to receive(:form_test_dependencies_filepath).and_return( 'build/deps' ) @@ -300,6 +302,26 @@ def stub_batchinator_exec @executor.stage_execute( @state ) end + it "runs the test fixture and clears any stale prior result when :force_test_rerun is enabled, even though the executable is unchanged" do + @testable.executable_rebuilt = false + allow(@configurator).to receive(:force_test_rerun).and_return( true ) + allow(@file_wrapper).to receive(:rm_f) + expect(@file_wrapper).to receive(:rm_f).with( Dir.glob( File.join( 'build/test/results', 'a_test.*' ) ) ) + expect(@generator).to receive(:generate_test_results).with( hash_including( skipped: false ) ) + + @executor.stage_execute( @state ) + end + + it "runs the test fixture and clears any stale prior result when :unity ↳ :shuffle_tests is enabled, even though the executable is unchanged" do + @testable.executable_rebuilt = false + allow(@configurator).to receive(:unity_shuffle_tests).and_return( true ) + allow(@file_wrapper).to receive(:rm_f) + expect(@file_wrapper).to receive(:rm_f).with( Dir.glob( File.join( 'build/test/results', 'a_test.*' ) ) ) + expect(@generator).to receive(:generate_test_results).with( hash_including( skipped: false ) ) + + @executor.stage_execute( @state ) + end + it "always fires the post_test plugin hook, rebuilt or not" do @testable.executable_rebuilt = false allow(@generator).to receive(:generate_test_results)