Force test executables to rerun past delta-build caching

bin/app_cfg.rb

@@ -55,6 +55,9 @@ def initialize()
       :include_test_case => '',
       :exclude_test_case => '',
 
+      # Default to trusting delta-build staleness tracking (no forced rerun)
+      :force_test_rerun => false,
+
       # Default to task categry other than build/plugin tasks
       :build_tasks? => false,
 
@@ -95,6 +98,10 @@ def set_exclude_test_case(matcher)
     @app_cfg[:exclude_test_case] = matcher
   end
 
+  def set_force_test_rerun(enable)
+    @app_cfg[:force_test_rerun] = enable
+  end
+
   def set_build_tasks(enable)
     @app_cfg[:build_tasks?] = enable
   end

bin/cli.rb

@@ -312,6 +312,8 @@ def upgrade(path)
                   :desc => "Filter for individual unit test names"
     method_option :exclude_test_case, :type => :string, :default => '', :lazy_default => CLI_MISSING_PARAMETER_DEFAULT,
                   :desc => "Prevent matched unit test names from running"
+    method_option :force_test_rerun, :type => :boolean, :default => false,
+                  :desc => "Run test executables and report fresh results even if unchanged"
     method_option :ruby_replacement, :type => :boolean, :default => false, :desc => DOC_RUBY_REPLACEMENT_FLAG
     # Include for consistency with other commands (override --verbosity)
     method_option :debug, :type => :boolean, :default => false, :hide => true
@@ -334,6 +336,11 @@ def upgrade(path)
       • `--test-case` and its inverse `--exclude-test-case` set test case name
       matchers to run only a subset of the unit test suite. See docs for full details.
 
+      • `--force-test-rerun` runs test executables and reports fresh results
+      even when nothing changed since the last build. A test build with
+      `:unity` ↳ `:shuffle_tests` enabled does this automatically, since a
+      shuffled run order is only meaningful if the executable actually runs.
+
       • `If --log and --logfile are both specified, --logfile will set the log file path.
       If --no-log and --logfile are both specified, no logging will occur.
 

bin/cli_handler.rb

@@ -204,6 +204,12 @@ def build(env:, app_cfg:, options:{}, tasks:)
       default_tasks: default_tasks
     )
 
+    @helper.process_force_test_rerun(
+      force_test_rerun: options[:force_test_rerun],
+      tasks: tasks,
+      default_tasks: default_tasks
+    )
+
     logging_path = @helper.process_logging_path( config )
     log_filepath = @helper.process_log_filepath( logging_path, options[:log], options[:logfile] )
 
@@ -215,6 +221,7 @@ def build(env:, app_cfg:, options:{}, tasks:)
     app_cfg.set_log_filepath( log_filepath )
     app_cfg.set_include_test_case( options[:test_case] )
     app_cfg.set_exclude_test_case( options[:exclude_test_case] )
+    app_cfg.set_force_test_rerun( options[:force_test_rerun] )
 
     # Set graceful_exit from command line & configuration options
     app_cfg.set_tests_graceful_fail(

bin/cli_helper.rb

@@ -228,6 +228,16 @@ def process_testcase_filters(config:, include:, exclude:, tasks:, default_tasks:
   end
 
 
+  def process_force_test_rerun(force_test_rerun:, tasks:, default_tasks:)
+    # Do nothing if the flag wasn't set
+    return if !force_test_rerun
+
+    unless test_task?( tasks: (tasks.empty? ? default_tasks : tasks ) )
+      raise CeedlingException.new( "The force test rerun option is only applicable to test tasks. No test tasks were specified." )
+    end
+  end
+
+
   def process_graceful_fail(config:, cmdline_graceful_fail:, tasks:, default_tasks:)
     # Precedence
     #  1. Command line option

lib/ceedling/config/configurator.rb

@@ -17,7 +17,7 @@
 class Configurator
 
   attr_reader :project_config_hash, :programmatic_plugins, :rake_plugins
-  attr_accessor :project_logging, :sanity_checks, :include_test_case, :exclude_test_case
+  attr_accessor :project_logging, :sanity_checks, :include_test_case, :exclude_test_case, :force_test_rerun
 
   constructor :configurator_setup, :configurator_builder, :configurator_plugins, :config_walkinator, :yaml_wrapper, :system_wrapper, :loginator, :reportinator, :ruby_expandinator
 

lib/ceedling/setupinator.rb

@@ -50,6 +50,9 @@ def do_setup( app_cfg )
     @configurator.include_test_case = app_cfg[:include_test_case]
     @configurator.exclude_test_case = app_cfg[:exclude_test_case]
 
+    # Set special purpose forced test re-execution (from command line)
+    @configurator.force_test_rerun = app_cfg[:force_test_rerun]
+
     # Verbosity handling
     @configurator.set_verbosity( config_hash )
 

lib/ceedling/test_invoker/test_build_executor.rb

@@ -908,18 +908,27 @@ def stage_build_executables(state)
   # An executable that didn't need relinking (see stage 16) still has valid
   # cached results on disk from whenever it was last built -- Generator#generate_test_results
   # reports those instead of actually (re)running it, per `skipped:` below.
+  #
+  # `:force_test_rerun` and `:unity ↳ :shuffle_tests` both override that skip:
+  # shuffled test-case order is decided at runtime inside the executable itself
+  # (Unity's generated `main()` reshuffles on every invocation), not at compile
+  # or link time, so an executable that's otherwise unchanged still needs to
+  # actually run again for shuffling to have any effect at all.
   def stage_execute(state)
     skipped = 0
+    force_rerun = @configurator.force_test_rerun || @configurator.unity_shuffle_tests
 
     @batchinator.exec(workload: :test, things: state.testables) do |_, testable|
       begin
+        run_now = testable.executable_rebuilt || force_rerun
+
         # Clear out any stale prior result (e.g. a lingering `.fail` from a test
         # that now passes) immediately before an actual (re)run -- not upfront
         # for every test regardless of whether it's about to run, which would
         # destroy the still-valid cached result of a test left unchanged.
-        clean_test_results( testable.paths[:results], File.basename( testable.name ) ) if testable.executable_rebuilt
+        clean_test_results( testable.paths[:results], File.basename( testable.name ) ) if run_now
 
-        unless testable.executable_rebuilt
+        unless run_now
           msg = @reportinator.generate_module_progress(
             operation:   'Skipping test execution for',
             module_name: testable.name,
@@ -935,7 +944,7 @@ def stage_execute(state)
           test_filepath: testable.filepath,
           executable:    testable.executable,
           result:        testable.results_pass,
-          skipped:       !testable.executable_rebuilt
+          skipped:       !run_now
         }
 
         run_fixture_now( **arg_hash )

spec/system/delta_builds_spec.rb

@@ -278,6 +278,59 @@ def probe_source_file!(relative_path, function_name)
     end
   end
 
+  # :unity ↳ :shuffle_tests decides test-case execution order at runtime, inside
+  # the compiled executable's own main() -- not at compile or link time. So an
+  # executable delta builds correctly judge unchanged (same source, same flags,
+  # same generated runner) still needs to actually run again for shuffling to do
+  # anything at all; otherwise the previous run's cached result is reported
+  # forever and the shuffled order is never realized. --force-test-rerun is the
+  # same override, available on demand independent of shuffling.
+  describe "Delta builds: forcing test re-execution despite no changes (temp_sensor)" do
+    before do
+      @c.with_context do
+        output = @c.ceedling_appcmd_exec("example temp_sensor")
+        expect(output).to match(/created/)
+      end
+    end
+
+    it "--force-test-rerun reruns test executables and reports fresh results on an otherwise-unchanged rebuild" do
+      @c.with_context do
+        Dir.chdir "temp_sensor" do
+          @c.ceedling_build_exec("test:all")
+
+          rebuild = @c.ceedling_build_exec("test:all", "--force-test-rerun")
+
+          expect(rebuild).to_not match(/^Compiling /)
+          expect(rebuild).to_not match(/^Linking /)
+          expect(rebuild).to match(/^Running /)
+
+          expect(rebuild).to match(/TESTED:\s+86/)
+          expect(rebuild).to match(/PASSED:\s+86/)
+        end
+      end
+    end
+
+    it ":unity ↳ :shuffle_tests automatically reruns test executables on an otherwise-unchanged rebuild" do
+      @c.with_context do
+        Dir.chdir "temp_sensor" do
+          settings = { :unity => { :shuffle_tests => true } }
+          @c.merge_project_yml_for_test(settings)
+
+          @c.ceedling_build_exec("test:all")
+
+          rebuild = @c.ceedling_build_exec("test:all")
+
+          expect(rebuild).to_not match(/^Compiling /)
+          expect(rebuild).to_not match(/^Linking /)
+          expect(rebuild).to match(/^Running /)
+
+          expect(rebuild).to match(/TESTED:\s+86/)
+          expect(rebuild).to match(/PASSED:\s+86/)
+        end
+      end
+    end
+  end
+
   # Partials preprocessing (stages 6-8) gates its directives-only/preserve-macros/
   # full-expansion passes on their own DependencyTracker targets, one per partial
   # header and one per partial source -- separate from (and in addition to) the

spec/units/bin/cli_helper_spec.rb

@@ -19,20 +19,28 @@
 
 require 'cli_helper'
 require 'ceedling/ruby_expandinator'
+# CliHelper#test_task? references RakeTaskRegistry::TAG_TEST, but only
+# bin/cli_handler.rb requires this file in the real bootstrap (loaded before
+# cli_helper.rb's methods are ever called) -- required directly here since this
+# spec exercises CliHelper in isolation.
+require 'ceedling/rake_app/rake_task_registry'
 
-# Scoped narrowly to #set_ruby_replacement, the new --ruby-replacement CLI flag
-# wiring. Broader CliHelper coverage is a pre-existing gap outside this feature's
-# scope (no spec file existed for this class before this feature).
+# Scoped narrowly to #set_ruby_replacement and #process_force_test_rerun, the
+# --ruby-replacement and --force-test-rerun CLI flags' wiring. Broader CliHelper
+# coverage (including the pre-existing #process_testcase_filters and
+# #process_graceful_fail this new method is modeled on) is a pre-existing gap
+# outside either feature's scope.
 describe CliHelper do
   before(:each) do
-    @ruby_expandinator = RubyExpandinator.new
+    @ruby_expandinator  = RubyExpandinator.new
+    @rake_task_registry = double('rake_task_registry').as_null_object
 
     @cli_helper = described_class.new({
       :file_wrapper       => double('file_wrapper').as_null_object,
       :actions_wrapper    => double('actions_wrapper').as_null_object,
       :config_walkinator  => double('config_walkinator').as_null_object,
       :path_validator     => double('path_validator').as_null_object,
-      :rake_task_registry => double('rake_task_registry').as_null_object,
+      :rake_task_registry => @rake_task_registry,
       :loginator          => double('loginator').as_null_object,
       :reportinator       => double('reportinator').as_null_object,
       :system_wrapper     => double('system_wrapper').as_null_object,
@@ -66,4 +74,38 @@
       expect(@ruby_expandinator.enabled?).to eq(true)
     end
   end
+
+  describe '#process_force_test_rerun' do
+    it 'does nothing when the flag is false, without even consulting the task registry' do
+      expect(@rake_task_registry).to_not receive(:task_is?)
+
+      expect {
+        @cli_helper.process_force_test_rerun( force_test_rerun: false, tasks: [], default_tasks: [] )
+      }.to_not raise_error
+    end
+
+    it 'raises when the flag is true and no test task is present in tasks or default_tasks' do
+      allow(@rake_task_registry).to receive(:task_is?).and_return( false )
+
+      expect {
+        @cli_helper.process_force_test_rerun( force_test_rerun: true, tasks: ['release'], default_tasks: ['test:all'] )
+      }.to raise_error( CeedlingException, /only applicable to test tasks/ )
+    end
+
+    it 'does not raise when the flag is true and a test task is present in tasks' do
+      allow(@rake_task_registry).to receive(:task_is?).with( 'test:all', RakeTaskRegistry::TAG_TEST ).and_return( true )
+
+      expect {
+        @cli_helper.process_force_test_rerun( force_test_rerun: true, tasks: ['test:all'], default_tasks: [] )
+      }.to_not raise_error
+    end
+
+    it 'does not raise when the flag is true, tasks is empty, and a test task is present in default_tasks' do
+      allow(@rake_task_registry).to receive(:task_is?).with( 'test:all', RakeTaskRegistry::TAG_TEST ).and_return( true )
+
+      expect {
+        @cli_helper.process_force_test_rerun( force_test_rerun: true, tasks: [], default_tasks: ['test:all'] )
+      }.to_not raise_error
+    end
+  end
 end

spec/units/test_build_executor_spec.rb

@@ -45,6 +45,8 @@
     allow(@configurator).to receive(:project_use_mocks).and_return( false )
     allow(@configurator).to receive(:project_use_exceptions).and_return( false )
     allow(@configurator).to receive(:collection_all_support).and_return( [] )
+    allow(@configurator).to receive(:force_test_rerun).and_return( false )
+    allow(@configurator).to receive(:unity_shuffle_tests).and_return( false )
 
     allow(@file_path_utils).to receive(:form_test_build_list_filepath).and_return( 'build/list' )
     allow(@file_path_utils).to receive(:form_test_dependencies_filepath).and_return( 'build/deps' )
@@ -300,6 +302,26 @@ def stub_batchinator_exec
       @executor.stage_execute( @state )
     end
 
+    it "runs the test fixture and clears any stale prior result when :force_test_rerun is enabled, even though the executable is unchanged" do
+      @testable.executable_rebuilt = false
+      allow(@configurator).to receive(:force_test_rerun).and_return( true )
+      allow(@file_wrapper).to receive(:rm_f)
+      expect(@file_wrapper).to receive(:rm_f).with( Dir.glob( File.join( 'build/test/results', 'a_test.*' ) ) )
+      expect(@generator).to receive(:generate_test_results).with( hash_including( skipped: false ) )
+
+      @executor.stage_execute( @state )
+    end
+
+    it "runs the test fixture and clears any stale prior result when :unity ↳ :shuffle_tests is enabled, even though the executable is unchanged" do
+      @testable.executable_rebuilt = false
+      allow(@configurator).to receive(:unity_shuffle_tests).and_return( true )
+      allow(@file_wrapper).to receive(:rm_f)
+      expect(@file_wrapper).to receive(:rm_f).with( Dir.glob( File.join( 'build/test/results', 'a_test.*' ) ) )
+      expect(@generator).to receive(:generate_test_results).with( hash_including( skipped: false ) )
+
+      @executor.stage_execute( @state )
+    end
+
     it "always fires the post_test plugin hook, rebuilt or not" do
       @testable.executable_rebuilt = false
       allow(@generator).to receive(:generate_test_results)