Merge pull request #5606 from smalia2001/fix/retry-failed-step-accumulation

fix: prevent retry config accumulation

Author: DenysKuchma

lib/helper/Playwright.js

@@ -584,7 +584,8 @@ class Playwright extends Helper {
     // Clear popup state to ensure clean state for each test
     popupStore.clear()
 
-    recorder.retry({
+    // Configure retry for this test; will clean up after test completes
+    this._retryConfig = {
       retries: test?.opts?.conditionalRetries || 3,
       when: err => {
         if (!err || typeof err.message !== 'string') {
@@ -593,7 +594,8 @@ class Playwright extends Helper {
         // ignore context errors
         return err.message.includes('context')
       },
-    })
+    }
+    recorder.retry(this._retryConfig)
 
     // Start browser if needed (initial start or browser restart strategy)
     if (!this.isRunning && !this.options.manualStart) await this._startBrowser()
@@ -698,6 +700,12 @@ class Playwright extends Helper {
   }
 
   async _after() {
+    // Clean up our retry config to prevent accumulation
+    if (this._retryConfig) {
+      recorder.retries = recorder.retries.filter(r => r !== this._retryConfig)
+      this._retryConfig = null
+    }
+
     if (!this.isRunning) return
 
     // Clear popup state to prevent leakage between tests

lib/helper/Puppeteer.js

@@ -354,7 +354,8 @@ class Puppeteer extends Helper {
   async _before(test) {
     this.sessionPages = {}
     this.currentRunningTest = test
-    recorder.retry({
+    // Configure retry for this test; will clean up after test completes
+    this._retryConfig = {
       retries: test?.opts?.conditionalRetries || 3,
       when: err => {
         if (!err || typeof err.message !== 'string') {
@@ -363,13 +364,20 @@ class Puppeteer extends Helper {
         // ignore context errors
         return err.message.includes('context')
       },
-    })
+    }
+    recorder.retry(this._retryConfig)
     if (this.options.restart && !this.options.manualStart) return this._startBrowser()
     if (!this.isRunning && !this.options.manualStart) return this._startBrowser()
     return this.browser
   }
 
   async _after() {
+    // Clean up our retry config to prevent accumulation
+    if (this._retryConfig) {
+      recorder.retries = recorder.retries.filter(r => r !== this._retryConfig)
+      this._retryConfig = null
+    }
+
     if (!this.isRunning) return
 
     // Clear popup state to prevent leakage between tests

lib/plugin/retryFailedStep.js

@@ -151,7 +151,9 @@ export default function (config) {
     test.opts.stepRetryPriority = stepRetryPriority
 
     debug('applying retries = %d for test %s', config.retries, test.title)
-    recorder.retry(config)
+    if (!recorder.retries.find(r => r === config)) {
+      recorder.retry(config)
+    }
   })
 
   event.dispatcher.on(event.test.started, test => {
@@ -170,4 +172,8 @@ export default function (config) {
       test.opts.conditionalRetries = test.opts.conditionalRetries || config.retries
     }
   })
+
+  event.dispatcher.on(event.test.after, () => {
+    recorder.retries = recorder.retries.filter(r => r !== config)
+  })
 }

test/data/sandbox/configs/retryHooks/codecept.retry.accumulation.conf.js

@@ -0,0 +1,22 @@
+export const config = {
+  tests: './retry_accumulation_test.js',
+  output: './output',
+  helpers: {
+    Playwright: {
+      url: 'http://localhost:8000',
+      manualStart: true,
+    },
+    AccumulationHelper: {
+      require: './helper.accumulation.js',
+    },
+  },
+  plugins: {
+    retryFailedStep: {
+      enabled: true,
+      retries: 2,
+    },
+  },
+  bootstrap: null,
+  mocha: {},
+  name: 'retryAccumulation',
+};

test/data/sandbox/configs/retryHooks/codecept.retry.multipleScenarios.conf.js

@@ -0,0 +1,23 @@
+export const config = {
+  tests: './retry_multiple_scenarios_test.js',
+  output: './output',
+  helpers: {
+    Playwright: {
+      url: 'http://localhost:8000',
+      show: false,
+      restart: false,
+    },
+    AccumulationHelper: {
+      require: './helper.accumulation.js',
+    },
+  },
+  plugins: {
+    retryFailedStep: {
+      enabled: true,
+      retries: 2,
+    },
+  },
+  bootstrap: null,
+  mocha: {},
+  name: 'retryMultipleScenarios',
+};

test/data/sandbox/configs/retryHooks/helper.accumulation.js

@@ -0,0 +1,16 @@
+import Helper from '../../../../../lib/helper.js';
+
+class AccumulationHelper extends Helper {
+  _before() {
+    this._failCount = 0;
+  }
+
+  failingStep() {
+    this._failCount++;
+    if (this._failCount <= 2) {
+      throw new Error('failing step - retry expected');
+    }
+  }
+}
+
+export default AccumulationHelper;

test/data/sandbox/configs/retryHooks/retry_accumulation_test.js

@@ -0,0 +1,9 @@
+Feature('Retry Config Accumulation Test');
+
+Scenario('first scenario', async ({ I }) => {
+  I.failingStep();
+});
+
+Scenario('second scenario', async ({ I }) => {
+  I.failingStep();
+});

test/data/sandbox/configs/retryHooks/retry_multiple_scenarios_test.js

@@ -0,0 +1,13 @@
+Feature('Retry FailedStep - Multiple Consequent Scenarios');
+
+Scenario('first scenario', async ({ I }) => {
+  I.failingStep();
+});
+
+Scenario('second scenario', async ({ I }) => {
+  I.failingStep();
+});
+
+Scenario('third scenario', async ({ I }) => {
+  I.failingStep();
+});

test/runner/retry_hooks_test.js

@@ -66,4 +66,20 @@ describe('CodeceptJS Retry Hooks', function () {
       done()
     })
   })
+
+  it('should prevent retry config accumulation across tests', done => {
+    exec(config_run_config('codecept.retry.accumulation.conf.js', ''), (err, stdout) => {
+      debug_this_test && console.log(stdout)
+      expect(stdout).toContain('2 passed')
+      done()
+    })
+  })
+
+  it('should retryFailedStep on multiple consequent scenarios', done => {
+    exec(config_run_config('codecept.retry.multipleScenarios.conf.js', ''), (err, stdout) => {
+      debug_this_test && console.log(stdout)
+      expect(stdout).toContain('3 passed')
+      done()
+    })
+  })
 })