diff --git a/scripts/ci/brotli-safe-composer.sh b/scripts/ci/brotli-safe-composer.sh index 7fb4c3c8b..ab6836be6 100755 --- a/scripts/ci/brotli-safe-composer.sh +++ b/scripts/ci/brotli-safe-composer.sh @@ -38,9 +38,21 @@ if test "${1:-}" = --check; then fi if ! docker image inspect "$image" > /dev/null 2>&1; then - printf 'FROM %s\nLABEL dw.composer.base="%s"\nRUN apk add --no-cache --upgrade "%s" && apk info -e "%s"\n' \ + # Fetching Alpine's package index fails now and then on CI runners; the + # build is retried twice before the failure is reported. + attempt=1 + until printf 'FROM %s\nLABEL dw.composer.base="%s"\nRUN apk add --no-cache --upgrade "%s" && apk info -e "%s"\n' \ "$base" "$base" "$floor" "$floor" \ | docker build --quiet --tag "$image" - > /dev/null + do + if test "$attempt" -ge 3; then + echo "Building $image failed $attempt times" >&2 + exit 1 + fi + echo "Building $image failed (attempt $attempt); retrying" >&2 + sleep $((attempt * 10)) + attempt=$((attempt + 1)) + done fi # The derived image must still be the pinned one plus the fixed decoder. parent=$(docker image inspect --format '{{index .Config.Labels "dw.composer.base"}}' "$image") diff --git a/scripts/conformance/server-parity/schedule-probe.php b/scripts/conformance/server-parity/schedule-probe.php index 1f10e3b91..fa8e9e38d 100644 --- a/scripts/conformance/server-parity/schedule-probe.php +++ b/scripts/conformance/server-parity/schedule-probe.php @@ -63,7 +63,9 @@ function prepareHttpSchedule(Client $client, array $fixture, string $scheduleId, } else { // The separate installed schedule:evaluate role fires the original // occurrence. This probe never moves a deadline or writes target rows. - $deadline = microtime(true) + 10; + // Each pass of that role boots the application twice, which can take + // several seconds on a loaded CI runner. + $deadline = microtime(true) + 25; do { $history = parityScheduleHistory($client, $scheduleId); $trigger = parityScheduleTrigger($history); @@ -73,7 +75,7 @@ function prepareHttpSchedule(Client $client, array $fixture, string $scheduleId, break; } if (microtime(true) >= $deadline) { - throw new RuntimeException('The installed scheduler did not fire the original occurrence within ten seconds.'); + throw new RuntimeException('The installed scheduler did not fire the original occurrence within 25 seconds.'); } usleep(50000); } while (true); diff --git a/tests/Unit/Support/composer_trust_test.py b/tests/Unit/Support/composer_trust_test.py index a52353639..5a97ba8bb 100644 --- a/tests/Unit/Support/composer_trust_test.py +++ b/tests/Unit/Support/composer_trust_test.py @@ -421,6 +421,46 @@ def test_other_failures_commands_and_kept_containers_are_not_retried(self) -> No result, runs = self.retrying(1, failure, *arguments) self.assertEqual((100, 1), (result.returncode, runs), result.stderr) + def building(self, failures: int) -> tuple[subprocess.CompletedProcess[str], int]: + """Runs a copy of the floor script whose image build fails `failures` times.""" + work = Path(tempfile.mkdtemp(prefix="dw-brotli-floor-retry-")) + self.addCleanup(shutil.rmtree, work, ignore_errors=True) + (work / "bin").mkdir() + shutil.copy(FLOOR_SCRIPT, work / "brotli-safe-composer.sh") + base = re.search(r"^base=(\S+)$", FLOOR_SCRIPT.read_text(), re.MULTILINE).group(1) + count = work / "builds" + docker = work / "bin" / "docker" + docker.write_text(f"""#!/bin/sh +case "$1 $2" in + "image inspect") + if [ "$3" = --format ]; then echo '{base}'; exit 0; fi + exit 1 ;; + "build --quiet") + cat > /dev/null + n=$(($(cat '{count}' 2>/dev/null || echo 0) + 1)); echo "$n" > '{count}' + if [ "$n" -le {failures} ]; then echo 'ERROR: unable to fetch the package index' >&2; exit 5; fi + exit 0 ;; + "run --rm") exit 0 ;; +esac +exit 64 +""") + (work / "bin" / "sleep").write_text("#!/bin/sh\nexit 0\n") + for stub in (docker, work / "bin" / "sleep"): + stub.chmod(0o755) + result = subprocess.run(["sh", str(work / "brotli-safe-composer.sh")], capture_output=True, text=True, + env={**os.environ, "PATH": f"{work / 'bin'}:{os.environ['PATH']}"}, timeout=60) + return result, int(count.read_text()) + + def test_the_floor_build_retries_a_transient_failure_twice(self) -> None: + result, builds = self.building(2) + self.assertEqual((0, 3), (result.returncode, builds), result.stderr) + self.assertEqual("dw-composer-brotli-floor:2.10.3-da624c2c705d\n", result.stdout) + self.assertEqual(2, result.stderr.count("retrying"), result.stderr) + + result, builds = self.building(3) + self.assertEqual((1, 3), (result.returncode, builds), "three attempts at most") + self.assertIn("failed 3 times", result.stderr) + def test_the_floor_refuses_the_pinned_image_and_accepts_its_own(self) -> None: docker_or_skip(self) refused = subprocess.run(["sh", str(FLOOR_SCRIPT), "--check", PINNED_COMPOSER],