diff --git a/src/nexus_deploy/_remote.py b/src/nexus_deploy/_remote.py index 26aa347c..8d9af803 100644 --- a/src/nexus_deploy/_remote.py +++ b/src/nexus_deploy/_remote.py @@ -22,7 +22,11 @@ from __future__ import annotations +import os +import shlex +import shutil import subprocess +import tempfile from pathlib import Path # No subprocess timeout by default. A slow Hetzner control-plane @@ -110,24 +114,60 @@ def ssh_run_script( ) -def rsync_to_remote( +def push_directory( local: Path, remote: str, *, delete: bool = False, timeout: float | None = _DEFAULT_TIMEOUT_S, ) -> subprocess.CompletedProcess[str]: - """Push a local directory to the nexus server via rsync. + """Push a local directory's CONTENTS to a path on the nexus server. - ``remote`` follows rsync syntax (e.g. ``"nexus:/tmp/infisical-push/"``); - the alias resolves through the same ssh config as ``ssh_run``. The - trailing slash on ``local`` is auto-appended so rsync uploads the - directory's CONTENTS rather than the directory itself. + ``remote`` is ``:`` (e.g. ``"nexus:/tmp/infisical-push/"``); + the alias resolves through the same ssh config as :func:`ssh_run`. - ``delete=True`` clears destination paths that don't exist locally — + ``delete=True`` clears destination paths that do not exist locally — used when the local dir is the canonical source-of-truth for that remote location. + + **Two transports, chosen at call time.** `rsync` when it is on PATH, + which is every GitHub runner, and `tar` over `ssh` when it is not. + A Forgejo runner's job image (`node:22-bookworm`) has ssh, scp, tar + and gzip but no rsync, and Python raises ``FileNotFoundError`` for a + missing executable — which surfaced as `stack-sync: transport + (FileNotFoundError)` on the first Conductor spin-up that reached this + phase (#897). + + Installing rsync there was the other option and was measured: it + needs the rsync and libpopt packages extracted per architecture, and + Debian removes old point releases from its pool — a URL pinned today + 404s within weeks, which is how that route was ruled out. tar and ssh + are in the image and in every runner image this project targets. + + Both transports raise :class:`subprocess.CalledProcessError` with + ``stderr`` populated, because callers print that excerpt. """ + if shutil.which("rsync") is not None: + return _rsync_push(local, remote, delete=delete, timeout=timeout) + return _tar_push(local, remote, delete=delete, timeout=timeout) + + +# Kept as the historical name: four call sites and their tests use it, +# and `stack_sync` names its result type after it. The transport is +# chosen inside `push_directory`, so this no longer promises rsync. +rsync_to_remote = push_directory + + +def _rsync_push( + local: Path, + remote: str, + *, + delete: bool, + timeout: float | None, +) -> subprocess.CompletedProcess[str]: + """The original path. The trailing slash on ``local`` is + auto-appended so rsync uploads the directory's CONTENTS rather than + the directory itself.""" src = f"{local}/" if not str(local).endswith("/") else str(local) args = ["rsync", "-aq"] if delete: @@ -140,3 +180,69 @@ def rsync_to_remote( text=True, timeout=timeout, ) + + +def _tar_push( + local: Path, + remote: str, + *, + delete: bool, + timeout: float | None, +) -> subprocess.CompletedProcess[str]: + """`tar` locally, `tar -x` on the far side, one ssh in between. + + Written as an archive file rather than a `tar | ssh` pipeline on + purpose. A pipeline hides the writer's exit status behind the + reader's, which is the shape that produced #883; here each half is + a separate call whose status is checked on its own. + + The archive inherits ``mkstemp``'s 0600, which matters because one + caller pushes an Infisical payload of secret values. It is removed + in a ``finally``, including when ssh fails. + + ``--no-same-owner`` on extraction: the numeric uid of whatever + account the CI job runs as means nothing on the server, and every + destination here is root-owned service configuration. Modes are + preserved, which is what the 0600 payload needs. + """ + host, separator, path = remote.partition(":") + if not separator or not path: + raise ValueError(f"remote must be ':', got {remote!r}") + + quoted = shlex.quote(path) + steps = [f"mkdir -p -- {quoted}"] + if delete: + # rsync --delete leaves the directory itself in place and clears + # what is under it, including dotfiles. `find -delete` is the + # same contract without a glob that would miss them. + steps.append(f"find {quoted} -mindepth 1 -delete") + steps.append(f"tar -C {quoted} -xpf - --no-same-owner") + # POSIX `set -eu`, not `set -euo pipefail`: `ssh host ""` runs the + # command in the remote account's login shell, which is not guaranteed to + # be bash — dash answers `-o pipefail` with "Illegal option", the same + # way a container job did in #886. There is no pipeline here to protect, + # so the bash-only option bought nothing. + script = "set -eu; " + "; ".join(steps) + + handle, archive_name = tempfile.mkstemp(prefix="nexus-push-", suffix=".tar") + os.close(handle) # tar writes the path; the descriptor is only how mkstemp reserves it + archive = Path(archive_name) + try: + subprocess.run( + ["tar", "-C", str(local), "-cf", archive_name, "."], + check=True, + capture_output=True, + text=True, + timeout=timeout, + ) + with archive.open("rb") as payload: + return subprocess.run( + ["ssh", host, script], + check=True, + stdin=payload, + capture_output=True, + text=True, + timeout=timeout, + ) + finally: + archive.unlink(missing_ok=True) diff --git a/src/nexus_deploy/orchestrator.py b/src/nexus_deploy/orchestrator.py index 7f39bba0..e6332237 100644 --- a/src/nexus_deploy/orchestrator.py +++ b/src/nexus_deploy/orchestrator.py @@ -184,6 +184,14 @@ def _transport_detail(exc: BaseException) -> str: labels — never secret values — so their output is safe to surface. A phase whose script does not hold to that must not use this. """ + # A missing program is not a transport failure, and saying "transport + # (FileNotFoundError)" sends the reader to the network. That is what + # #897 cost an hour of: rsync was simply absent from the job image, + # and Python raises FileNotFoundError — an OSError, so it landed in + # the transport arm. subprocess puts the program name in `filename`. + if isinstance(exc, FileNotFoundError) and exc.filename: + return f"executable not found: {exc.filename} (not installed in this job image)" + parts: list[str] = [type(exc).__name__] returncode = getattr(exc, "returncode", None) if isinstance(returncode, int): diff --git a/src/nexus_deploy/ssh.py b/src/nexus_deploy/ssh.py index b682392a..9e4fed93 100644 --- a/src/nexus_deploy/ssh.py +++ b/src/nexus_deploy/ssh.py @@ -39,6 +39,8 @@ from pathlib import Path from types import TracebackType +from nexus_deploy import _remote + class SSHError(Exception): """Raised for SSH-side errors not modelled by ``CalledProcessError``. @@ -159,19 +161,15 @@ def rsync_to( ``delete=True`` clears destination paths that don't exist locally — used when the local dir is the canonical source-of-truth for that remote location. + + Delegates to :func:`nexus_deploy._remote.push_directory`, which + falls back to tar over ssh where rsync is absent (#897). This + method had its own copy of the rsync invocation until then — + identical, and identically broken on a job image without rsync. + One implementation means the next runner image surprises exactly + one place. """ - src = f"{local}/" if not str(local).endswith("/") else str(local) - args = ["rsync", "-aq"] - if delete: - args.append("--delete") - args += [src, remote] - return subprocess.run( - args, - check=True, - capture_output=True, - text=True, - timeout=timeout, - ) + return _remote.push_directory(local, remote, delete=delete, timeout=timeout) @contextmanager def port_forward( diff --git a/tests/unit/test_push_directory.py b/tests/unit/test_push_directory.py new file mode 100644 index 00000000..b5b4e2a7 --- /dev/null +++ b/tests/unit/test_push_directory.py @@ -0,0 +1,330 @@ +"""`_remote.push_directory` copies a directory with rsync, or with tar over ssh +where rsync is absent (#897). + +The tar path is exercised **for real**: a fake `ssh` on PATH runs the remote +command locally, so the test asserts which files arrived and with which modes, +not which arguments were assembled. That is the half worth testing — the +argument list was never the thing that broke. +""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +from pathlib import Path + +import pytest + +from nexus_deploy import _remote + +# `ssh ` where the command runs here instead of over the wire. +# It ignores the host, and passes stdin through, which is what carries the +# archive. +_FAKE_SSH = """#!/usr/bin/env bash +shift # drop the host +exec bash -c "$*" +""" + +_FAILING_SSH = """#!/usr/bin/env bash +echo "kex_exchange_identification: connection closed by remote host" >&2 +exit 255 +""" + + +def _fake_ssh(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str = _FAKE_SSH) -> Path: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + (bin_dir / "ssh").write_text(body) + (bin_dir / "ssh").chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + return bin_dir + + +def _no_rsync(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the fallback the only option, whatever the test machine has.""" + real_which = shutil.which + monkeypatch.setattr( + "nexus_deploy._remote.shutil.which", + lambda name, *a, **k: None if name == "rsync" else real_which(name, *a, **k), + ) + + +def _tree(root: Path) -> dict[str, str]: + return {str(p.relative_to(root)): p.read_text() for p in sorted(root.rglob("*")) if p.is_file()} + + +# --------------------------------------------------------------------------- +# Which transport +# --------------------------------------------------------------------------- + + +def test_rsync_is_used_when_it_is_installed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The GitHub path must not change: rsync present means rsync used.""" + calls: list[list[str]] = [] + monkeypatch.setattr("nexus_deploy._remote.shutil.which", lambda name, *a, **k: "/usr/bin/rsync") + monkeypatch.setattr( + "nexus_deploy._remote.subprocess.run", + lambda args, **kwargs: ( + calls.append(list(args)) # type: ignore[func-returns-value] + or subprocess.CompletedProcess(args, 0, "", "") + ), + ) + + _remote.push_directory(tmp_path, "nexus:/dst/", delete=True) + + assert calls == [["rsync", "-aq", "--delete", f"{tmp_path}/", "nexus:/dst/"]] + + +def test_the_fallback_needs_no_rsync_at_all( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The point of #897: the whole push must work on an image without it.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "src", tmp_path / "dst" + src.mkdir() + (src / "docker-compose.yml").write_text("services: {}\n") + + _remote.push_directory(src, f"nexus:{dst}") + + assert _tree(dst) == {"docker-compose.yml": "services: {}\n"} + + +# --------------------------------------------------------------------------- +# What actually arrives +# --------------------------------------------------------------------------- + + +def test_the_directory_contents_arrive_not_the_directory_itself( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """rsync's trailing-slash semantics, which four call sites rely on.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "stacks-kestra", tmp_path / "dst" + (src / "nested").mkdir(parents=True) + (src / "a.txt").write_text("a") + (src / "nested" / "b.txt").write_text("b") + + _remote.push_directory(src, f"nexus:{dst}") + + assert _tree(dst) == {"a.txt": "a", "nested/b.txt": "b"} + assert not (dst / "stacks-kestra").exists() + + +def test_dotfiles_come_along(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Every stack push carries a rendered `.env`. A glob-based copy would + silently leave it behind, and the stack would come up unconfigured.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "src", tmp_path / "dst" + src.mkdir() + (src / ".env").write_text("KESTRA_DB_PASSWORD=x\n") + + _remote.push_directory(src, f"nexus:{dst}") + + assert _tree(dst) == {".env": "KESTRA_DB_PASSWORD=x\n"} + + +def test_restrictive_modes_survive_the_trip( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The Infisical push writes 0600 files of secret values and relies on + the transport keeping them that way.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "src", tmp_path / "dst" + src.mkdir() + secret = src / "payload.json" + secret.write_text('{"k": "v"}') + secret.chmod(0o600) + + _remote.push_directory(src, f"nexus:{dst}") + + assert stat.S_IMODE((dst / "payload.json").stat().st_mode) == 0o600 + + +def test_the_destination_is_created_when_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """rsync creates the last path component; a bare `tar -x` would not.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "src", tmp_path / "deep" / "not" / "there" + src.mkdir() + (src / "f").write_text("x") + + _remote.push_directory(src, f"nexus:{dst}") + + assert _tree(dst) == {"f": "x"} + + +# --------------------------------------------------------------------------- +# delete=True +# --------------------------------------------------------------------------- + + +def test_delete_clears_what_is_no_longer_local( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Three callers push a directory they treat as the source of truth.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "src", tmp_path / "dst" + src.mkdir() + (src / "keep").write_text("new") + dst.mkdir() + (dst / "stale").write_text("old") + (dst / ".hidden-stale").write_text("old") + + _remote.push_directory(src, f"nexus:{dst}", delete=True) + + assert _tree(dst) == {"keep": "new"} + + +def test_without_delete_the_remote_keeps_what_is_there( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """stack-sync and the woodpecker push deliberately do not delete: the + server-side dir also holds files the deploy generated there.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src, dst = tmp_path / "src", tmp_path / "dst" + src.mkdir() + (src / "docker-compose.yml").write_text("new") + dst.mkdir() + (dst / "docker-compose.firewall.yml").write_text("generated on the server") + + _remote.push_directory(src, f"nexus:{dst}") + + assert _tree(dst) == { + "docker-compose.yml": "new", + "docker-compose.firewall.yml": "generated on the server", + } + + +# --------------------------------------------------------------------------- +# Failures +# --------------------------------------------------------------------------- + + +def test_a_failing_ssh_raises_with_its_stderr( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """stack_sync prints `exc.stderr` as the per-service diagnostic, so the + fallback has to fail the same shape as rsync did.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch, body=_FAILING_SSH) + src = tmp_path / "src" + src.mkdir() + (src / "f").write_text("x") + + with pytest.raises(subprocess.CalledProcessError) as caught: + _remote.push_directory(src, "nexus:/dst") + + assert caught.value.returncode == 255 + assert "kex_exchange_identification" in caught.value.stderr + + +def test_a_remote_without_a_path_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`nexus` alone would make `tar -C ''` run in the remote home dir.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch) + src = tmp_path / "src" + src.mkdir() + + with pytest.raises(ValueError, match=":"): + _remote.push_directory(src, "nexus") + + +def test_no_archive_is_left_behind(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """It holds a copy of whatever was pushed — for one caller, secrets.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch, body=_FAILING_SSH) + src = tmp_path / "src" + src.mkdir() + (src / "f").write_text("x") + # `tempfile.tempdir`, not the TMPDIR env var: tempfile resolves the + # directory once and caches it, so setting the variable here would have + # no effect — and the assertion below would then inspect a directory + # nothing ever wrote to and pass for the wrong reason. It did, until a + # mutation that removed the cleanup failed to fail. + spool = tmp_path / "tmp" + spool.mkdir() + monkeypatch.setattr("nexus_deploy._remote.tempfile.tempdir", str(spool)) + + with pytest.raises(subprocess.CalledProcessError): + _remote.push_directory(src, "nexus:/dst") + + assert list(spool.iterdir()) == [], "the archive outlived a failed push" + + +# `ssh ` that only records what it was asked to run. +_RECORDING_SSH = """#!/usr/bin/env bash +shift +printf '%s' "$*" > "$REMOTE_SCRIPT_LOG" +""" + + +def test_the_remote_script_runs_in_a_posix_shell( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`ssh host ""` runs the command in the remote account's login + shell, which is not guaranteed to be bash. dash answers `-o pipefail` + with "Illegal option" — the same failure a container job hit in #886 — + and there is no pipeline in this script for pipefail to protect.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch, body=_RECORDING_SSH) + log = tmp_path / "remote-script.txt" + monkeypatch.setenv("REMOTE_SCRIPT_LOG", str(log)) + src = tmp_path / "src" + src.mkdir() + (src / "f").write_text("x") + + _remote.push_directory(src, "nexus:/dst", delete=True) + + script = log.read_text() + assert script.startswith("set -eu;") + assert "pipefail" not in script + # Still POSIX with a pipe would be a different question — there is none. + assert "|" not in script + + +def test_the_remote_path_is_quoted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The path reaches a remote shell as part of a command string.""" + _no_rsync(monkeypatch) + _fake_ssh(tmp_path, monkeypatch, body=_RECORDING_SSH) + log = tmp_path / "remote-script.txt" + monkeypatch.setenv("REMOTE_SCRIPT_LOG", str(log)) + src = tmp_path / "src" + src.mkdir() + + _remote.push_directory(src, "nexus:/opt/docker server/stacks") + + assert "'/opt/docker server/stacks'" in log.read_text() + + +# --------------------------------------------------------------------------- +# The diagnostic that sent the reader to the network +# --------------------------------------------------------------------------- + + +def test_a_missing_executable_is_named_as_such() -> None: + """#897 read as `transport (FileNotFoundError)` for an hour. A missing + program is not a network problem and needs a different fix.""" + from nexus_deploy.orchestrator import _transport_detail + + try: + subprocess.run(["definitely-not-installed-xyz"], check=True) + except FileNotFoundError as exc: + detail = _transport_detail(exc) + + assert "definitely-not-installed-xyz" in detail + assert "transport" not in detail + assert "job image" in detail diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index cca23946..94fb618a 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil import subprocess from pathlib import Path from typing import Any @@ -62,8 +63,24 @@ def fake_run(*_args: Any, **kwargs: Any) -> subprocess.CompletedProcess[str]: assert result.returncode == 1 +@pytest.fixture +def rsync_installed(monkeypatch: pytest.MonkeyPatch) -> None: + """Pin the transport for the tests below. + + Since #897 `push_directory` picks tar-over-ssh when rsync is absent, and + CI for a Conductor fork runs on a job image where it IS absent. Without + this, these tests would pass on GitHub and fail there — on the very + machine the fallback exists for. + """ + real_which = shutil.which + monkeypatch.setattr( + "nexus_deploy._remote.shutil.which", + lambda name, *a, **k: "/usr/bin/rsync" if name == "rsync" else real_which(name, *a, **k), + ) + + def test_rsync_to_remote_appends_trailing_slash( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rsync_installed: None ) -> None: """``Path('/foo')`` is normalised to ``/foo/`` so rsync copies the directory contents (not the directory itself, which would land at @@ -86,6 +103,7 @@ def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: def test_rsync_to_remote_preserves_existing_trailing_slash( monkeypatch: pytest.MonkeyPatch, + rsync_installed: None, ) -> None: """Passing a string path that already ends in ``/`` is left alone.""" captured: dict[str, Any] = {} @@ -101,7 +119,9 @@ def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: assert captured["args"][-2].endswith("/") -def test_rsync_to_remote_delete_flag(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rsync_to_remote_delete_flag( + monkeypatch: pytest.MonkeyPatch, rsync_installed: None +) -> None: captured: dict[str, Any] = {} def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: @@ -175,7 +195,9 @@ def test_ssh_run_actually_invokes_subprocess(tmp_path: Path) -> None: assert "err-line" in result.stdout -def test_rsync_to_remote_no_delete_by_default(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rsync_to_remote_no_delete_by_default( + monkeypatch: pytest.MonkeyPatch, rsync_installed: None +) -> None: captured: dict[str, Any] = {} def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: diff --git a/tests/unit/test_ssh.py b/tests/unit/test_ssh.py index e20cf61e..9a892d72 100644 --- a/tests/unit/test_ssh.py +++ b/tests/unit/test_ssh.py @@ -7,6 +7,7 @@ from __future__ import annotations +import shutil import socket import subprocess import threading @@ -116,9 +117,27 @@ def fake_run(*_args: Any, **kwargs: Any) -> subprocess.CompletedProcess[str]: # -- rsync_to ----------------------------------------------------------- +# +# These three pin the rsync invocation, so they have to pin the transport +# too: since #897 `push_directory` picks tar-over-ssh when rsync is absent, +# and CI for a Conductor fork runs on a job image where it IS absent. Without +# this fixture they would pass on GitHub and fail there — on the very machine +# the fallback exists for. -def test_rsync_to_appends_trailing_slash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.fixture +def rsync_installed(monkeypatch: pytest.MonkeyPatch) -> None: + + real_which = shutil.which + monkeypatch.setattr( + "nexus_deploy._remote.shutil.which", + lambda name, *a, **k: "/usr/bin/rsync" if name == "rsync" else real_which(name, *a, **k), + ) + + +def test_rsync_to_appends_trailing_slash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rsync_installed: None +) -> None: captured: dict[str, Any] = {} def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: @@ -134,7 +153,7 @@ def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: assert cmd[-1] == "nexus:/dst/" -def test_rsync_to_delete_flag(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rsync_to_delete_flag(monkeypatch: pytest.MonkeyPatch, rsync_installed: None) -> None: captured: dict[str, Any] = {} def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: @@ -146,7 +165,9 @@ def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: assert "--delete" in captured["args"] -def test_rsync_to_no_delete_by_default(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rsync_to_no_delete_by_default( + monkeypatch: pytest.MonkeyPatch, rsync_installed: None +) -> None: captured: dict[str, Any] = {} def fake_run(*args: Any, **_kwargs: Any) -> subprocess.CompletedProcess[str]: