fix(update): retry the Windows self-update in copy link mode after a hardlink error (#786)

plugins/kbagent/skills/kbagent/references/gotchas.md

@@ -4286,6 +4286,35 @@ applied. `kbagent update` printed `(scheduled)` and every later launch printed
   redirected-output encoding all landed there.
 - POSIX was never affected; it uses the inline install plus re-exec.
 
+## Windows self-update on a OneDrive / cloud-synced profile can delete kbagent
+
+*(since vNEXT, #786)*
+
+If a Windows user reports that `kbagent` vanished after a background update,
+check `%LOCALAPPDATA%\keboola-agent-cli\keboola-agent-cli\pending_update.log`
+for `os error 396` ("The cloud operation cannot be performed on a file with
+incompatible hardlinks"). The uv cache or tool directory sits on a
+cloud-synced volume that cannot hardlink, and `uv tool install --force
+--reinstall` had already removed the old tool venv when the install failed.
+
+- **Fixed:** the Windows self-update keeps uv's default hardlink mode. When the
+  install fails and uv reports a hardlink failure, the background helper runs
+  the same install once more with `--link-mode copy`; `pending_update.log`
+  holds both attempts. The recovery command kbagent prints after a failure
+  also passes `--link-mode copy`. A `UV_LINK_MODE` the user set is respected
+  (no retry, no flag). POSIX command lines are unchanged. The in-place path
+  (`KBAGENT_DEFER_UPDATE=0`) gets no retry.
+- **The update into the fixed release still runs the old code**, which has no
+  retry. An affected user can set the variable permanently before that update
+  (PowerShell, then open a new shell):
+  `[Environment]::SetEnvironmentVariable('UV_LINK_MODE', 'copy', 'User')`.
+- **A user stranded by an older version** must reinstall with copy mode; the
+  printed recovery command fails identically without it. PowerShell:
+
+      $env:UV_LINK_MODE="copy"; uv tool install --force --reinstall "keboola-cli @ https://github.com/keboola/cli/releases/download/v<version>/keboola_cli-<version>-py3-none-any.whl"
+
+  POSIX-style shells (Git Bash): `UV_LINK_MODE=copy uv tool install ...`.
+
 ## Source files are read as UTF-8, not the host codepage (since v0.80.3)
 
 `lineage build` reads `transform.sql` / `code.py` as UTF-8 regardless of the

src/keboola_agent_cli/auto_update.py

@@ -34,6 +34,7 @@
     KbagentUpdatePlan,
     _fetch_kbagent_latest_version,
     _is_up_to_date,
+    build_hardlink_retry_command,
     build_kbagent_upgrade_command,
     get_update_timeout,
     prepare_kbagent_update_plan,
@@ -414,6 +415,7 @@ def _schedule_deferred_update(plan: KbagentUpdatePlan) -> None:
         target_version=target,
         install_command=plan.command,
         recovery_command=plan.recovery_command,
+        hardlink_retry_command=build_hardlink_retry_command(plan.command),
     )
     if request_deferred_update(request):
         sys.stderr.write(

src/keboola_agent_cli/constants.py

@@ -456,6 +456,13 @@ def _resolve_app_name() -> str:
 # A marker older than this whose helper never wrote an exit file is treated as
 # lost (helper killed, machine rebooted mid-wait) and reported once.
 DEFERRED_UPDATE_STALE_SECONDS: int = 86400
+# Text in uv's error chain when a hardlink failed and uv did not fall back
+# ("Caused by: failed to hardlink file from ... (os error 396)", issue #786,
+# OneDrive / cloud-synced volume). The helper retries the install once in copy
+# link mode only when a failed install printed this. Matched case-sensitively:
+# uv's warning "Failed to hardlink files; falling back to full copy" means the
+# fallback worked, so it must not trigger a retry.
+DEFERRED_UPDATE_HARDLINK_FAILURE_TEXT: str = "failed to hardlink file"
 
 # --- Native (frozen / PyInstaller) distribution ---
 # kbagent also ships as a self-contained PyInstaller binary with NO Python

src/keboola_agent_cli/services/version_service.py

@@ -139,6 +139,51 @@ def get_update_timeout() -> float:
     return float(UPDATE_TIMEOUT_SECONDS)
 
 
+def _uv_link_mode_args() -> tuple[str, ...]:
+    """Extra ``uv tool install`` args forcing copy link mode on Windows.
+
+    Issue #786: when the uv cache or the tool directory sits on a cloud-synced
+    volume (OneDrive Files-On-Demand and similar), uv's default hardlink mode
+    fails with ``os error 396`` (ERROR_CLOUD_FILE_INCOMPATIBLE_HARDLINKS).
+    By then ``--force --reinstall`` has already removed the old tool venv, so
+    kbagent disappears entirely.
+
+    Used only on the failure path: the one-time retry after a hardlink error
+    (:func:`build_hardlink_retry_command`) and the printed recovery command.
+    The regular install keeps uv's default, so users whose disks can hardlink
+    do not pay for the slower copy. Windows only: POSIX command lines stay
+    byte-identical. A user-set ``UV_LINK_MODE`` is respected -- uv reads it
+    itself, and an explicit flag would override the user's choice.
+    """
+    if os.name != "nt" or os.environ.get("UV_LINK_MODE"):
+        return ()
+    return ("--link-mode", "copy")
+
+
+def _is_uv_command(command: tuple[str, ...]) -> bool:
+    """Whether ``command`` runs uv (as opposed to the pip fallback)."""
+    executable = command[0].replace("\\", "/").rsplit("/", maxsplit=1)[-1].casefold()
+    return executable in {"uv", "uv.exe"}
+
+
+def build_hardlink_retry_command(command: tuple[str, ...] | None) -> tuple[str, ...] | None:
+    """Return ``command`` in copy link mode, for one retry after a hardlink error.
+
+    The deferred Windows helper runs this only when the first install failed
+    and uv's output reports a hardlink failure (issue #786).
+
+    Returns:
+        The retry argv, or ``None`` when no retry applies: no command, a pip
+        command, a non-Windows host, or a user-set ``UV_LINK_MODE``.
+    """
+    if command is None or not _is_uv_command(command):
+        return None
+    link_mode = _uv_link_mode_args()
+    if not link_mode:
+        return None
+    return (*command[:-1], *link_mode, command[-1])
+
+
 def build_kbagent_upgrade_command(
     *, prerelease: bool = False, target_version: str | None = None, wheel_url: str | None = None
 ) -> list[str] | None:
@@ -241,18 +286,30 @@ def _render_command(command: tuple[str, ...]) -> str:
 def _recovery_command(command: tuple[str, ...] | None, target_version: str | None) -> str | None:
     """Render an exact forced-reinstall command safe to copy after failure."""
     if command is not None:
-        executable = command[0].replace("\\", "/").rsplit("/", maxsplit=1)[-1].casefold()
-        if executable in {"uv", "uv.exe"}:
-            recovery = ("uv", *command[1:])
+        if _is_uv_command(command):
+            # The regular install keeps uv's default link mode; the recovery
+            # runs after a failure, so it gets copy mode on Windows (#786).
+            recovery = ("uv", *command[1:-1], *_uv_link_mode_args(), command[-1])
         else:
             prerelease = ("--prerelease=allow",) if "--pre" in command else ()
-            recovery = ("uv", "tool", "install", "--force", "--reinstall", *prerelease, command[-1])
+            recovery = (
+                "uv",
+                "tool",
+                "install",
+                "--force",
+                "--reinstall",
+                *_uv_link_mode_args(),
+                *prerelease,
+                command[-1],
+            )
         return _render_command(recovery)
     if target_version is None:
         return None
     extras = "[server]" if has_server_extras() else ""
     source = f"keboola-cli{extras} @ {KBAGENT_INSTALL_SOURCE}@v{target_version}"
-    return _render_command(("uv", "tool", "install", "--force", "--reinstall", source))
+    return _render_command(
+        ("uv", "tool", "install", "--force", "--reinstall", *_uv_link_mode_args(), source)
+    )
 
 
 def _fetch_kbagent_latest_version(
@@ -731,6 +788,7 @@ def _update_kbagent(plan: KbagentUpdatePlan) -> dict[str, Any]:
                     target_version=kbagent_latest or old_version,
                     install_command=plan.command,
                     recovery_command=plan.recovery_command,
+                    hardlink_retry_command=build_hardlink_retry_command(plan.command),
                 )
             )
             if scheduled:

src/keboola_agent_cli/update_runner.py

@@ -50,6 +50,7 @@
 from .constants import (
     DEFERRED_UPDATE_ABANDONED_MARKER,
     DEFERRED_UPDATE_EXIT_FILENAME,
+    DEFERRED_UPDATE_HARDLINK_FAILURE_TEXT,
     DEFERRED_UPDATE_LOG_FILENAME,
     DEFERRED_UPDATE_LOG_MAX_BYTES,
     DEFERRED_UPDATE_MARKER_FILENAME,
@@ -124,6 +125,9 @@ class DeferredUpdateRequest:
     target_version: str
     install_command: tuple[str, ...]
     recovery_command: str | None
+    #: ``install_command`` in copy link mode, run once only when the first
+    #: install fails on a uv hardlink error (issue #786). ``None`` = no retry.
+    hardlink_retry_command: tuple[str, ...] | None = None
 
 
 class DeferredUpdateStatus(Enum):
@@ -309,6 +313,30 @@ def quote_for_powershell(value: str) -> str:
     return "'" + value.replace("'", "''") + "'"
 
 
+def _hardlink_retry_lines(request: DeferredUpdateRequest) -> list[str]:
+    """PowerShell lines that retry the install once in copy link mode (issue #786).
+
+    They run only when the first install failed and its output carries uv's
+    hardlink error (case-sensitive ``-cmatch``, so uv's successful-fallback
+    warning does not count). A host that can hardlink keeps uv's faster
+    default. Both outputs go to the same log, with a notice between them.
+    """
+    if request.hardlink_retry_command is None:
+        return []
+    quoted_retry = " ".join(quote_for_powershell(part) for part in request.hardlink_retry_command)
+    failure_text = quote_for_powershell(DEFERRED_UPDATE_HARDLINK_FAILURE_TEXT)
+    notice = quote_for_powershell(
+        "kbagent: the install failed on a hardlink error; retrying with --link-mode copy"
+    )
+    return [
+        f"  if (($code -ne 0) -and ($output -cmatch [regex]::Escape({failure_text}))) {{",
+        f"    $output += {notice} + [Environment]::NewLine",
+        f"    $output += (& {quoted_retry} 2>&1 | Out-String -Width 4096)",
+        "    $code = $LASTEXITCODE",
+        "  }",
+    ]
+
+
 def build_waiter_script(
     request: DeferredUpdateRequest,
     *,
@@ -371,6 +399,7 @@ def build_waiter_script(
             f"  $output = (& {quoted_argv} 2>&1 | Out-String -Width 4096)",
             # Captured before anything else runs, so nothing can clobber it.
             "  $code = $LASTEXITCODE",
+            *_hardlink_retry_lines(request),
             (
                 "  [System.IO.File]::AppendAllText("
                 "$logFile, $output, (New-Object System.Text.UTF8Encoding $false))"

tests/test_auto_update.py

@@ -983,8 +983,12 @@ def _quiet_cache(self):
     @patch("keboola_agent_cli.auto_update.request_deferred_update", return_value=True)
     @patch("keboola_agent_cli.auto_update._perform_update")
     @patch("keboola_agent_cli.auto_update._re_exec")
+    @patch(
+        "keboola_agent_cli.auto_update.build_hardlink_retry_command",
+        return_value=("uv", "retry-in-copy-mode"),
+    )
     def test_schedules_instead_of_installing_in_place(
-        self, mock_reexec, mock_perform, mock_request, mock_defer, capsys
+        self, mock_retry, mock_reexec, mock_perform, mock_request, mock_defer, capsys
     ):
         maybe_auto_update()
 
@@ -993,6 +997,9 @@ def test_schedules_instead_of_installing_in_place(
         request = mock_request.call_args.args[0]
         assert request.from_version == "1.0.0"
         assert request.target_version == "2.0.0"
+        # The hardlink retry (#786) is derived from the same install command.
+        mock_retry.assert_called_once_with(request.install_command)
+        assert request.hardlink_retry_command == ("uv", "retry-in-copy-mode")
         assert "background" in capsys.readouterr().err
 
     @patch("keboola_agent_cli.auto_update.should_defer", return_value=True)

tests/test_update_runner.py

@@ -17,6 +17,7 @@
 import subprocess
 import sys
 import time
+from dataclasses import replace
 from pathlib import Path
 from typing import Any
 
@@ -211,6 +212,33 @@ def test_records_the_installer_exit_code(self, script: str) -> None:
     def test_a_powershell_level_error_is_recorded_as_a_failure(self, script: str) -> None:
         assert "Set-Content -LiteralPath $exitFile -Value 'failed'" in script
 
+    def test_runs_the_installer_once_without_a_retry_command(self, script: str) -> None:
+        """No retry command (pip, POSIX, user-set UV_LINK_MODE) -> one attempt (#786)."""
+        assert "--link-mode" not in script
+        assert script.count("$LASTEXITCODE") == 1
+
+    def test_retries_in_copy_mode_only_after_a_hardlink_failure(self, tmp_path: Path) -> None:
+        """Issue #786: one retry, and only when a failed install reports a hardlink error."""
+        cmd = REQUEST.install_command
+        retry = (*cmd[:-1], "--link-mode", "copy", cmd[-1])
+        script = build_waiter_script(
+            replace(REQUEST, hardlink_retry_command=retry),
+            pid=4321,
+            exit_file=tmp_path / DEFERRED_UPDATE_EXIT_FILENAME,
+            install_log=tmp_path / "install.log",
+        )
+        guard = (
+            "if (($code -ne 0) -and ($output -cmatch [regex]::Escape('failed to hardlink file'))) {"
+        )
+        quoted_retry = " ".join(quote_for_powershell(part) for part in retry)
+        assert guard in script
+        assert f"$output += (& {quoted_retry} 2>&1 | Out-String -Width 4096)" in script
+        # Between the first attempt and the log write, so the recorded exit
+        # code and the log cover both attempts.
+        first_attempt = script.index("$code = $LASTEXITCODE")
+        log_write = script.index("AppendAllText($logFile, $output")
+        assert first_attempt < script.index(guard) < log_write
+
 
 class TestBuildHelperCommand:
     """The helper must not read a profile, prompt, or show a window."""
@@ -515,6 +543,81 @@ def test_failing_installer_is_reported_not_swallowed(self, tmp_path: Path) -> No
 
         assert exit_file.read_text(encoding="utf-8").strip() == "3"
 
+    def _run_with_retry(
+        self, tmp_path: Path, first_install: str, retry_install: str
+    ) -> tuple[str, str]:
+        """Run the helper with a retry command; return (exit file, log)."""
+        exit_file = tmp_path / DEFERRED_UPDATE_EXIT_FILENAME
+        install_log = tmp_path / "install.log"
+        request = DeferredUpdateRequest(
+            from_version="1.0.0",
+            target_version="2.0.0",
+            install_command=(sys.executable, "-c", first_install),
+            recovery_command=None,
+            hardlink_retry_command=(sys.executable, "-c", retry_install),
+        )
+        self._run_helper(
+            build_waiter_script(
+                request,
+                pid=self._already_exited_pid(),
+                exit_file=exit_file,
+                install_log=install_log,
+                max_wait_seconds=60,
+                poll_seconds=1,
+                process_name="kbagent",
+            )
+        )
+        return (
+            exit_file.read_text(encoding="utf-8").strip(),
+            install_log.read_text(encoding="utf-8"),
+        )
+
+    # uv writes its errors to stderr, which PowerShell 5.1 wraps as ErrorRecords
+    # under `2>&1`, so the fake installers below write to stderr too.
+
+    def test_hardlink_error_runs_the_retry_command(self, tmp_path: Path) -> None:
+        """Issue #786: a uv hardlink error runs the retry, and its exit code counts."""
+        exit_code, log = self._run_with_retry(
+            tmp_path,
+            first_install=(
+                "import sys; sys.stderr.write('Caused by: failed to hardlink file from a to b: "
+                "The cloud operation cannot be performed (os error 396)'); sys.exit(2)"
+            ),
+            retry_install="print('RETRY-RAN')",
+        )
+
+        assert exit_code == "0"
+        assert "failed to hardlink file" in log
+        assert "retrying with --link-mode copy" in log
+        assert "RETRY-RAN" in log
+
+    def test_other_install_failures_are_not_retried(self, tmp_path: Path) -> None:
+        """Copy mode cannot fix a network or resolver error, so no retry runs."""
+        exit_code, log = self._run_with_retry(
+            tmp_path,
+            first_install=(
+                "import sys; sys.stderr.write('error: network unreachable'); sys.exit(3)"
+            ),
+            retry_install="print('RETRY-RAN')",
+        )
+
+        assert exit_code == "3"
+        assert "RETRY-RAN" not in log
+
+    def test_the_fallback_warning_alone_does_not_trigger_the_retry(self, tmp_path: Path) -> None:
+        """uv's "falling back to full copy" warning means copy mode already ran."""
+        exit_code, log = self._run_with_retry(
+            tmp_path,
+            first_install=(
+                "import sys; sys.stderr.write('warning: Failed to hardlink files; falling back "
+                "to full copy.\\nerror: network unreachable'); sys.exit(1)"
+            ),
+            retry_install="print('RETRY-RAN')",
+        )
+
+        assert exit_code == "1"
+        assert "RETRY-RAN" not in log
+
     def test_gives_up_without_installing_while_a_process_is_still_running(
         self, tmp_path: Path
     ) -> None: