From 18357c407eb02e5d7a57091b834921b316c8b623 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 3 Aug 2026 22:59:38 +0000 Subject: [PATCH] feat(sandbox-vms): wire Coder workspaces to docker-vm and talos-vm sandboxes Automates the manual setup that made the KubeVirt sandbox VMs (docker-vm for DOCKER_HOST over SSH, talos-vm for a disposable Talos cluster) usable from a Coder workspace, so it survives a workspace rebuild instead of being redone from memory: - ~/.ssh/config + a dedicated known_hosts.sandbox pin docker-vm's now-stable host key and point IdentityFile at a dedicated key (IdentitiesOnly, no ssh-agent -- that's per-shell and doesn't survive a new terminal). - DOCKER_HOST is set once the sandbox key exists (private_dot_env.tmpl). - talosctl gets its context/endpoint/node fixed up every apply (run_after_63_sandbox.sh.tmpl), since a talosconfig restored from Bitwarden can come back with an empty context field. - The sandbox kubeconfig is merged into ~/.kube/config using a temp-file flatten (KUBECONFIG=a:b kubectl config view --flatten > a truncates a before it's read) with the existing config listed first so current-context is never silently switched onto the sandbox. - act gets an actrc mapping ubuntu-24.04 (these repos' runs-on pin) and act's own "Medium" tier, so nobody hits the first-run image-size prompt. - controlplane.yaml (which embeds the sandbox cluster's CA private key) deliberately stays off the shared, RWX workspace home PVC by default -- fetch_sandbox-talos-machine-config fetches it via bws on demand instead. The talosconfig alone covers day-to-day use; the machine config is only needed for disaster recovery. Every network call against the VMs (only the kubeconfig fetch makes one) is timeout-bounded and fails soft, so a down sandbox never blocks a workspace from coming up -- verified by pointing the endpoint at a dead host and confirming the script logs a warning and exits 0 with the kubeconfig untouched. Known gap: sandbox_docker_vm_ssh_private_key does not exist yet in Bitwarden Secrets Manager (only its public half was ever uploaded), so private_sandbox_docker_vm.tmpl carries a placeholder secret ID and a TODO with the exact command to create it. Everything else in this change was exercised live against the real VMs. Closes #710 Co-Authored-By: Claude Opus 5 (1M context) --- .chezmoiscripts/run_after_63_sandbox.sh.tmpl | 104 ++++++++++++++++++ CLAUDE.md | 10 +- private_dot_config/act/actrc | 10 ++ private_dot_env.tmpl | 7 ++ ...cutable_fetch-sandbox-talos-machine-config | 63 +++++++++++ private_dot_ssh/config | 17 +++ private_dot_ssh/known_hosts.sandbox | 7 ++ .../private_sandbox_docker_vm.pub.tmpl | 1 + .../private_sandbox_docker_vm.tmpl | 12 ++ private_dot_talos/private_config.tmpl | 1 + 10 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 .chezmoiscripts/run_after_63_sandbox.sh.tmpl create mode 100644 private_dot_config/act/actrc create mode 100755 private_dot_local/bin/executable_fetch-sandbox-talos-machine-config create mode 100644 private_dot_ssh/config create mode 100644 private_dot_ssh/known_hosts.sandbox create mode 100644 private_dot_ssh/private_sandbox_docker_vm.pub.tmpl create mode 100644 private_dot_ssh/private_sandbox_docker_vm.tmpl create mode 100644 private_dot_talos/private_config.tmpl diff --git a/.chezmoiscripts/run_after_63_sandbox.sh.tmpl b/.chezmoiscripts/run_after_63_sandbox.sh.tmpl new file mode 100644 index 00000000..bffed644 --- /dev/null +++ b/.chezmoiscripts/run_after_63_sandbox.sh.tmpl @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +set -o allexport +{{ template "aqua.env" . -}} +{{ template "krew.env" . -}} +{{ template "homebrew.env" . -}} +set +o allexport +export PATH=${AQUA_ROOT_DIR}/bin:${KREW_ROOT}/bin:$HOME/.local/bin:${HOMEBREW_PREFIX}/opt/coreutils/libexec/gnubin:${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:$PATH + +{{ template "script_header.sh" . }} + +# ppat/dotfiles#710: wires talosctl + kubectl at the KubeVirt sandbox Talos cluster (talos-vm). +# docker-vm's SSH transport is configured declaratively via private_dot_ssh/config and needs no +# script. Every step below must degrade gracefully when the sandbox is down or doesn't exist yet +# -- a workspace must never fail to come up because a disposable sandbox VM is unreachable. +readonly SANDBOX_TALOS_CONTEXT="sandbox-talos" +readonly SANDBOX_TALOS_ENDPOINT="talos-vm.sandbox-talos.svc.cluster.local" + +configure_talosctl() { + if [[ ! -f "$HOME/.talos/config" ]]; then + log_warning "Sandbox | ~/.talos/config not present, skipping talosctl setup." + return 0 + fi + + log_info "Sandbox | Configuring talosctl context/endpoint/node..." + # A talosconfig restored from Bitwarden can come back with an empty top-level `context:` field + # (error constructing client: default context "" not found in config) -- so this must run every + # apply, not just once, and stay idempotent when context/endpoint/node are already correct. + talosctl config context "${SANDBOX_TALOS_CONTEXT}" 2>&1 | sed -E 's|^(.*)| \1|g' + talosctl config endpoint "${SANDBOX_TALOS_ENDPOINT}" 2>&1 | sed -E 's|^(.*)| \1|g' + talosctl config node "${SANDBOX_TALOS_ENDPOINT}" 2>&1 | sed -E 's|^(.*)| \1|g' + log_success "Sandbox | talosctl configured for context ${SANDBOX_TALOS_CONTEXT}." +} + +merge_sandbox_kubeconfig() { + if [[ ! -f "$HOME/.talos/config" ]]; then + return 0 + fi + + # Deliberately not using a RETURN/EXIT trap for cleanup here: bash traps aren't function-scoped, + # so a trap set here would still be armed (referencing these now-unset `local`s, fatal under + # `set -u`) when a *later*, unrelated function in this script returns. Clean up explicitly at + # every exit point instead. + local tmp_kubeconfig + tmp_kubeconfig=$(mktemp "sandbox-kubeconfig.$(date +%Y%m%d_%H%M%S).XXXXX" -p /tmp) + + log_info "Sandbox | Fetching sandbox-talos kubeconfig..." + # talosctl kubeconfig actually contacts the Talos API (port 50000), unlike the local-only + # `talosctl config` commands above -- this is the one call in this script that can hang or fail + # if the VM is down, so it's bounded by a timeout and any failure just skips the merge below, + # leaving the existing kubeconfig untouched, rather than failing the whole chezmoi apply. + if ! timeout 10 talosctl --context "${SANDBOX_TALOS_CONTEXT}" kubeconfig "${tmp_kubeconfig}" --force --merge=false > /dev/null 2>&1; then + log_warning "Sandbox | talos-vm is unreachable, skipping kubeconfig merge. Existing kubeconfig left untouched." + rm -f "${tmp_kubeconfig}" + return 0 + fi + + if [[ ! -d "$HOME/.kube" ]]; then + mkdir -p "$HOME/.kube" + fi + if [[ ! -f "$HOME/.kube/config" ]]; then + : > "$HOME/.kube/config" + fi + + log_info "Sandbox | Merging sandbox-talos context into ~/.kube/config..." + cp "$HOME/.kube/config" "$HOME/.kube/config.bak" + + # `KUBECONFIG=a:b kubectl config view --flatten > a` truncates `a` before it's fully read, so + # merge into a temp file first and move it into place -- never redirect straight into the target. + # The existing config is listed first so its current-context wins the merge (see below). + local flattened + flattened=$(mktemp "kubeconfig-flattened.$(date +%Y%m%d_%H%M%S).XXXXX" -p /tmp) + KUBECONFIG="$HOME/.kube/config:${tmp_kubeconfig}" kubectl config view --flatten > "${flattened}" + mv "${flattened}" "$HOME/.kube/config" + chmod 600 "$HOME/.kube/config" + rm -f "${tmp_kubeconfig}" + + # Never let a fresh merge silently switch the default context: the OIDC prod contexts + # (run_after_61_kubeconfig.sh.tmpl) must stay current-context so a stray command fails instead + # of landing on the sandbox. AI agents run in this same environment, so this stays deliberate. + local current_context + current_context=$(kubectl config current-context 2>/dev/null || echo "") + if [[ "${current_context}" == "" ]]; then + log_info "Sandbox | No default context set, falling back to homelab..." + kubectl config use-context homelab 2>&1 | sed -E 's|^(.*)| \1|g' + elif [[ "${current_context}" == "admin@${SANDBOX_TALOS_CONTEXT}" ]]; then + log_warning "Sandbox | current-context landed on the sandbox cluster, resetting to homelab." + kubectl config use-context homelab 2>&1 | sed -E 's|^(.*)| \1|g' + fi + + log_success "Sandbox | Kubeconfig merged; sandbox cluster reachable as context admin@${SANDBOX_TALOS_CONTEXT}." +} + +main() { + log_info "Sandbox | Setting up sandbox VM access..." + configure_talosctl + echo + merge_sandbox_kubeconfig + echo "-------------------------------------------------------------------------------------------" + echo +} + +main diff --git a/CLAUDE.md b/CLAUDE.md index 70ca137a..cd0d354d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,9 +83,13 @@ dotenv-linter --skip QuoteCharacter Every credential is pulled via the `bitwardenSecrets` template function keyed by a fixed secret UUID, resolved only at `chezmoi apply` time — never hardcoded. See [DESIGN.md](DESIGN.md#secrets-bitwarden-secrets-manager-not-plaintext) -for why. In practice: `private_dot_env.secrets.tmpl` (API keys, cloud creds, Terraform vars) and -`.chezmoiscripts/run_after_61_kubeconfig.sh.tmpl` (OIDC client credentials) are the two places new secret -references get added. +for why. In practice: `private_dot_env.secrets.tmpl` (API keys, cloud creds, Terraform vars), +`.chezmoiscripts/run_after_61_kubeconfig.sh.tmpl` (OIDC client credentials), and `private_dot_ssh/` / +`private_dot_talos/` (sandbox VM SSH key and talosconfig, see `.chezmoiscripts/run_after_63_sandbox.sh.tmpl`) +are where new secret references get added. Not every credential is meant to land on disk automatically, though: +`private_dot_local/bin/executable_fetch-sandbox-talos-machine-config` fetches its secret on demand via `bws` +at *invocation* time rather than at `chezmoi apply` time, specifically to keep a CA private key off the +(shared, RWX) workspace home PVC by default — see the script's own header comment for the reasoning. ## Renovate diff --git a/private_dot_config/act/actrc b/private_dot_config/act/actrc new file mode 100644 index 00000000..251a291f --- /dev/null +++ b/private_dot_config/act/actrc @@ -0,0 +1,10 @@ +# ppat/dotfiles#710: shipped so act never hits its first-run interactive image-size prompt (which +# writes its answer to this same file) and so nobody picks the Micro image (<200MB, NodeJS-only, +# insufficient for real runs) by accident. This is act's own "Medium" tier -- the size it +# recommends -- plus an ubuntu-24.04 mapping act's default platform map doesn't have yet: these +# repos pin `runs-on: ubuntu-24.04`, which act only resolves once ppat/github-workflows#577 lands. +-P ubuntu-latest=catthehacker/ubuntu:act-latest +-P ubuntu-24.04=catthehacker/ubuntu:act-24.04 +-P ubuntu-22.04=catthehacker/ubuntu:act-22.04 +-P ubuntu-20.04=catthehacker/ubuntu:act-20.04 +-P ubuntu-18.04=catthehacker/ubuntu:act-18.04 diff --git a/private_dot_env.tmpl b/private_dot_env.tmpl index c3a4f6e4..3d9162d0 100644 --- a/private_dot_env.tmpl +++ b/private_dot_env.tmpl @@ -46,6 +46,13 @@ LESSHISTFILE=/dev/null RIPGREP_CONFIG_PATH={{ .chezmoi.homeDir }}/.ripgreprc {{- end }} +# ppat/dotfiles#710: KubeVirt sandbox docker-vm, reached over SSH via private_dot_ssh/config. +# Guarded on the sandbox key existing so this stays unset (rather than pointing at a dead host) +# on a machine that hasn't fetched sandbox_docker_vm_ssh_private_key from Bitwarden yet. +{{- if stat (joinPath .chezmoi.homeDir ".ssh/sandbox_docker_vm") }} +DOCKER_HOST=ssh://docker@docker-vm.sandbox-docker.svc.cluster.local +{{- end }} + {{ if eq .chezmoi.os "linux" -}} # X11: Use X11 display if available DISPLAY=:0 diff --git a/private_dot_local/bin/executable_fetch-sandbox-talos-machine-config b/private_dot_local/bin/executable_fetch-sandbox-talos-machine-config new file mode 100755 index 00000000..4ead80c5 --- /dev/null +++ b/private_dot_local/bin/executable_fetch-sandbox-talos-machine-config @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ppat/dotfiles#710: fetches controlplane.yaml (the sandbox Talos cluster's machine config) from +# Bitwarden on demand, rather than shipping it to every workspace via chezmoi apply. +# +# controlplane.yaml contains the sandbox cluster's CA private key. The talosconfig alone +# (~/.talos/config, chezmoi-managed) is sufficient for all day-to-day use; this file is only +# needed for disaster recovery -- `talosctl gen secrets --from-controlplane-config` reconstructs +# the secrets bundle from it if the talosconfig is ever lost, regenerating a client cert against +# the running cluster since the CA itself is unchanged. The workspace home PVC is shared RWX +# across workspaces, so this deliberately isn't part of the default chezmoi apply: fetch it only +# when you actually need it, and don't leave it lying around afterward. + +usage() { + cat < /dev/null; then + echo "ERROR: bws binary is not in path, cannot proceed!" >&2 + exit 1 + fi + if ! command -v chezmoi > /dev/null; then + echo "ERROR: chezmoi binary is not in path, cannot proceed!" >&2 + exit 1 + fi + if ! command -v jq > /dev/null; then + echo "ERROR: jq binary is not in path, cannot proceed!" >&2 + exit 1 + fi + + local access_token + access_token=$(chezmoi data --format=json | jq -r .bwsAccessToken) + if [[ -z "${access_token}" || "${access_token}" == "null" ]]; then + echo "ERROR: could not resolve bwsAccessToken from 'chezmoi data' -- run 'chezmoi init' first." >&2 + exit 1 + fi + + echo "Fetching sandbox_talos_vm_machine_config from Bitwarden..." + BWS_ACCESS_TOKEN="${access_token}" bws secret get 039ae8ac-21d4-4ed8-af12-b49a012c0d12 -o json | jq -r .value > "${destination}" + chmod 600 "${destination}" + + echo "Wrote ${destination} (mode 0600)." + echo "WARNING: this file contains the sandbox cluster's CA private key. It is only needed for" + echo "disaster recovery (talosctl gen secrets --from-controlplane-config) -- delete it when done:" + echo " rm -f ${destination}" +} + +main "$@" diff --git a/private_dot_ssh/config b/private_dot_ssh/config new file mode 100644 index 00000000..dff90f1d --- /dev/null +++ b/private_dot_ssh/config @@ -0,0 +1,17 @@ +# ppat/dotfiles#710: dedicated Host block for the KubeVirt sandbox docker-vm. +# +# DOCKER_HOST=ssh://docker@docker-vm.sandbox-docker.svc.cluster.local (private_dot_env.tmpl) fails +# with "Permission denied (publickey)" without this: the key isn't at a default filename, so +# ssh:// won't find it unaided, and ssh-agent doesn't help either since it's per-shell and doesn't +# survive a new terminal. IdentitiesOnly keeps ssh from offering any other identity first. +# +# UserKnownHostsFile points at a dedicated file (not the main known_hosts) so this repo can pin +# the VM's fingerprint declaratively -- chezmoi fully owns known_hosts.sandbox and overwrites it +# on every apply, which would be destructive against the main known_hosts file where ssh itself +# accumulates entries for every other host. +Host docker-vm docker-vm.sandbox-docker.svc.cluster.local + HostName docker-vm.sandbox-docker.svc.cluster.local + User docker + IdentityFile ~/.ssh/sandbox_docker_vm + IdentitiesOnly yes + UserKnownHostsFile ~/.ssh/known_hosts.sandbox ~/.ssh/known_hosts diff --git a/private_dot_ssh/known_hosts.sandbox b/private_dot_ssh/known_hosts.sandbox new file mode 100644 index 00000000..7694a344 --- /dev/null +++ b/private_dot_ssh/known_hosts.sandbox @@ -0,0 +1,7 @@ +# ppat/dotfiles#710: pinned host key for the KubeVirt sandbox docker-vm. +# +# Supplied via cloud-init `ssh_keys:` from Bitwarden, so it's stable across VM restarts (unlike a +# freshly re-provisioned VM, which would get a new host key every time). That stability is what +# makes pinning it here worthwhile instead of clearing StrictHostKeyChecking reflexively. +# Fingerprint: SHA256:uCqIXbsfvNMxdq8RPjy/CAiUFNFezWbNt+I8JtcGxt4 (verified live 2026-08-03). +docker-vm.sandbox-docker.svc.cluster.local ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFu5PJPAhtfmcbCFhotGVA/t56laYRddeia/6YaZvAL5 diff --git a/private_dot_ssh/private_sandbox_docker_vm.pub.tmpl b/private_dot_ssh/private_sandbox_docker_vm.pub.tmpl new file mode 100644 index 00000000..79dfd707 --- /dev/null +++ b/private_dot_ssh/private_sandbox_docker_vm.pub.tmpl @@ -0,0 +1 @@ +{{ (bitwardenSecrets "05814710-d9d1-4ca2-8961-b49a00467af8" .bwsAccessToken).value }} diff --git a/private_dot_ssh/private_sandbox_docker_vm.tmpl b/private_dot_ssh/private_sandbox_docker_vm.tmpl new file mode 100644 index 00000000..fb1be4eb --- /dev/null +++ b/private_dot_ssh/private_sandbox_docker_vm.tmpl @@ -0,0 +1,12 @@ +{{- /* +TODO(ppat/dotfiles#710): "sandbox_docker_vm_ssh_private_key" does not exist yet in Bitwarden +Secrets Manager -- only its public half ("sandbox_docker_vm_ssh_public_key", +05814710-d9d1-4ca2-8961-b49a00467af8) was ever uploaded, so this file cannot render until the +private key is added. Create it once, from a machine that still has the key (it was verified +working from ~/.ssh/sandbox_docker_vm during the 2026-08-02 live test): + bws secret create sandbox_docker_vm_ssh_private_key "$(cat ~/.ssh/sandbox_docker_vm)" \ + e9c6c45e-e8d9-480c-b2cf-b204011e80e6 \ + --note "Client private key for SSH auth as 'docker' to docker-vm (sandbox-docker). Public half: sandbox_docker_vm_ssh_public_key." +then replace the placeholder secret ID below with the ID bws returns. +*/ -}} +{{ (bitwardenSecrets "TODO-REPLACE-WITH-sandbox_docker_vm_ssh_private_key-SECRET-ID" .bwsAccessToken).value }} diff --git a/private_dot_talos/private_config.tmpl b/private_dot_talos/private_config.tmpl new file mode 100644 index 00000000..08a4fc2f --- /dev/null +++ b/private_dot_talos/private_config.tmpl @@ -0,0 +1 @@ +{{ (bitwardenSecrets "6c1b1020-2777-4e61-9e66-b49a012c3bad" .bwsAccessToken).value }}