From f14d7304f937ef62ee7a30ba436985981e88cc81 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 13:41:53 -0600 Subject: [PATCH 01/11] fix(inject): quote bootstrap inputs and preserve installed agents --- .github/workflows/pr-ci.yml | 13 ++ docs/issue-1621/README.md | 107 +++++++++ docs/issue-1621/inject-harness.go | 191 ++++++++++++++++ docs/issue-1621/matrix.py | 61 +++++ docs/issue-1621/real-tunnel.go | 163 +++++++++++++ docs/issue-1621/ssh-fixture.py | 71 ++++++ pkg/agent/inject.go | 47 +++- pkg/agent/inject_literal_test.go | 95 ++++++++ pkg/inject/inject.go | 4 +- pkg/inject/inject.sh | 192 +++++++++------- pkg/inject/script.go | 28 ++- pkg/inject/script_test.go | 215 ++++++++++++++++++ .../troubleshooting/linux-troubleshooting.mdx | 12 +- 13 files changed, 1097 insertions(+), 102 deletions(-) create mode 100644 docs/issue-1621/README.md create mode 100644 docs/issue-1621/inject-harness.go create mode 100644 docs/issue-1621/matrix.py create mode 100644 docs/issue-1621/real-tunnel.go create mode 100644 docs/issue-1621/ssh-fixture.py create mode 100644 pkg/agent/inject_literal_test.go create mode 100644 pkg/inject/script_test.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index acd0d26358..05e9d396c9 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -26,6 +26,7 @@ jobs: filters: | go: - '**/*.go' + - 'pkg/inject/**' - 'go.mod' - 'go.sum' - '.goreleaser.yml' @@ -121,6 +122,18 @@ jobs: printf '%s\n' "$packages" | xargs go test -race -short + - name: test agent bootstrap with BusyBox sh + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y busybox-static + original_sh="$(readlink /bin/sh)" + trap 'sudo ln -sf "$original_sh" /bin/sh' EXIT + sudo ln -sf /bin/busybox /bin/sh + go test -race -count=1 -timeout 90s ./pkg/inject -run '^(TestBootstrapSuite|TestInjectScript_RealShell.*)$' + - name: test E2E runtime helpers run: go test -race -count=1 ./e2e/framework/runenv/... ./e2e/framework/dockerwsl ./e2e/cmd/devsy-ci diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md new file mode 100644 index 0000000000..17db6c9220 --- /dev/null +++ b/docs/issue-1621/README.md @@ -0,0 +1,107 @@ +# Issue #1621 verification and release coordination + +This evidence concerns the SSH provider plus core agent injection. It does not qualify every provider. No release, merge, or managed activation is authorized by this document. + +## Actual Linux fixture + +The 2026-10-20 run used disposable Ubuntu 26.04 aarch64 containers on a macOS host, OpenSSH 10.2p1, Bash 5.3.9, Zsh 5.9, Fish 4.2.1, Dash `/bin/sh`, and BusyBox 1.37.0. Five distinct non-root passwd entries used `/bin/{bash,zsh,fish,dash,ash}`. Password authentication, PAM, root login, and MOTD were disabled; public-key authentication used fresh temporary keys. Only an ephemeral localhost port was published. No host accounts, servers, shell configuration, or SSH keys were changed. + +Independent clones tested BusyBox `/bin/sh`, missing `/bin/sh`, and non-executable `/bin/sh`; the shared server retained Dash. Runtime results and credentials remained outside the repository. The fixture image actually used was the cached `mcr.microsoft.com/devcontainers/base@sha256:91eff046d5c12318fc98e59258982b87ba896c467427ed8b5dab5a02107a5db7`. The helper defaults to the cached GHCR base and never pulls images; OS packages are installed inside the disposable container. + +## Baseline and corrected transport + +Original core v1.24.0 (`1f47cd185bd134b45cdce12b947f6c601d76d329`) and original provider v1.1.44 (`e37a6c68a695d9d1d3806b2d42a755750816f60e`) were independently exported with `git archive` and built from immutable temporary sources. Working-tree implementation edits did not enter the baseline builds. + +| Actual passwd shell | Original external SSH | Original built-in SSH | +| --- | --- | --- | +| Bash | Success: `ping`, `ARM-true`, `done`, payload output | Same success | +| Zsh | Exit 127; `main:7: command not found: sh -c`; raw stdout only `ping\n` | Exit 127; raw `ping\n`; remote diagnostic lost | +| Fish | Exit 127; unsupported assignment syntax; no stdout | Exit 127; no stdout; diagnostic lost | + +For successful baseline injection the transferred synthetic executable printed `agent-started\ntunnel-after-handshake\n`. Baseline failure surfaced as `read inject signal: EOF` for Zsh and `read ping: EOF` for Fish. Baseline used original raw injection commands, without a fixture-side POSIX wrapper. + +The corrected matrix used the actual provider worker's executable, not a substitute SSH launcher, plus the real core `GenerateScript` and `Inject` APIs. A synthetic Python executable distinguished protocol/byte verification from real Devsy startup. + +- Five passwd shells × external/built-in SSH × cold/warm/forced replacement: **30/30 passed**. Each returned exactly 4,110 bytes: 4 KiB containing all byte values, `READY\n`, then `LIVE-OK\n`. The host sent the follow-on request only after receiving output. SHA-256: `7085d9188f2d7d361c78e10a5075b746a0004d17a740085b0363dc49c5c14b43`. +- Final `chmod -- +x` core rerun: **30/30 passed** with random 4 KiB including NUL, quotes, dollar, backtick, CR, newline and backslash; hostile literal destination paths in every cell. Exact output SHA-256: `4d77b6416f4dfdf3a243a3cc27c79dced201dc7b01462e104b1b0d53af6ae9a4`. +- Cold/replacement requested one payload; warm requested none. All successful normal cells had empty stderr and error, with raw `ping\nARM-true\ndone\n` or warm `ping\ndone\n` before payload bytes. +- Unchanged original v1.24.0 core with corrected provider: **6/6 passed**, Bash/Zsh/Fish × both transports, random 4 KiB and genuinely live follow-on. This was original core injection with a synthetic executable, not a rebuilt v1.24.0 full agent. +- Provider init: **10/10 passed**, five shells × both transports; writable per-user agent path and `/bin/true` Docker stand-in. This verifies init transport/output checks, not a real remote Docker daemon. + +## Actual Linux failures and recovery + +| Case | Observed result | +| --- | --- | +| BusyBox `/bin/sh` clone, both transports | Exact binary/live exchange passed | +| Missing `/bin/sh`, both transports | Exit 127, diagnostic preserved, no `ping`, no payload request | +| Denied `/bin/sh`, both transports | Exit 126, diagnostic preserved, no `ping`, no payload request | +| Remote stdout/stderr and exit 37, both transports | Exact `ordinary-out`, exact `remote-diagnostic`, exit 37 preserved | +| Sudo denied/restrictive `/root` destination, both transports | Error explicitly requires passwordless sudo; only `ping`; no payload request | +| Real sudo installer, native synthetic ELF, both transports | Root-owned unwritable destination installed successfully; normal non-root execution returned exact binary/live exchange | +| Download failure, then stdin, both transports | Three failed localhost downloads followed by successful exact binary/live exchange; test-local `sleep` shim bypassed retry waits | +| No curl/wget in invocation-local PATH, both transports | Clear diagnostic, then successful stdin binary/live exchange | +| Truncated frame plus no download tools, both transports | Exit 1; raw `ping\nARM-true\n`, no `done`; complete-receive and both-methods-failed diagnostics; pre-existing executable still prints `old-valid` | +| Noisy Zsh startup in isolated clone, both transports | Noise remains visible before command output; core built-in test rejects it with `unexpected start line "noisy-rc"`, executed=false, zero payload requests | + +An independent read-only reviewer ran the actual `quote_installer_argument`/`run_installer_script` functions with Linux util-linux 2.41.3 and BusyBox 1.37.0 `su -s /bin/sh -c ... root`. Both preserved empty strings, apostrophes, backslashes, Unicode, CR/newlines, substitutions, leading dash and trailing newlines as exact NUL-delimited arguments. Both preserved `abcdeLIVE\n` across `head -c 5; printf '|'; cat`, producing `abcde|LIVE\n`. GNU and BusyBox accepted `chmod -- +x`. + +## Real agent and remaining acceptance + +The actual development Linux aarch64 Devsy binary (217,260,362 bytes) was transferred and started as `internal ssh-server --stdio`. Initial Zsh external and Bash cold/warm/replacement tunnels completed `SSH-2.0-Go` and two separate command sessions, returning `first-command` and `live-followon`, without stderr. The real-agent startup matrix passed **10/10**: all five passwd shells × both transports, each with SSH handshake and two sequential commands. Six cells reused an installed executable; Dash/ash × both transports performed cold transfers. Earlier Bash/Zsh runs additionally exercised cold/reuse/forced replacement. Some initial cold development-binary transfers exceeded the original 30-second injection/40-second client bounds; those attempts are recorded harness-limit failures, superseded by the successful 150-second-bound startup matrix. The reusable tunnel harness allows 150 seconds for large development binaries. + +Whole CLI `--ide=none` workspace acceptance: **BLOCKED**. Automatic approval review rejected the proposed privileged Docker-daemon fixture because it broadens the host boundary for container workloads; that action was not delivered or run. A safer capability check in the existing fixture as non-root `fixture-zsh` returned `unshare: Operation not permitted` (exit 1). No Docker socket, Docker client/daemon, or rootlesskit was present. No privileged container, host socket mount, seccomp relaxation, or indirect workaround was attempted. The SSH-only fixture uses `/bin/true` during provider init and does not establish the whole CLI workspace flow. Desktop acceptance: **BLOCKED**; no configured Desktop session/workspace fixture was provided. Native macOS unit/static checks are separate from the actual Linux/OpenSSH results above. Original v1.24.0 full-agent startup is **NOT RUN**; original core injection compatibility is verified above. + +Earlier provider artifact (`final5`) SHA-256: `7e18087de9fbc9137a7a561c6fce200b7fe50d523fd99db08b6e8ce7b27015c7`. The complete random/hostile 30-cell synthetic matrix was rerun against that artifact and passed; exact output SHA-256: `475f5b02c861d431216e56c8f7ccd56e5fada5e0e25c1c7fce5172dcefd5f349`. Earlier transport matrix and real cold transfers used the provider worker's first corrected artifact SHA-256 `b7e79349ee68e8685f03ba98ea03427dcb8cd02a31bea54648a902c1dc322c22`; final5 adds stricter initialization and stdin/context shutdown. A final5 rerun also passed all ten real-agent startup/two-session cells using the installed actual Devsy binary. The staged provider diff digest supplied by its owner was `5d9b03da0b349f36b2f3b539e29a454dd1490afa63def9bec145119e167d51b4`; this is a working-diff identifier, not a release SHA. + +The post-review provider artifact SHA-256 is `39468e46550ddf4a9ea3a1033cae10ef02d0feabcce42ad7b89a7f88f40ba989`, with production-diff fingerprint `fde9ee3eceaba52ab03246f0d59151c7070c967168edbce6b63a04ec1ddef1a0` supplied by its owner. This artifact passed a fresh **30/30** random/hostile synthetic matrix, **10/10** real-agent warm startup/two-session matrix, **6/6** unchanged-v1.24.0-core compatibility matrix, and both actual-agent SIGTERM checks (external 0.46 s, built-in 0.41 s; no local children or remote agent remaining). Final reviewed init and exact stdout/stderr/exit-37 checks passed in all ten shell/transport cells (20 checks). The final random-matrix exact stdout SHA-256 is `b8f99178b99fa86953567a3672e32a7427d140ef372cf563c3d296ad8b495ca2`. The final warm fixture was restored from the same cached base and prepopulated through Docker copy with the identical agent artifact; these are warm-start results, not new cold-transfer claims. The post-review changes address platform-specific redirected stdin and input-copy errors. Windows tests were cross-compiled locally; native Windows execution remains **NOT RUN**. + +Native Linux execution of the actual `TestBootstrapSuite` and unchanged `TestInjectScript_RealShell*` tests passed with `/bin/sh` pointing at BusyBox. Cross-compilation only produced the test executable; these results came from executing it in the Linux clone. + +Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/source SHA-256: + +| Input | SHA-256 | +| --- | --- | +| Original v1.24.0 `inject.sh` | `c2effffd6bbb9e42146bce22e2393f55f19b88d7b152f59d09c3c1e749bc2e0b` | +| Corrected `pkg/inject/inject.sh` | `b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3` | +| `pkg/inject/script.go` | `752c82c5f3df1e60359658bc961c557b35995636dab88618ca8e791e21bf8e3e` | +| `pkg/agent/inject.go` | `3ea3bb44cbc37132bde6569b68e0425ea8c91d744c3a6ea94988d99d3e6c61e7` | +| Actual Linux agent executable | `5e5853e9e0653830858fe2438b46a7e8eb94a927bbbb4fda3142c2ddbe173bc4` | + +SIGTERM cancellation of the final5 provider with blocked stdin completed in under 0.5 seconds on both transports after observing the actual agent SSH banner. Local SSH children/connections and the remote Devsy stdio agent terminated. The built-in command reports context cancellation and exits 1; external SSH exits 255. An arbitrary remote `exec sleep 1621` survived the closed non-PTY channel; those disposable probes were explicitly removed. Cancellation guarantees local transport/stdin-pump shutdown and observed agent cleanup, not remote arbitrary-process reaping or SIGKILL cleanup. + +## Reproduction + +Require a cached supported Ubuntu image, Docker, Python 3, OpenSSH client and the repository Go toolchain. Use a fresh temporary state directory and fresh container name. Never put state/keys/output under tracked docs. + +```sh +python3 docs/issue-1621/ssh-fixture.py setup --state "$FIXTURE_STATE" --name "$FIXTURE_NAME" +python3 docs/issue-1621/ssh-fixture.py metadata --state "$FIXTURE_STATE" --name "$FIXTURE_NAME" +go build -o "$CORE_HARNESS" docs/issue-1621/inject-harness.go +go build -o "$TUNNEL_HARNESS" docs/issue-1621/real-tunnel.go +python3 docs/issue-1621/matrix.py --state "$FIXTURE_STATE" --core "$CORE_HARNESS" --provider "$PROVIDER_BINARY" --output "$RESULT_FILE" --hostile +PATH="$FIXTURE_STATE/bin:$PATH" "$TUNNEL_HARNESS" -provider "$PROVIDER_BINARY" -payload "$LINUX_AGENT" -port "$FIXTURE_PORT" -host fixture-zsh -builtin true -dest /home/fixture-zsh/real-agent +python3 docs/issue-1621/ssh-fixture.py teardown --state "$FIXTURE_STATE" --name "$FIXTURE_NAME" +``` + +`fixture.json` supplies the selected port and wrapper directory. The wrappers supply only the fixture SSH config, including the `ssh -G` lookup used by the built-in transport. Harnesses have `go:build ignore` and must be built explicitly. For baseline reproduction, export the original core into a fresh temporary directory, copy only `inject-harness.go` there, build it against that export's module, and run `matrix.py --baseline` with the original provider executable. Original-provider stderr contains its original timestamped logger. + +The `forced-replacement` synthetic case explicitly sets the installation condition true; it is not a version-parser test. `real-tunnel.go -check` accepts the actual generated version check when testing version selection. Low-level framing failures can be reproduced with `inject-harness -script-out`, then passing `pong\n100\nabc` and EOF to the provider `COMMAND` while its invocation-local PATH omits curl/wget. Do not modify host binaries or accounts. + +## T9 boundary audit and release coordination + +Read-only inventory at base core `d3f6e421642575c5072badcf56bf5b0a5606bd41`: + +| Boundary | Existing contract / required disposition | +| --- | --- | +| `pkg/devcontainer/config/userenvprobe.go` | Keep configured `GetShell` and `-lic`/`-lc`/`-ic` semantics. Probe intentionally evaluates the user's environment. | +| `pkg/ssh/server/ssh_container.go:getCommand`, `pkg/shell/shell.go:GetShell` | Keep passwd/configured shell for terminal commands and PTY login `-l`; do not force user terminals to POSIX shell. | +| `pkg/agent/delivery/factory.go` | Local Docker uses named-volume delivery; remote Docker uses native `docker cp`/`mkdir`/`chmod` argv. Available `ArgvExec` uses native driver calls; Apple and fallback custom drivers retain `LegacyShellDelivery`. | +| `pkg/agent/delivery/kubernetes.go` | Driver receives explicit argv, including deliberately owned `sh -c` scripts for staging/download/commit; do not wrap argv again as a remote login command. | +| `pkg/devcontainer/setup.go:execSetupSSHServer` | Native `ArgvExecDriver` starts the agent with literal argv; fallback `CommandDevContainer` string path carries raw protocol stdout. | +| Docker `CommandDevContainer` | Native CLI argv already selects `sh -c`; no remote login-shell wrapper needed. | +| Kubernetes `CommandDevContainer` | Root uses `sh -c`; non-root uses `su USER -c STRING`. Follow up separately on that user-shell contract if broadening scope. | +| Apple `CommandDevContainer` | Native container CLI argv explicitly selects `sh -c`; Apple delivery remains shell-based and requires its own actual macOS qualification. | +| Custom `CommandDevContainer` | Passes `DEVCONTAINER_COMMAND`/`DEVCONTAINER_USER` to configured provider command. Provider implementation owns final shell evaluation; audit each provider before making broader claims. | +| External runtime / microsandbox argv | Preserve raw literal argv and streaming; no login-shell wrapper should be added. | + +Core and SSH-provider changes require independently versioned releases. Releasing only core does not repair Fish parsing in the old provider; corrected provider with old core passes this bounded compatibility matrix. Keep one canonical issue, cross-link both PRs and record exact released artifacts before claiming a shipped fix. Rollback must remain independent: revert/pin the provider separately from core, preserve a previously valid agent on failed replacement, and report which transport/version was restored. Release order, published-artifact qualification, full CLI/Desktop acceptance and merge remain pending owner decisions. diff --git a/docs/issue-1621/inject-harness.go b/docs/issue-1621/inject-harness.go new file mode 100644 index 0000000000..8440970b34 --- /dev/null +++ b/docs/issue-1621/inject-harness.go @@ -0,0 +1,191 @@ +//go:build ignore + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/exec" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/inject" +) + +type harnessConfig struct { + provider, host, port, builtin, dest, check, command string + payload, input, download, scriptOut, prefix string + prefer bool + timeout time.Duration +} + +type liveOutput struct { + out bytes.Buffer + once sync.Once + ready chan struct{} + count int +} + +func (w *liveOutput) Write(p []byte) (int, error) { + n, err := w.out.Write(p) + if w.out.Len() >= w.count+6 { + w.once.Do(func() { close(w.ready) }) + } + return n, err +} + +type injectionRun struct { + config harnessConfig + output *liveOutput + stderr, raw bytes.Buffer + stdin io.Reader + expected []byte + requests int + execError string +} + +func main() { + config := parseHarnessFlags() + if config.scriptOut != "" { + writeGeneratedScript(config) + return + } + ctx, cancel := context.WithTimeout(context.Background(), config.timeout) + defer cancel() + run := &injectionRun{config: config, output: &liveOutput{ready: make(chan struct{})}} + cleanup := run.prepareInput(ctx) + defer cleanup() + ran, err := inject.Inject(ctx, inject.InjectOptions{ + Exec: run.execute, LocalFile: run.openPayload, ScriptParams: config.params(), + Stdin: run.stdin, Stdout: run.output, Stderr: &run.stderr, Timeout: 5 * time.Second, + }) + run.report(ran, err) +} + +func parseHarnessFlags() harnessConfig { + var c harnessConfig + flag.StringVar(&c.provider, "provider", "", "provider executable") + flag.StringVar(&c.host, "host", "fixture-bash", "fixture host alias") + flag.StringVar(&c.port, "port", "22", "SSH port") + flag.StringVar(&c.builtin, "builtin", "false", "builtin SSH") + flag.StringVar(&c.dest, "dest", "/tmp/verify1621/agent", "remote destination") + flag.StringVar(&c.check, "check", "true", "installation condition") + flag.StringVar(&c.command, "command", "/tmp/verify1621/agent", "post-bootstrap command") + flag.StringVar(&c.payload, "payload", "", "binary source") + flag.StringVar(&c.input, "input", "", "4096-byte binary duplex test input") + flag.StringVar(&c.download, "download", "", "download URL") + flag.StringVar(&c.scriptOut, "script-out", "", "write generated script and exit") + flag.StringVar(&c.prefix, "prefix", "", "POSIX prefix for generated script") + flag.BoolVar(&c.prefer, "prefer-download", false, "prefer download before stdin") + flag.DurationVar(&c.timeout, "timeout", 12*time.Second, "overall bound") + flag.Parse() + return c +} + +func (c harnessConfig) params() *inject.Params { + return &inject.Params{ + Command: c.command, AgentRemotePath: c.dest, + DownloadURLs: &inject.DownloadURLs{Amd: c.download, Arm: c.download}, + ExistsCheck: c.check, ShouldChmodPath: true, PreferAgentDownload: c.prefer, + } +} + +func writeGeneratedScript(c harnessConfig) { + script, err := inject.GenerateScript(inject.Script, c.params()) + if err != nil { + panic(err) + } + // #nosec G703 -- The operator explicitly selects the private fixture output file. + if err = os.WriteFile(c.scriptOut, []byte(script), 0o600); err != nil { + panic(err) + } +} + +func (r *injectionRun) prepareInput(ctx context.Context) func() { + r.stdin = bytes.NewReader([]byte("tunnel-after-handshake\n")) + if r.config.input == "" { + return func() {} + } + // #nosec G304 G703 -- The operator explicitly selects the fixture byte input file. + data, err := os.ReadFile(r.config.input) + if err != nil { + panic(err) + } + r.output.count = len(data) + r.expected = append( + append(append([]byte{}, data...), []byte("READY\n")...), + []byte("LIVE-OK\n")...) + reader, writer := io.Pipe() + r.stdin = reader + go sendLiveInput(ctx, writer, data, r.output.ready) + return func() { _ = reader.Close() } +} + +func sendLiveInput(ctx context.Context, writer *io.PipeWriter, data []byte, ready <-chan struct{}) { + defer func() { _ = writer.Close() }() + if _, err := writer.Write(data); err != nil { + return + } + select { + case <-ready: + _, _ = writer.Write([]byte("LIVE-FOLLOWON\n")) + case <-ctx.Done(): + } +} + +//nolint:revive // The callback must match inject.ExecFunc's five-argument signature. +func (r *injectionRun) execute( + ctx context.Context, + script string, + stdin io.Reader, + stdout, stderr io.Writer, +) error { + // #nosec G204 -- This harness deliberately executes the operator-selected provider artifact. + cmd := exec.CommandContext(ctx, r.config.provider, "command") + cmd.Env = append( + os.Environ(), + "COMMAND="+r.config.prefix+script, + "HOST="+r.config.host, + "PORT="+r.config.port, + "USE_BUILTIN_SSH="+r.config.builtin, + "DOCKER_PATH=/bin/true", + "AGENT_PATH="+r.config.dest, + ) + cmd.Stdin = stdin + cmd.Stdout = io.MultiWriter(stdout, &r.raw) + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + r.execError = err.Error() + } + return err +} + +func (r *injectionRun) openPayload(_ bool) (io.ReadCloser, error) { + r.requests++ + // #nosec G304 G703 -- The operator explicitly selects the agent artifact being tested. + return os.Open(r.config.payload) +} + +func (r *injectionRun) report(ran bool, err error) { + message := "" + if err != nil { + message = err.Error() + } + result := map[string]any{ + "executed": ran, "error": message, "execError": r.execError, + "stdout": r.output.out.String(), "stdoutBytes": r.output.out.Len(), + "stdoutSHA256": fmt.Sprintf("%x", sha256.Sum256(r.output.out.Bytes())), + "duplexExact": r.expected != nil && bytes.Equal(r.expected, r.output.out.Bytes()), + "localRequests": r.requests, "stderr": r.stderr.String(), "rawStdout": r.raw.String(), + } + if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { + panic(err) + } +} diff --git a/docs/issue-1621/matrix.py b/docs/issue-1621/matrix.py new file mode 100644 index 0000000000..d755b50cee --- /dev/null +++ b/docs/issue-1621/matrix.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Run generated core injection through the actual SSH-provider executable.""" +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--state', required=True, type=Path) + p.add_argument('--core', required=True, type=Path) + p.add_argument('--provider', required=True, type=Path) + p.add_argument('--output', required=True, type=Path) + p.add_argument('--baseline', action='store_true') + p.add_argument('--hostile', action='store_true') + a = p.parse_args() + fixture = json.loads((a.state / 'fixture.json').read_text()) + env = dict(os.environ, PATH=fixture['bin'] + ':' + os.environ['PATH']) + payload = a.state / 'duplex-agent' + payload.write_text('''#!/usr/bin/python3 +import sys +r = sys.stdin.buffer +w = sys.stdout.buffer +w.write(r.read(4096) + b"READY\\n") +w.flush() +if r.readline() == b"LIVE-FOLLOWON\\n": + w.write(b"LIVE-OK\\n") + w.flush() +''') + input_file = a.state / 'input.bin' + data = bytearray(os.urandom(4096)) + data[:8] = b'\x00\'"$`\n\r\\' + input_file.write_bytes(data) + rows = [] + for shell in (['bash', 'zsh', 'fish'] if a.baseline else ['bash', 'zsh', 'fish', 'dash', 'ash']): + for builtin in ['false', 'true']: + suffix = f'/literal space\'"$`(quote)\ncarriage\rsemi;{builtin}' if a.hostile else f'/matrix-{builtin}' + dest = f'/home/fixture-{shell}' + suffix + '/agent' + for phase in (['cold'] if a.baseline else ['cold', 'warm', 'forced-replacement']): + cmd = [str(a.core.resolve()), '-provider', str(a.provider.resolve()), + '-host', 'fixture-' + shell, '-port', str(fixture['port']), + '-builtin', builtin, '-dest', dest, '-command', shlex.quote(dest), + '-payload', str(payload), '-input', str(input_file), + '-check', 'false' if phase == 'warm' else 'true'] + c = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=30) + row = dict(shell=shell, builtin=builtin, phase=phase, harnessExit=c.returncode, + **json.loads(c.stdout)) + row['rawPrefix'] = row.pop('rawStdout')[:30] + row.pop('stdout') + rows.append(row) + print(json.dumps(row), flush=True) + a.output.write_text(json.dumps(rows, indent=2) + '\n') + if not a.baseline: + assert all(r['duplexExact'] and not r['error'] and not r['stderr'] for r in rows) + + +if __name__ == '__main__': + main() diff --git a/docs/issue-1621/real-tunnel.go b/docs/issue-1621/real-tunnel.go new file mode 100644 index 0000000000..1986de8f3a --- /dev/null +++ b/docs/issue-1621/real-tunnel.go @@ -0,0 +1,163 @@ +//go:build ignore + +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "maps" + "net" + "os" + "os/exec" + "time" + + "github.com/devsy-org/devsy/pkg/inject" + "golang.org/x/crypto/ssh" +) + +type tunnelConfig struct { + provider, host, port, builtin, dest, payload, check string +} + +type tunnelRun struct { + config tunnelConfig + bridge net.Conn + stderr bytes.Buffer + requests int + completed chan map[string]any +} + +func main() { + result := runTunnel(parseTunnelFlags()) + if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { + panic(err) + } +} + +func parseTunnelFlags() tunnelConfig { + var c tunnelConfig + flag.StringVar(&c.provider, "provider", "", "provider executable") + flag.StringVar(&c.host, "host", "fixture-zsh", "fixture host") + flag.StringVar(&c.port, "port", "22", "SSH port") + flag.StringVar(&c.builtin, "builtin", "false", "builtin transport") + flag.StringVar(&c.dest, "dest", "/home/fixture-zsh/real-agent", "agent destination") + flag.StringVar(&c.payload, "payload", "", "actual Linux agent") + flag.StringVar(&c.check, "check", "true", "replacement condition") + flag.Parse() + return c +} + +func runTunnel(config tunnelConfig) map[string]any { + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + client, bridge := net.Pipe() + defer func() { _ = client.Close(); _ = bridge.Close() }() + run := &tunnelRun{config: config, bridge: bridge, completed: make(chan map[string]any, 1)} + go func() { run.completed <- run.inject(ctx) }() + result := map[string]any{"host": config.host, "builtin": config.builtin} + err := run.connect(client, result) + result["tunnelError"] = "" + if err != nil { + result["tunnelError"] = err.Error() + } + cancel() + _ = bridge.Close() + _ = client.Close() + select { + case injection := <-run.completed: + maps.Copy(result, injection) + result["stderr"] = run.stderr.String() + case <-time.After(8 * time.Second): + result["injectWaitTimeout"] = true + } + return result +} + +func (r *tunnelRun) inject(ctx context.Context) map[string]any { + quoted, err := inject.QuotePOSIXLiteral(r.config.dest) + if err != nil { + return map[string]any{"injectError": err.Error()} + } + ran, err := inject.Inject(ctx, inject.InjectOptions{ + Exec: r.execute, LocalFile: r.openPayload, + ScriptParams: &inject.Params{ + AgentRemotePath: r.config.dest, DownloadURLs: &inject.DownloadURLs{}, + ExistsCheck: r.config.check, ShouldChmodPath: true, + Command: "exec " + quoted + " internal ssh-server --stdio --log-level=error", + }, + Stdin: r.bridge, Stdout: r.bridge, Stderr: &r.stderr, Timeout: 150 * time.Second, + }) + message := "" + if err != nil { + message = err.Error() + } + return map[string]any{"executed": ran, "injectError": message, "localRequests": r.requests} +} + +//nolint:revive // The callback must match inject.ExecFunc's five-argument signature. +func (r *tunnelRun) execute( + ctx context.Context, + script string, + stdin io.Reader, + stdout, stderr io.Writer, +) error { + // #nosec G204 -- This harness deliberately executes the operator-selected provider artifact. + cmd := exec.CommandContext(ctx, r.config.provider, "command") + cmd.Env = append( + os.Environ(), + "HOST="+r.config.host, + "PORT="+r.config.port, + "USE_BUILTIN_SSH="+r.config.builtin, + "COMMAND="+script, + "DOCKER_PATH=/bin/true", + "AGENT_PATH="+r.config.dest, + ) + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr + return cmd.Run() +} + +func (r *tunnelRun) openPayload(_ bool) (io.ReadCloser, error) { + r.requests++ + // #nosec G304 G703 -- The operator explicitly selects the Linux agent artifact being tested. + return os.Open(r.config.payload) +} + +func (r *tunnelRun) connect(client net.Conn, result map[string]any) error { + if err := client.SetDeadline(time.Now().Add(150 * time.Second)); err != nil { + return err + } + conn, channels, requests, err := ssh.NewClientConn(client, "fixture-agent", &ssh.ClientConfig{ + User: r.config.host, + // #nosec G106 -- The SSH server is the ephemeral agent just injected into the isolated fixture. + HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: 150 * time.Second, + }) + if err != nil { + return err + } + sshClient := ssh.NewClient(conn, channels, requests) + defer func() { _ = sshClient.Close() }() + result["serverVersion"] = string(conn.ServerVersion()) + return runTunnelCommands(sshClient, result) +} + +func runTunnelCommands(client *ssh.Client, result map[string]any) error { + for index, command := range []string{"printf first-command", "printf live-followon"} { + session, err := client.NewSession() + if err != nil { + return err + } + out, err := session.Output(command) + _ = session.Close() + result[fmt.Sprintf("command%d", index+1)] = string(out) + if err != nil { + return err + } + } + return nil +} diff --git a/docs/issue-1621/ssh-fixture.py b/docs/issue-1621/ssh-fixture.py new file mode 100644 index 0000000000..f309c45f3c --- /dev/null +++ b/docs/issue-1621/ssh-fixture.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Disposable Linux SSH fixture; credentials remain in the private state directory.""" +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess + + +def run(*args, **kw): + return subprocess.run(args, check=True, text=True, **kw) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('action', choices=['setup', 'metadata', 'teardown']) + p.add_argument('--state', required=True, type=Path) + p.add_argument('--name', default='devsy-1621-sshd') + p.add_argument('--image', default='ghcr.io/devsy-org/test-images/base@sha256:4bcb1b466771b1ba1ea110e2a27daea2f6093f9527fb75ee59703ec89b5561cb') + p.add_argument('--reuse', action='store_true') + a = p.parse_args() + if a.action == 'teardown': + run('docker', 'rm', '-f', a.name) + print('Container removed; delete private state after workers finish.') + return + if a.action == 'metadata': + run('docker', 'exec', a.name, 'sh', '-c', "uname -sm; /usr/sbin/sshd -V 2>&1; bash --version | head -1; zsh --version; fish --version; busybox | head -1; readlink -f /bin/sh; getent passwd fixture-bash fixture-zsh fixture-fish fixture-dash fixture-ash; /usr/sbin/sshd -T -f /tmp/fixture-sshd.conf | grep -E '^(passwordauthentication|pubkeyauthentication|permitrootlogin|usepam|printmotd|authorizedkeysfile)'") + return + a.state.mkdir(parents=True, exist_ok=True) + os.chmod(a.state, 0o700) + key = a.state / 'id_ed25519' + if not key.exists(): + run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(key)) + if not a.reuse: + run('docker', 'run', '-d', '--pull=never', '--name', a.name, '-p', '127.0.0.1::22', a.image, 'sleep', 'infinity') + run('docker', 'exec', a.name, 'sh', '-c', 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openssh-server bash zsh fish dash busybox python3 sudo') + run('docker', 'cp', str(key) + '.pub', a.name + ':/tmp/fixture.pub') + run('docker', 'exec', a.name, 'sh', '-ec', ''' +mkdir -p /run/sshd +ln -sf /bin/busybox /bin/ash +for shell in bash zsh fish dash ash; do + name=fixture-$shell + id "$name" >/dev/null 2>&1 || useradd -m -s /bin/$shell "$name" + usermod -p '*' "$name" + mkdir -p /home/$name/.ssh + cp /tmp/fixture.pub /home/$name/.ssh/authorized_keys + chown -R $name:$name /home/$name/.ssh + chmod 700 /home/$name/.ssh + chmod 600 /home/$name/.ssh/authorized_keys +done +printf '%s\\n' 'PasswordAuthentication no' 'KbdInteractiveAuthentication no' 'PermitRootLogin no' 'UsePAM no' 'PrintMotd no' 'AuthorizedKeysFile .ssh/authorized_keys' 'PidFile /run/fixture-sshd.pid' 'Subsystem sftp internal-sftp' > /tmp/fixture-sshd.conf +ssh-keygen -A +/usr/sbin/sshd -f /tmp/fixture-sshd.conf -E /tmp/fixture-sshd.log +''') + port = run('docker', 'port', a.name, '22/tcp', capture_output=True).stdout.strip().split(':')[-1] + config = a.state / 'ssh_config' + shells = ['bash', 'zsh', 'fish', 'dash', 'ash'] + config.write_text('Host fixture-*\n HostName 127.0.0.1\n Port ' + port + '\n IdentityFile ' + str(key) + '\n IdentitiesOnly yes\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no\n BatchMode yes\n LogLevel ERROR\n ConnectTimeout 5\n' + ''.join('Host fixture-' + s + '\n User fixture-' + s + '\n' for s in shells)) + bindir = a.state / 'bin' + bindir.mkdir(exist_ok=True) + for command in ['ssh', 'scp']: + wrapper = bindir / command + wrapper.write_text('#!/bin/sh\nexec /usr/bin/' + command + ' -F ' + shlex.quote(str(config)) + ' "$@"\n') + wrapper.chmod(0o700) + (a.state / 'fixture.json').write_text(json.dumps({'container': a.name, 'port': int(port), 'config': str(config), 'bin': str(bindir)}, indent=2) + '\n') + print(json.dumps({'port': int(port), 'state': str(a.state), 'hosts': ['fixture-' + s for s in shells]})) + + +if __name__ == '__main__': + main() diff --git a/pkg/agent/inject.go b/pkg/agent/inject.go index 65ff6c2988..011220ae22 100644 --- a/pkg/agent/inject.go +++ b/pkg/agent/inject.go @@ -82,6 +82,11 @@ func (o *InjectOptions) Validate() error { if o.Exec == nil { return fmt.Errorf("exec function is required") } + for _, value := range []string{o.RemoteAgentPath, o.RemoteVersion, o.DownloadURL, o.Command} { + if strings.ContainsRune(value, 0) { + return fmt.Errorf("agent injection parameter contains NUL") + } + } return nil } @@ -277,12 +282,16 @@ type injectConfig struct { } func injectAgent(ctx context.Context, cfg *injectConfig) error { + params, err := buildScriptParams(cfg) + if err != nil { + return err + } buf := &bytes.Buffer{} //nolint:staticcheck wasExecuted, err := inject.Inject(ctx, inject.InjectOptions{ Exec: cfg.opts.Exec, LocalFile: createBinaryLoader(ctx, cfg), - ScriptParams: buildScriptParams(cfg), + ScriptParams: params, Stdin: cfg.opts.Stdin, Stdout: cfg.opts.Stdout, Stderr: setupStderr(cfg.opts, buf), @@ -312,16 +321,20 @@ func createBinaryLoader(ctx context.Context, cfg *injectConfig) func(bool) (io.R } } -func buildScriptParams(cfg *injectConfig) *inject.Params { +func buildScriptParams(cfg *injectConfig) (*inject.Params, error) { opts := cfg.opts + existsCheck, err := cfg.vc.buildExistsCheck(opts.RemoteAgentPath) + if err != nil { + return nil, err + } return &inject.Params{ Command: opts.Command, AgentRemotePath: opts.RemoteAgentPath, DownloadURLs: inject.NewDownloadURLs(opts.DownloadURL), - ExistsCheck: cfg.vc.buildExistsCheck(opts.RemoteAgentPath), + ExistsCheck: existsCheck, PreferAgentDownload: *opts.PreferDownloadFromRemoteUrl, ShouldChmodPath: true, - } + }, nil } func handleInjectError(err error, wasExecuted bool, buf *bytes.Buffer) error { @@ -394,12 +407,20 @@ func newVersionChecker(opts *InjectOptions) *versionChecker { } } -func (vc *versionChecker) buildExistsCheck(agentPath string) string { +func (vc *versionChecker) buildExistsCheck(agentPath string) (string, error) { + quotedPath, err := inject.QuotePOSIXLiteral(agentPath) + if err != nil { + return "", err + } if vc.skipCheck { - return fmt.Sprintf(`! [ -x "%s" ]`, agentPath) + return fmt.Sprintf(`! [ -x %s ]`, quotedPath), nil } - return fmt.Sprintf(`! { [ -x "%s" ] && [ "$("%s" --version 2>/dev/null)" = "%s" ]; }`, - agentPath, agentPath, vc.remoteVersion) + quotedVersion, err := inject.QuotePOSIXLiteral(vc.remoteVersion) + if err != nil { + return "", err + } + return fmt.Sprintf(`! { [ -x %s ] && [ "$(%s --version 2>/dev/null)" = %s ]; }`, + quotedPath, quotedPath, quotedVersion), nil } // session encapsulates the state and resources for an injection session. @@ -409,12 +430,16 @@ func (vc *versionChecker) detectRemoteAgentVersion( agentPath string, ) (string, error) { buf := &bytes.Buffer{} - versionCmd := fmt.Sprintf("%s --version", agentPath) + quotedPath, err := inject.QuotePOSIXLiteral(agentPath) + if err != nil { + return "", err + } + versionCmd := fmt.Sprintf("%s --version", quotedPath) checkCtx, cancel := context.WithTimeout(ctx, versionCheckTimeout) defer cancel() - log.Debugf("checking remote agent version: %s", versionCmd) - err := exec(checkCtx, versionCmd, nil, buf, io.Discard) + log.Debug("checking remote agent version") + err = exec(checkCtx, versionCmd, nil, buf, io.Discard) if err != nil { if errors.Is(checkCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil { return "", fmt.Errorf( diff --git a/pkg/agent/inject_literal_test.go b/pkg/agent/inject_literal_test.go new file mode 100644 index 0000000000..8f1d8e03fd --- /dev/null +++ b/pkg/agent/inject_literal_test.go @@ -0,0 +1,95 @@ +//nolint:gosec // These regressions execute generated shell checks and a private executable fixture. +package agent + +import ( + "context" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/devsy-org/devsy/pkg/inject" + "github.com/stretchr/testify/require" +) + +func TestVersionCheckerLiteralPathAndVersion(t *testing.T) { + if _, err := exec.LookPath("/bin/sh"); err != nil { + t.Skip("POSIX /bin/sh is unavailable") + } + root := t.TempDir() + agentPath := filepath.Join(root, "agent '\"\\\n雪 $(touch sentinel)`touch sentinel`") + version := "v1 ' \" \\ 雪 $(touch sentinel)`touch sentinel`" + literalVersion, err := inject.QuotePOSIXLiteral(version) + require.NoError(t, err) + require.NoError( + t, + os.WriteFile(agentPath, []byte("#!/bin/sh\nprintf '%s\\n' "+literalVersion+"\n"), 0o700), + ) + vc := &versionChecker{remoteVersion: version} + predicate, err := vc.buildExistsCheck(agentPath) + require.NoError(t, err) + cmd := exec.CommandContext( + t.Context(), + "/bin/sh", + "-c", + "if "+predicate+"; then printf install; else printf reuse; fi", + ) + cmd.Dir = root + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + require.Equal(t, "reuse", string(output)) + + run := func(ctx context.Context, code string, stdin io.Reader, stdout, stderr io.Writer) error { + command := exec.CommandContext(ctx, "/bin/sh", "-c", code) + command.Dir = root + command.Stdin, command.Stdout, command.Stderr = stdin, stdout, stderr + return command.Run() + } + detected, err := vc.detectRemoteAgentVersion(t.Context(), run, agentPath) + require.NoError(t, err) + require.Equal(t, version, detected) + _, err = os.Stat(filepath.Join(root, "sentinel")) + require.True(t, os.IsNotExist(err)) + + vc.remoteVersion = "mismatch" + predicate, err = vc.buildExistsCheck(agentPath) + require.NoError(t, err) + require.NoError(t, exec.CommandContext(t.Context(), "/bin/sh", "-c", predicate).Run()) +} + +func TestVersionCheckerRejectsNULBeforeExecution(t *testing.T) { + vc := &versionChecker{remoteVersion: "v1"} + _, err := vc.buildExistsCheck("bad\x00path") + require.Error(t, err) + vc.remoteVersion = "v1\x00bad" + _, err = vc.buildExistsCheck("/tmp/agent") + require.Error(t, err) + _, err = vc.detectRemoteAgentVersion( + t.Context(), + func(context.Context, string, io.Reader, io.Writer, io.Writer) error { + t.Fatal("must not execute an invalid path") + return nil + }, + "bad\x00path", + ) + require.Error(t, err) + + for _, field := range []string{"path", "version", "url", "command"} { + opts := &InjectOptions{ + Exec: func(context.Context, string, io.Reader, io.Writer, io.Writer) error { return nil }, + } + switch field { + case "path": + opts.RemoteAgentPath = "bad\x00path" + case "version": + opts.RemoteVersion = "bad\x00version" + case "url": + opts.DownloadURL = "bad\x00url" + case "command": + opts.Command = "bad\x00command" + } + require.Error(t, opts.Validate(), strings.ToUpper(field)) + } +} diff --git a/pkg/inject/inject.go b/pkg/inject/inject.go index ef65c34817..0f8ed102f1 100644 --- a/pkg/inject/inject.go +++ b/pkg/inject/inject.go @@ -64,14 +64,14 @@ func Inject(ctx context.Context, opts InjectOptions) (bool, error) { logPreferredAgentDownloadURL(opts.ScriptParams) scriptRawCode, err := GenerateScript(Script, opts.ScriptParams) if err != nil { - return true, err + return false, err } log.Debug("execute inject script") defer log.Debug("done injecting") sess, err := newSession(opts, scriptRawCode, start) if err != nil { - return true, err + return false, err } defer sess.Close() select { diff --git a/pkg/inject/inject.sh b/pkg/inject/inject.sh index 07df29c3df..467de14611 100644 --- a/pkg/inject/inject.sh +++ b/pkg/inject/inject.sh @@ -1,11 +1,13 @@ #!/bin/sh set -e -INSTALL_DIR="{{ .InstallDir }}" -INSTALL_FILENAME="{{ .InstallFilename }}" +INSTALL_DIR={{ .InstallDir }} +INSTALL_FILENAME={{ .InstallFilename }} INSTALL_PATH="$INSTALL_DIR/$INSTALL_FILENAME" -PREFER_DOWNLOAD="{{ .PreferDownload }}" -CHMOD_PATH="{{ .ChmodPath }}" +PREFER_DOWNLOAD={{ .PreferDownload }} +CHMOD_PATH={{ .ChmodPath }} +INSTALLER_MODE=direct +temp_file= command_exists() { command -v "$@" >/dev/null 2>&1 @@ -18,8 +20,49 @@ is_arm() { esac } +quote_installer_argument() { + quote_value=$1 + quoted_argument="'" + while :; do + case "$quote_value" in + *"'"*) + quoted_argument="$quoted_argument${quote_value%%\'*}'\\''" + quote_value=${quote_value#*\'} + ;; + *) break ;; + esac + done + quoted_argument="$quoted_argument$quote_value'" +} + +run_installer_script() { + installer_code=$1 + shift + case "$INSTALLER_MODE" in + direct) /bin/sh -c "$installer_code" devsy-install "$@" ;; + sudo) sudo -n -E /bin/sh -c "$installer_code" devsy-install "$@" ;; + su) + # su implementations disagree about forwarding arguments after -c. + quote_installer_argument "$installer_code" + su_command="/bin/sh -c $quoted_argument devsy-install" + for installer_argument; do + quote_installer_argument "$installer_argument" + su_command="$su_command $quoted_argument" + done + su -s /bin/sh -c "$su_command" root + ;; + esac +} + +cleanup_temp() { + if [ -n "$temp_file" ]; then + run_installer_script 'rm -f -- "$1"' "$temp_file" || return 1 + temp_file= + fi +} + check_noexec() { - mount_path="$(df "${INSTALL_DIR}" | tail -n +2 | rev | cut -d' ' -f1 | rev)" + mount_path="$(df "$INSTALL_DIR" | tail -n +2 | rev | cut -d' ' -f1 | rev)" if mount | grep "on ${mount_path} " | grep -q noexec; then >&2 echo "Error: Installation directory $INSTALL_DIR is mounted with noexec flag" return 1 @@ -29,131 +72,112 @@ check_noexec() { handshake() { echo "ping" - read -r DEVSY_PING + read -r DEVSY_PING || return 1 if [ "$DEVSY_PING" != "pong" ]; then >&2 echo "Error: Handshake failed - received '$DEVSY_PING' instead of 'pong'" - exit 1 + return 1 fi } -inject_binary() { - echo "ARM-$(is_arm && echo -n 'true' || echo -n 'false')" +prepare_temp() { + cleanup_temp || return 1 + temp_file=$(run_installer_script 'mktemp "$1.XXXXXX"' "$INSTALL_PATH") || { + >&2 echo "Error: Failed to create a temporary agent in $INSTALL_DIR" + return 1 + } +} - temp_file="$(mktemp "$INSTALL_PATH.XXXXXX" 2>/dev/null || echo "$INSTALL_PATH.$$-$(date +%s)")" +publish_binary() { + if [ "$CHMOD_PATH" = "true" ]; then + run_installer_script 'chmod -- +x "$1"' "$temp_file" || { + >&2 echo "Error: Failed to make temporary agent executable" + return 1 + } + fi + run_installer_script '[ ! -d "$2" ] && mv -f -- "$1" "$2"' "$temp_file" "$INSTALL_PATH" || { + >&2 echo "Error: Failed to replace agent at $INSTALL_PATH" + return 1 + } + temp_file= +} +inject_binary() { + echo "ARM-$(is_arm && printf true || printf false)" if ! read -r BINARY_SIZE; then >&2 echo "Error: Failed to read binary size" return 1 fi - case "$BINARY_SIZE" in '' | *[!0-9]*) >&2 echo "Error: Invalid binary size: $BINARY_SIZE" return 1 ;; esac - - if ! $sh_c "head -c \"$BINARY_SIZE\" > \"$temp_file\""; then - >&2 echo "Error: Failed to write binary to $temp_file" - $sh_c "rm -f \"$temp_file\"" 2>/dev/null || true + prepare_temp || return 1 + run_installer_script 'head -c "$1" > "$2" && [ "$(wc -c < "$2" | tr -d "[:space:]")" = "$1" ]' "$BINARY_SIZE" "$temp_file" || { + >&2 echo "Error: Failed to receive complete agent binary" + cleanup_temp || true return 1 - fi - - if [ "$CHMOD_PATH" = "true" ]; then - $sh_c "chmod +x \"$temp_file\"" || { - >&2 echo "Error: Failed to make $temp_file executable" - $sh_c "rm -f \"$temp_file\"" 2>/dev/null || true - return 1 - } - fi - - if ! $sh_c "mv \"$temp_file\" \"$INSTALL_PATH\""; then - >&2 echo "Error: Failed to move binary to $INSTALL_PATH" - $sh_c "rm -f \"$temp_file\"" 2>/dev/null || true + } + publish_binary || { + cleanup_temp || true return 1 - fi - - return 0 + } } download_binary() { - DOWNLOAD_URL="{{ .DownloadAmd }}" + DOWNLOAD_URL={{ .DownloadAmd }} if is_arm; then - DOWNLOAD_URL="{{ .DownloadArm }}" + DOWNLOAD_URL={{ .DownloadArm }} fi - iteration=1 max_iteration=3 - while [ "$iteration" -le "$max_iteration" ]; do - temp_file="$(mktemp "$INSTALL_PATH.XXXXXX" 2>/dev/null || echo "$INSTALL_PATH.$$-$(date +%s)")" - trap 'rm -f "$temp_file"' EXIT - + prepare_temp || return 1 if command_exists curl; then - $sh_c "curl -fsSL \"$DOWNLOAD_URL\" -o \"$temp_file\"" && break + run_installer_script 'curl -fsSL --connect-timeout 10 --max-time 120 -o "$2" -- "$1"' "$DOWNLOAD_URL" "$temp_file" && break cmd_status=$? elif command_exists wget; then - $sh_c "wget -q \"$DOWNLOAD_URL\" -O \"$temp_file\"" && break + run_installer_script 'wget -q -T 120 -O "$2" -- "$1"' "$DOWNLOAD_URL" "$temp_file" && break cmd_status=$? else >&2 echo "Error: No download tool found (curl or wget required)" + cleanup_temp || true return 127 fi - - $sh_c "rm -f \"$temp_file\"" 2>/dev/null || true - - >&2 echo "Error: Download attempt $iteration failed (exit code: ${cmd_status})" - >&2 echo " URL: $DOWNLOAD_URL" - + cleanup_temp || return 1 + >&2 echo "Error: Download attempt $iteration failed (exit code: $cmd_status)" if [ "$iteration" -lt "$max_iteration" ]; then >&2 echo " Retrying in 10 seconds" - sleep 10 + sleep 10 || return 1 fi iteration=$((iteration + 1)) done - if [ "$iteration" -gt "$max_iteration" ]; then >&2 echo "Error: Failed to download devsy after $max_iteration attempts" return 1 fi - - if [ "$CHMOD_PATH" = "true" ]; then - $sh_c "chmod +x \"$temp_file\"" || { - >&2 echo "Error: Failed to make $temp_file executable" - $sh_c "rm -f \"$temp_file\"" 2>/dev/null || true - return 1 - } - fi - - if ! $sh_c "mv \"$temp_file\" \"$INSTALL_PATH\""; then - >&2 echo "Error: Failed to move downloaded binary to $INSTALL_PATH" - $sh_c "rm -f \"$temp_file\"" 2>/dev/null || true + publish_binary || { + cleanup_temp || true return 1 - fi - - return 0 + } } setup_sudo() { - if ! mkdir -p "$INSTALL_DIR" 2>/dev/null || - ! touch "$INSTALL_PATH" 2>/dev/null || - ! chmod +x "$INSTALL_PATH" 2>/dev/null || - ! rm -f "$INSTALL_PATH" 2>/dev/null; then - + if ! run_installer_script 'mkdir -p -- "$1" && probe=$(mktemp "$1/.devsy-write.XXXXXX") && rm -f -- "$probe"' "$INSTALL_DIR" 2>/dev/null; then if command_exists sudo; then if ! sudo -nl >/dev/null 2>&1; then >&2 echo "Error: sudo requires a password. Please configure NOPASSWD" - exit 1 + return 1 fi - sh_c='sudo -E sh -c' + INSTALLER_MODE=sudo elif command_exists su; then - sh_c='su -c' + INSTALLER_MODE=su else >&2 echo "Error: Need root access but sudo/su not available" - exit 1 + return 1 fi - - $sh_c "mkdir -p \"$INSTALL_DIR\"" + run_installer_script 'mkdir -p -- "$1"' "$INSTALL_DIR" || return 1 fi } @@ -163,7 +187,7 @@ install_agent() { >&2 echo "Download failed, attempting stdin injection" inject_binary || { >&2 echo "Error: Both download and injection failed" - exit 1 + return 1 } fi else @@ -171,7 +195,7 @@ install_agent() { >&2 echo "Injection failed, attempting download" download_binary || { >&2 echo "Error: Both injection and download failed" - exit 1 + return 1 } fi fi @@ -179,22 +203,20 @@ install_agent() { execute_command() { echo "done" - export DEVSY_AGENT_URL="{{ .DownloadBase }}" + export DEVSY_AGENT_URL={{ .DownloadBase }} {{ .Command }} } main() { - handshake - + handshake || return 1 if {{ .ExistsCheck }}; then - sh_c='sh -c' - setup_sudo - check_noexec || exit 1 - $sh_c "rm -f \"$INSTALL_PATH\" 2>/dev/null || true" - install_agent + setup_sudo || return 1 + check_noexec || return 1 + install_agent || return 1 fi - execute_command } +trap 'cleanup_temp' 0 +trap 'exit 1' HUP INT TERM main diff --git a/pkg/inject/script.go b/pkg/inject/script.go index 45eda1ac1f..1ee6fce59c 100644 --- a/pkg/inject/script.go +++ b/pkg/inject/script.go @@ -1,6 +1,7 @@ package inject import ( + "fmt" "path" "strconv" "strings" @@ -9,7 +10,7 @@ import ( ) func GenerateScript(script string, params *Params) (string, error) { - rawCode, err := template.FillTemplate(script, map[string]string{ + values := map[string]string{ "Command": params.Command, "ExistsCheck": params.ExistsCheck, "InstallDir": params.InstallDir(), @@ -19,12 +20,33 @@ func GenerateScript(script string, params *Params) (string, error) { "DownloadBase": params.DownloadURLs.Base, "DownloadAmd": params.DownloadURLs.Amd, "DownloadArm": params.DownloadURLs.Arm, - }) + } + for _, name := range []string{"Command", "ExistsCheck"} { + if strings.ContainsRune(values[name], 0) { + return "", fmt.Errorf("%s contains NUL", name) + } + } + for _, name := range []string{"InstallDir", "InstallFilename", "DownloadBase", "DownloadAmd", "DownloadArm"} { + quoted, err := QuotePOSIXLiteral(values[name]) + if err != nil { + return "", fmt.Errorf("quote %s: %w", name, err) + } + values[name] = quoted + } + rawCode, err := template.FillTemplate(stripCarriageReturns(script), values) if err != nil { return "", err } - return stripCarriageReturns(rawCode), nil + return rawCode, nil +} + +// QuotePOSIXLiteral encodes one literal word for a POSIX shell, not a login shell. +func QuotePOSIXLiteral(value string) (string, error) { + if strings.ContainsRune(value, 0) { + return "", fmt.Errorf("shell literal contains NUL") + } + return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'", nil } type Params struct { diff --git a/pkg/inject/script_test.go b/pkg/inject/script_test.go new file mode 100644 index 0000000000..6522d2eead --- /dev/null +++ b/pkg/inject/script_test.go @@ -0,0 +1,215 @@ +//nolint:gosec // These regressions execute generated scripts and executable fixtures under private test directories. +package inject + +import ( + "bytes" + "context" + "crypto/rand" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type BootstrapSuite struct{ suite.Suite } + +func TestBootstrapSuite(t *testing.T) { suite.Run(t, new(BootstrapSuite)) } + +func (s *BootstrapSuite) SetupSuite() { + if _, err := exec.LookPath("/bin/sh"); err != nil { + s.T().Skip("POSIX /bin/sh is unavailable") + } +} + +func (s *BootstrapSuite) TestLiteralRoundTrip() { + for _, value := range []string{"", "a'b\\c\n雪\r", "$(exit 42)`exit 43`\";$HOME()"} { + s.Run(fmt.Sprintf("%q", value), func() { + quoted, err := QuotePOSIXLiteral(value) + s.Require().NoError(err) + cmd := exec.CommandContext(s.T().Context(), "/bin/sh", "-c", "printf %s "+quoted) + output, err := cmd.Output() + s.Require().NoError(err) + s.Equal(value, string(output)) + }) + } + _, err := QuotePOSIXLiteral("a\x00b") + s.Error(err) +} + +func bootstrapParams(installPath string) *Params { + return &Params{ + Command: "cat", AgentRemotePath: installPath, DownloadURLs: &DownloadURLs{}, + ExistsCheck: "true", ShouldChmodPath: true, + } +} + +func runBootstrap( + t *testing.T, + params *Params, + stdin []byte, + extraEnv ...string, +) (string, string, error) { + t.Helper() + script, err := GenerateScript(Script, params) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) + cmd.Env = append(os.Environ(), extraEnv...) + cmd.Stdin = bytes.NewReader(stdin) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err = cmd.Run() + return stdout.String(), stderr.String(), err +} + +func binaryInput(binary []byte) []byte { + return append(fmt.Appendf(nil, "pong\n%d\n", len(binary)), binary...) +} + +func fakeBootstrapTools(t *testing.T, chmodFails bool) string { + t.Helper() + tools := t.TempDir() + scripts := map[string]string{"curl": "exit 1", "sleep": "exit 0"} + if chmodFails { + scripts["chmod"] = "exit 1" + } + for name, body := range scripts { + require.NoError( + t, + os.WriteFile(filepath.Join(tools, name), []byte("#!/bin/sh\n"+body+"\n"), 0o700), + ) + } + return "PATH=" + tools + string(os.PathListSeparator) + os.Getenv("PATH") +} + +func (s *BootstrapSuite) TestHostilePathAndLiveBinaryStream() { + root := s.T().TempDir() + sentinel := filepath.Join(root, "substitution-ran") + installPath := filepath.Join(root, "bin ' \\ \n雪 $(touch substitution-ran)", "agent '\"`$()") + params := bootstrapParams(installPath) + params.DownloadURLs = &DownloadURLs{Base: "https://invalid/'\"`$(touch " + sentinel + ")\\\r\n"} + params.Command = "printf '%s\\n' \"$DEVSY_AGENT_URL\"; cat" + binary := make([]byte, 4096) + _, err := rand.Read(binary) + s.Require().NoError(err) + copy(binary, []byte{0, '\'', '\\', '"', '$', '\n'}) + input := append(binaryInput(binary), []byte("live-tunnel\x00bytes\n")...) + stdout, stderr, err := runBootstrap(s.T(), params, input) + s.Require().NoError(err, stderr) + s.True(strings.HasPrefix(stdout, "ping\nARM-"), stdout) + s.Equal(1, strings.Count(stdout, "done\n")) + s.True( + strings.HasSuffix(stdout, "done\n"+params.DownloadURLs.Base+"\nlive-tunnel\x00bytes\n"), + stdout, + ) + installed, err := os.ReadFile(installPath) + s.Require().NoError(err) + s.Equal(binary, installed) + _, err = os.Stat(sentinel) + s.True(os.IsNotExist(err)) +} + +func (s *BootstrapSuite) TestFailedReplacementPreservesAgent() { + for _, failure := range []string{"short-transfer", "chmod"} { + s.Run(failure, func() { + installPath := filepath.Join(s.T().TempDir(), "agent") + original := []byte("old-working-agent") + s.Require().NoError(os.WriteFile(installPath, original, 0o750)) + before, err := os.Stat(installPath) + s.Require().NoError(err) + input := binaryInput([]byte("new-agent")) + if failure == "short-transfer" { + input = []byte("pong\n100\nshort") + } + stdout, stderr, err := runBootstrap( + s.T(), + bootstrapParams(installPath), + input, + fakeBootstrapTools(s.T(), failure == "chmod"), + ) + s.Error(err, stderr) + s.NotContains(stdout, "done\n") + installed, err := os.ReadFile(installPath) + s.Require().NoError(err) + s.Equal(original, installed) + after, err := os.Stat(installPath) + s.Require().NoError(err) + s.Equal(before.Mode(), after.Mode()) + files, err := filepath.Glob(installPath + ".*") + s.Require().NoError(err) + s.Empty(files) + }) + } +} + +func (s *BootstrapSuite) TestDownloadFailureFallsBackToStdin() { + installPath := filepath.Join(s.T().TempDir(), "agent") + params := bootstrapParams(installPath) + params.PreferAgentDownload = true + stdout, stderr, err := runBootstrap( + s.T(), + params, + binaryInput([]byte("replacement")), + fakeBootstrapTools(s.T(), false), + ) + s.Require().NoError(err, stderr) + s.Contains(stderr, "Download failed, attempting stdin injection") + s.Contains(stdout, "done\n") + installed, err := os.ReadFile(installPath) + s.Require().NoError(err) + s.Equal("replacement", string(installed)) +} + +func (s *BootstrapSuite) TestWarmAgentDoesNotInstall() { + params := bootstrapParams(filepath.Join(s.T().TempDir(), "missing", "agent")) + params.ExistsCheck = "false" + stdout, stderr, err := runBootstrap(s.T(), params, []byte("pong\nlive\n")) + s.Require().NoError(err, stderr) + s.Equal("ping\ndone\nlive\n", stdout) +} + +func (s *BootstrapSuite) TestGenerateRejectsNULAndParses() { + params := bootstrapParams("/tmp/agent") + fields := []*string{ + ¶ms.AgentRemotePath, + ¶ms.DownloadURLs.Base, + ¶ms.DownloadURLs.Amd, + ¶ms.DownloadURLs.Arm, + ¶ms.Command, + ¶ms.ExistsCheck, + } + for _, field := range fields { + old := *field + *field = "bad\x00value" + _, err := GenerateScript(Script, params) + s.Error(err) + *field = old + } + script, err := GenerateScript(Script, params) + s.Require().NoError(err) + cmd := exec.CommandContext(s.T().Context(), "/bin/sh", "-n") + cmd.Stdin = strings.NewReader(script) + output, err := cmd.CombinedOutput() + s.NoError(err, string(output)) +} + +func (s *BootstrapSuite) TestInvalidLiteralDoesNotReportExecution() { + params := bootstrapParams("bad\x00path") + executed, err := Inject(s.T().Context(), InjectOptions{ + Exec: func(context.Context, string, io.Reader, io.Writer, io.Writer) error { + s.T().Fatal("invalid literal must fail before execution") + return nil + }, + ScriptParams: params, + }) + s.Error(err) + s.False(executed) +} diff --git a/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx b/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx index c8e9237212..1080194911 100644 --- a/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx +++ b/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx @@ -15,7 +15,17 @@ sudo chown -R $(id -un):$(id -gn) . Local folders suit early work. Once the devcontainer works, push it to a Git repository and use that instead. -### Fish shell +### SSH host login shells + +The SSH provider supports Bash, Zsh, and Fish login shells on Linux. Agent installation runs through the host's executable `/bin/sh`; interactive terminals and environment probing continue to use the account's configured shell. The workspace container's shell is configured separately. + +The host needs standard bootstrap tools, including `mkdir`, `mktemp`, `head` with `-c`, `wc`, `chmod`, `mv`, and `rm`. A download uses `curl` or `wget`; when neither is available, Devsy can transfer the agent over the SSH connection. The agent directory must permit execution and either be writable by the account or allow noninteractive privilege escalation. + +If startup reports `command not found: sh -c`, check the installed SSH provider version and its release notes for the POSIX bootstrap fix. Changing the account's shell is unnecessary. Both the external OpenSSH client and the built-in SSH transport use the same bootstrap launcher; switching transports does not repair a missing `/bin/sh` or a restricted account. + +For locked-down accounts, ask the administrator to permit `/bin/sh` and the required tools, provide a writable executable agent directory, and check forced-command restrictions. Keep noninteractive shell startup output off stdout: it carries the bootstrap protocol and the live SSH tunnel. Send diagnostics to stderr and limit banners and prompts to interactive sessions. + +### Fish startup configuration Anything in `config.fish` runs on every `fish -c` call, which can interfere with `devsy agent workspace up`. Move your customizations inside the `if status is-interactive` block: From 8c9231803f980ba02b2a1fb47c0d74b4761ff859 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 13:45:05 -0600 Subject: [PATCH 02/11] fix(docs): clarify shell metadata in SSH matrix --- docs/issue-1621/matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/issue-1621/matrix.py b/docs/issue-1621/matrix.py index d755b50cee..d714f69d7d 100644 --- a/docs/issue-1621/matrix.py +++ b/docs/issue-1621/matrix.py @@ -46,8 +46,8 @@ def main(): '-payload', str(payload), '-input', str(input_file), '-check', 'false' if phase == 'warm' else 'true'] c = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=30) - row = dict(shell=shell, builtin=builtin, phase=phase, harnessExit=c.returncode, - **json.loads(c.stdout)) + row = {'shell': shell, 'builtin': builtin, 'phase': phase, + 'harnessExit': c.returncode, **json.loads(c.stdout)} row['rawPrefix'] = row.pop('rawStdout')[:30] row.pop('stdout') rows.append(row) From 7a7ffbeb80f728186f5d0ed9466938c75ca1e730 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 14:18:07 -0600 Subject: [PATCH 03/11] test(inject): make bootstrap faults deterministic under BusyBox --- docs/issue-1621/README.md | 2 ++ pkg/inject/script_test.go | 44 ++++++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index 17db6c9220..d94c96724e 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -57,6 +57,8 @@ The post-review provider artifact SHA-256 is `39468e46550ddf4a9ea3a1033cae10ef02 Native Linux execution of the actual `TestBootstrapSuite` and unchanged `TestInjectScript_RealShell*` tests passed with `/bin/sh` pointing at BusyBox. Cross-compilation only produced the test executable; these results came from executing it in the Linux clone. +The first GitHub Ubuntu BusyBox step exposed a test-double portability issue: Ubuntu's BusyBox static standalone applets bypassed PATH-based `sleep` and `chmod` doubles. The original failures were reproduced in a disposable Linux ARM64 container using Ubuntu Noble `busybox-static` `1:1.36.1-6ubuntu3.1`, the same package version as CI. Test-only shell functions now inject the faults in the outer and nested installer shells; successful binary/live-stream tests still execute the unmodified generated script. The chmod fault must emit its marker and the installer error, while download fallback must follow three failed attempts. The corrected bootstrap suite and unchanged real-shell transfer tests passed natively with that package as `/bin/sh`. No production installer or CI assertion was relaxed; final-head GitHub AMD64 execution remains a separate gate. + Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/source SHA-256: | Input | SHA-256 | diff --git a/pkg/inject/script_test.go b/pkg/inject/script_test.go index 6522d2eead..a3ac5e0634 100644 --- a/pkg/inject/script_test.go +++ b/pkg/inject/script_test.go @@ -54,15 +54,25 @@ func runBootstrap( t *testing.T, params *Params, stdin []byte, - extraEnv ...string, + faultHooks ...string, ) (string, string, error) { t.Helper() script, err := GenerateScript(Script, params) require.NoError(t, err) + if len(faultHooks) > 0 { + hooks := strings.Join(faultHooks, "\n") + "\n" + quotedHooks, quoteErr := QuotePOSIXLiteral(hooks) + require.NoError(t, quoteErr) + const installerAssignment = "installer_code=$1" + require.Equal(t, 1, strings.Count(script, installerAssignment)) + // BusyBox standalone applets bypass PATH doubles; functions override them in both shells. + script = hooks + strings.Replace( + script, installerAssignment, "installer_code="+quotedHooks+"\"$1\"", 1, + ) + } ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) - cmd.Env = append(os.Environ(), extraEnv...) cmd.Stdin = bytes.NewReader(stdin) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr @@ -74,20 +84,12 @@ func binaryInput(binary []byte) []byte { return append(fmt.Appendf(nil, "pong\n%d\n", len(binary)), binary...) } -func fakeBootstrapTools(t *testing.T, chmodFails bool) string { - t.Helper() - tools := t.TempDir() - scripts := map[string]string{"curl": "exit 1", "sleep": "exit 0"} +func bootstrapFailureHooks(chmodFails bool) string { + hooks := "curl() { return 1; }\nsleep() { :; }\n" if chmodFails { - scripts["chmod"] = "exit 1" + hooks += "chmod() { printf '%s\\n' 'injected chmod failure' >&2; return 1; }\n" } - for name, body := range scripts { - require.NoError( - t, - os.WriteFile(filepath.Join(tools, name), []byte("#!/bin/sh\n"+body+"\n"), 0o700), - ) - } - return "PATH=" + tools + string(os.PathListSeparator) + os.Getenv("PATH") + return hooks } func (s *BootstrapSuite) TestHostilePathAndLiveBinaryStream() { @@ -118,7 +120,8 @@ func (s *BootstrapSuite) TestHostilePathAndLiveBinaryStream() { } func (s *BootstrapSuite) TestFailedReplacementPreservesAgent() { - for _, failure := range []string{"short-transfer", "chmod"} { + const chmodFailure = "chmod" + for _, failure := range []string{"short-transfer", chmodFailure} { s.Run(failure, func() { installPath := filepath.Join(s.T().TempDir(), "agent") original := []byte("old-working-agent") @@ -133,9 +136,15 @@ func (s *BootstrapSuite) TestFailedReplacementPreservesAgent() { s.T(), bootstrapParams(installPath), input, - fakeBootstrapTools(s.T(), failure == "chmod"), + bootstrapFailureHooks(failure == chmodFailure), ) s.Error(err, stderr) + if failure == chmodFailure { + s.Contains(stderr, "injected chmod failure") + s.Contains(stderr, "Failed to make temporary agent executable") + } else { + s.Contains(stderr, "Failed to receive complete agent binary") + } s.NotContains(stdout, "done\n") installed, err := os.ReadFile(installPath) s.Require().NoError(err) @@ -158,10 +167,11 @@ func (s *BootstrapSuite) TestDownloadFailureFallsBackToStdin() { s.T(), params, binaryInput([]byte("replacement")), - fakeBootstrapTools(s.T(), false), + bootstrapFailureHooks(false), ) s.Require().NoError(err, stderr) s.Contains(stderr, "Download failed, attempting stdin injection") + s.Equal(3, strings.Count(stderr, "Download attempt ")) s.Contains(stdout, "done\n") installed, err := os.ReadFile(installPath) s.Require().NoError(err) From 25d320a0d414a3c3c2a6c0d7b0bf56188de04a96 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 14:45:23 -0600 Subject: [PATCH 04/11] test(delivery): observe cancellation after runtime completion --- docs/issue-1621/README.md | 2 ++ .../exec_stream_cancellation_unix_test.go | 20 +++++-------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index d94c96724e..b801d9deba 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -59,6 +59,8 @@ Native Linux execution of the actual `TestBootstrapSuite` and unchanged `TestInj The first GitHub Ubuntu BusyBox step exposed a test-double portability issue: Ubuntu's BusyBox static standalone applets bypassed PATH-based `sleep` and `chmod` doubles. The original failures were reproduced in a disposable Linux ARM64 container using Ubuntu Noble `busybox-static` `1:1.36.1-6ubuntu3.1`, the same package version as CI. Test-only shell functions now inject the faults in the outer and nested installer shells; successful binary/live-stream tests still execute the unmodified generated script. The chmod fault must emit its marker and the installer error, while download fallback must follow three failed attempts. The corrected bootstrap suite and unchanged real-shell transfer tests passed natively with that package as `/bin/sh`. No production installer or CI assertion was relaxed; final-head GitHub AMD64 execution remains a separate gate. +A later Ubuntu race run exposed an existing delivery-test observer race: stopping its pending `context.AfterFunc` could discard notification of a real attempt cancellation. The fixture now records the actual context cause after the runtime joins. The expected timeout causes, fresh-source retries, exact source close counts, bounded completion, and uncanceled outer-context assertions remain intact. Production delivery behavior is unchanged. + Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/source SHA-256: | Input | SHA-256 | diff --git a/pkg/agent/delivery/exec_stream_cancellation_unix_test.go b/pkg/agent/delivery/exec_stream_cancellation_unix_test.go index a73b98a70f..528b4930a4 100644 --- a/pkg/agent/delivery/exec_stream_cancellation_unix_test.go +++ b/pkg/agent/delivery/exec_stream_cancellation_unix_test.go @@ -129,27 +129,17 @@ func cancellationDeliveryExecutor( t *testing.T, runtime driver.ArgvExecDriver, canceled chan<- error, ) func(context.Context, []string, driver.Streams) error { return func(ctx context.Context, argv []string, streams driver.Streams) error { + err := runtime.CommandContainerArgv(ctx, "owned-delivery-test", argv, streams) if streams.Stdin != nil { - callbackDone := make(chan struct{}) - stop := context.AfterFunc(ctx, func() { - defer close(callbackDone) + if cause := context.Cause(ctx); cause != nil { select { - case canceled <- context.Cause(ctx): + case canceled <- cause: default: t.Error("unexpected extra attempt cancellation") } - }) - defer func() { - if !stop() { - select { - case <-callbackDone: - case <-time.After(execStreamCancellationTestBound): - t.Error("attempt observer callback did not join") - } - } - }() + } } - return runtime.CommandContainerArgv(ctx, "owned-delivery-test", argv, streams) + return err } } From b3dcd04c3b08bd164fef3eadf8479630f09bf41a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 15:18:12 -0600 Subject: [PATCH 05/11] test(inject): bridge hostile-path input after bootstrap completion --- docs/issue-1621/README.md | 2 ++ pkg/inject/script_test.go | 30 +++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index b801d9deba..d93deeac31 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -61,6 +61,8 @@ The first GitHub Ubuntu BusyBox step exposed a test-double portability issue: Ub A later Ubuntu race run exposed an existing delivery-test observer race: stopping its pending `context.AfterFunc` could discard notification of a real attempt cancellation. The fixture now records the actual context cause after the runtime joins. The expected timeout causes, fresh-source retries, exact source close counts, bounded completion, and uncanceled outer-context assertions remain intact. Production delivery behavior is unchanged. +The macOS hostile-path fixture previously pipelined tunnel bytes immediately after the binary, before `done`. Independent fragmented-write reproductions showed BSD `head` reading ahead into those premature bytes. Production `Inject` waits for `done` before bridging user stdin. The fixture now exercises that actual API and sequencing, retaining exact binary, URL, raw protocol, live NUL-containing payload, and substitution-sentinel assertions. The existing real-shell transfer test remains unchanged. + Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/source SHA-256: | Input | SHA-256 | diff --git a/pkg/inject/script_test.go b/pkg/inject/script_test.go index a3ac5e0634..1a210fbc9d 100644 --- a/pkg/inject/script_test.go +++ b/pkg/inject/script_test.go @@ -93,25 +93,45 @@ func bootstrapFailureHooks(chmodFails bool) string { } func (s *BootstrapSuite) TestHostilePathAndLiveBinaryStream() { + const liveInput = "live-tunnel\x00bytes\n" root := s.T().TempDir() sentinel := filepath.Join(root, "substitution-ran") installPath := filepath.Join(root, "bin ' \\ \n雪 $(touch substitution-ran)", "agent '\"`$()") params := bootstrapParams(installPath) params.DownloadURLs = &DownloadURLs{Base: "https://invalid/'\"`$(touch " + sentinel + ")\\\r\n"} - params.Command = "printf '%s\\n' \"$DEVSY_AGENT_URL\"; cat" + params.Command = fmt.Sprintf("printf '%%s\\n' \"$DEVSY_AGENT_URL\"; head -c %d", len(liveInput)) binary := make([]byte, 4096) _, err := rand.Read(binary) s.Require().NoError(err) copy(binary, []byte{0, '\'', '\\', '"', '$', '\n'}) - input := append(binaryInput(binary), []byte("live-tunnel\x00bytes\n")...) - stdout, stderr, err := runBootstrap(s.T(), params, input) - s.Require().NoError(err, stderr) + var rawOutput, stderr syncBuffer + var userOutput bytes.Buffer + executed, err := Inject(s.T().Context(), InjectOptions{ + Exec: func(ctx context.Context, script string, stdin io.Reader, stdout, diagnostics io.Writer) error { + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) + cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, io.MultiWriter( + stdout, + &rawOutput, + ), diagnostics + return cmd.Run() + }, + LocalFile: func(bool) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(binary)), nil }, + ScriptParams: params, + Stdin: strings.NewReader(liveInput), + Stdout: &userOutput, + Stderr: &stderr, + Timeout: 5 * time.Second, + }) + s.Require().NoError(err, stderr.String()) + s.True(executed) + stdout := rawOutput.String() s.True(strings.HasPrefix(stdout, "ping\nARM-"), stdout) s.Equal(1, strings.Count(stdout, "done\n")) s.True( - strings.HasSuffix(stdout, "done\n"+params.DownloadURLs.Base+"\nlive-tunnel\x00bytes\n"), + strings.HasSuffix(stdout, "done\n"+params.DownloadURLs.Base+"\n"+liveInput), stdout, ) + s.Equal(params.DownloadURLs.Base+"\n"+liveInput, userOutput.String()) installed, err := os.ReadFile(installPath) s.Require().NoError(err) s.Equal(binary, installed) From 16532a90792a78839335ffb066f95f1163776da4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 15:42:57 -0600 Subject: [PATCH 06/11] ci: serialize Linux CLI snapshots to bound disk usage --- .github/workflows/pr-ci.yml | 17 ++++++++++++++++- docs/issue-1621/README.md | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 05e9d396c9..14e775d647 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -229,6 +229,7 @@ jobs: matrix: include: - runner: ubuntu-latest + job_timeout_minutes: 45 - runner: macos-latest - runner: windows-latest # Cold builds need room to save the Go cache after artifact uploads. @@ -278,14 +279,28 @@ jobs: name: devsy-ci-podman-${{ steps.os.outputs.runner_os }} path: ${{ runner.temp }}/devsy-ci-podman* + - name: disk usage before Linux snapshot + if: runner.os == 'Linux' + run: | + df -h /tmp "$RUNNER_TEMP" "$GITHUB_WORKSPACE" + du -sh "$(go env GOCACHE)" "$(go env GOMODCACHE)" + - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7 with: distribution: goreleaser version: "~> v2" - args: build --id devsy-${{ steps.os.outputs.runner_os }} --snapshot + args: >- + build --id devsy-${{ steps.os.outputs.runner_os }} --snapshot + ${{ runner.os == 'Linux' && '--parallelism=1' || '' }} env: DEVSY_CLI_VERSION: v0.0.0 + - name: disk usage after Linux snapshot + if: always() && runner.os == 'Linux' + run: | + df -h /tmp "$RUNNER_TEMP" "$GITHUB_WORKSPACE" + du -sh "$(go env GOCACHE)" "$(go env GOMODCACHE)" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: devsy-${{ steps.os.outputs.runner_os }} diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index d93deeac31..1b13e56937 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -63,6 +63,8 @@ A later Ubuntu race run exposed an existing delivery-test observer race: stoppin The macOS hostile-path fixture previously pipelined tunnel bytes immediately after the binary, before `done`. Independent fragmented-write reproductions showed BSD `head` reading ahead into those premature bytes. Production `Inject` waits for `done` before bridging user stdin. The fixture now exercises that actual API and sequencing, retaining exact binary, URL, raw protocol, live NUL-containing payload, and substitution-sentinel assertions. The existing real-shell transfer test remains unchanged. +Repeated Ubuntu 26.04 CLI snapshot builds exhausted runner disk quota while compiling or linking concurrent Linux architectures. The Linux build job now serializes those same two targets and allows 45 minutes for cold serial execution and follow-on steps. Read-only disk/cache diagnostics surround the snapshot. No target, test, action pin, permission, cache, or unrelated runner file is removed; macOS and Windows build concurrency is unchanged. Final-head GitHub execution remains the resource-fix gate. + Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/source SHA-256: | Input | SHA-256 | From ca50d198f1f415f4e859d4468b851ef529eb0afa Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 17:04:31 -0600 Subject: [PATCH 07/11] fix(inject): preserve bundled Dockerless bootstrap shell --- docs/issue-1621/README.md | 19 +++++- pkg/inject/dockerless_test.go | 110 ++++++++++++++++++++++++++++++++++ pkg/inject/inject.sh | 20 +++++-- 3 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 pkg/inject/dockerless_test.go diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index 1b13e56937..abc0a97665 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -2,6 +2,8 @@ This evidence concerns the SSH provider plus core agent injection. It does not qualify every provider. No release, merge, or managed activation is authorized by this document. +Fixture snapshots below retain their original artifact revisions and scope. Later provider PR #152 CI on `e9c4c75e81853bcb11cf707d4aff9a49bb739411` passed all 18 Linux/Bash workspace integration cases with released core v1.24.0, five-platform snapshot packaging, and native Windows stdin tests. This adds bounded whole-CLI and Windows-runner evidence; it does not establish Desktop acceptance, whole-workspace coverage across every login shell, or published artifacts. + ## Actual Linux fixture The 2026-10-20 run used disposable Ubuntu 26.04 aarch64 containers on a macOS host, OpenSSH 10.2p1, Bash 5.3.9, Zsh 5.9, Fish 4.2.1, Dash `/bin/sh`, and BusyBox 1.37.0. Five distinct non-root passwd entries used `/bin/{bash,zsh,fish,dash,ash}`. Password authentication, PAM, root login, and MOTD were disabled; public-key authentication used fresh temporary keys. Only an ephemeral localhost port was published. No host accounts, servers, shell configuration, or SSH keys were changed. @@ -70,7 +72,8 @@ Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/sour | Input | SHA-256 | | --- | --- | | Original v1.24.0 `inject.sh` | `c2effffd6bbb9e42146bce22e2393f55f19b88d7b152f59d09c3c1e749bc2e0b` | -| Corrected `pkg/inject/inject.sh` | `b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3` | +| Pre-Dockerless-fix `pkg/inject/inject.sh` | `b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3` | +| Dockerless-compatible `pkg/inject/inject.sh` | `61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb097090294` | | `pkg/inject/script.go` | `752c82c5f3df1e60359658bc961c557b35995636dab88618ca8e791e21bf8e3e` | | `pkg/agent/inject.go` | `3ea3bb44cbc37132bde6569b68e0425ea8c91d744c3a6ea94988d99d3e6c61e7` | | Actual Linux agent executable | `5e5853e9e0653830858fe2438b46a7e8eb94a927bbbb4fda3142c2ddbe173bc4` | @@ -95,6 +98,20 @@ python3 docs/issue-1621/ssh-fixture.py teardown --state "$FIXTURE_STATE" --name The `forced-replacement` synthetic case explicitly sets the installation condition true; it is not a version-parser test. `real-tunnel.go -check` accepts the actual generated version check when testing version selection. Low-level framing failures can be reproduced with `inject-harness -script-out`, then passing `pong\n100\nabc` and EOF to the provider `COMMAND` while its invocation-local PATH omits curl/wget. Do not modify host binaries or accounts. +## Dockerless installer compatibility + +Core E2E on `16532a90792a78839335ffb066f95f1163776da4` exposed a separate container boundary. Kubernetes native staging failed first with a broken stdin stream/WebSocket EOF. Its cleanup call supplied no streams, which the Kubernetes API rejects. Those driver/delivery implementations are unchanged from the baseline; the initiating stream failure remains unproven. The subsequent legacy fallback exposed a confirmed regression: the Dockerless bootstrap filesystem has `/.dockerless/bin/sh` but no `/bin/sh`. + +Ordinary disposable execution of `ghcr.io/devsy-org/dockerless@sha256:8910f52719e38ffe986652efa04796d781575797627cdba3dd1ba8f977078303` confirmed both image variants use a static BusyBox shell at that bundled path. The ARM64 manifest is `sha256:1c1e4242bfe3662fe6531f52833b185c1cefc6e0ff01aa4279df3edd3a359663`; AMD64 is `sha256:58621764ce6d3921b68b05b7fb9fdca7250fc17f906e919961ba8725d87c7fd4`. AMD64 executed through existing emulation on the ARM64 host, rather than native AMD64 hardware. + +The immutable pre-fix bootstrap (`b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3`) failed after `ping`, without requesting a payload. Original v1.24.0 injection succeeded in the same ARM64 image with a synthetic ELF payload and live follow-on. The corrected bootstrap selects the fixed bundled interpreter only when `DOCKERLESS=true`, `/bin/sh` is not executable, and the bundled interpreter is executable. Ordinary targets retain `/bin/sh`, which is preferred even in Dockerless containers when available. Missing prerequisites fail before permission probes or escalation; warm reuse does not require an installer. + +The corrected script (`61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb097090294`) passed six ordinary-image cold/warm/forced synthetic cells across both architectures, including hostile literal paths and exact 4 KiB plus live follow-on bytes. Four negative cells (disabled Dockerless or both interpreters missing, each architecture) failed explicitly without requesting payloads. The source-exact native staging command and cleanup separately passed on both architectures. These process tests do not reproduce Dockerless `start --wait`, its filesystem transition, or Kubernetes exec/WebSocket transport, and do not establish the initiating native failure's cause. No builder, privileged container, host mount, or security override was used. + +The actual bundled BusyBox `su` wrapper also preserved empty and hostile literal arguments byte-for-byte and live stdin on both architectures, using the existing root account from a root process. This verifies shell/argument compatibility, not non-root privilege escalation. Both diagnostic containers were removed after validation. + +Follow-ups remain precise: investigate Kubernetes staging diagnostics/stream closure in the Dockerless filesystem transition, and provide an explicit diagnostic stream for staged-file cleanup rather than submitting an all-false exec stream request. This change preserves native argv routing and configured interactive shells. SSH provider bootstrapping still requires `/bin/sh` before the core script executes. + ## T9 boundary audit and release coordination Read-only inventory at base core `d3f6e421642575c5072badcf56bf5b0a5606bd41`: diff --git a/pkg/inject/dockerless_test.go b/pkg/inject/dockerless_test.go new file mode 100644 index 0000000000..d41b5fe1ba --- /dev/null +++ b/pkg/inject/dockerless_test.go @@ -0,0 +1,110 @@ +//nolint:gosec // Generated scripts and interpreter fixtures execute only under private test directories. +package inject + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + standardInterpreter = "standard" + bundledInterpreter = "bundled" +) + +type installerShellCase struct { + name, flag, selected string + standard, bundled bool +} + +func (s *BootstrapSuite) TestInstallerShellSelection() { + for _, tc := range []installerShellCase{ + {name: "ordinary", standard: true, bundled: true, selected: standardInterpreter}, + { + name: "dockerless keeps standard", flag: strconv.FormatBool(true), + standard: true, bundled: true, selected: standardInterpreter, + }, + {name: "dockerless bundled", flag: strconv.FormatBool(true), bundled: true, selected: bundledInterpreter}, + {name: "ordinary missing standard", bundled: true}, + {name: "disabled dockerless", flag: "false", bundled: true}, + {name: "missing both", flag: strconv.FormatBool(true)}, + } { + s.Run(tc.name, func() { + root := s.T().TempDir() + installPath := filepath.Join(root, "agent '\\\n雪 $(touch substitution-ran)") + original := []byte("working-agent") + s.Require().NoError(os.WriteFile(installPath, original, 0o750)) + params := bootstrapParams(installPath) + params.Command = "printf 'ready\\n'" + script, err := GenerateScript(Script, params) + s.Require().NoError(err) + script = installerShellFixture(s.T(), script, root, tc) + ctx, cancel := context.WithTimeout(s.T().Context(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) + cmd.Dir = root + cmd.Env = append(os.Environ(), "DOCKERLESS="+tc.flag) + payload := bytes.Repeat([]byte{0, '\'', '\\', '$', '\n', 255}, 700) + cmd.Stdin = bytes.NewReader(binaryInput(payload)) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err = cmd.Run() + if tc.selected == "" { + s.Require().Error(err) + s.Contains(stderr.String(), "requires executable /bin/sh") + s.NotContains(stderr.String(), "su:") + s.Equal("ping\n", stdout.String()) + payload = original + } else { + s.Require().NoError(err, stderr.String()) + s.True(strings.HasSuffix(stdout.String(), "done\nready\n"), stdout.String()) + s.Equal(1, strings.Count(stdout.String(), "done\n")) + } + installed, readErr := os.ReadFile(installPath) + s.Require().NoError(readErr) + s.Equal(payload, installed) + for _, name := range []string{standardInterpreter, bundledInterpreter} { + _, statErr := os.Stat(filepath.Join(root, name+"-used")) + s.Equal(name == tc.selected, statErr == nil) + } + _, statErr := os.Stat(filepath.Join(root, "substitution-ran")) + s.True(os.IsNotExist(statErr)) + }) + } +} + +func installerShellFixture(t *testing.T, script, root string, tc installerShellCase) string { + t.Helper() + for _, interpreter := range []struct { + assignment, name string + available bool + }{ + {"INSTALLER_SHELL=/bin/sh", standardInterpreter, tc.standard}, + {"DOCKERLESS_SHELL=/.dockerless/bin/sh", bundledInterpreter, tc.bundled}, + } { + fixture := filepath.Join(root, interpreter.name+" shell '") + if interpreter.available { + marker, quoteErr := QuotePOSIXLiteral(filepath.Join(root, interpreter.name+"-used")) + require.NoError(t, quoteErr) + require.NoError(t, os.WriteFile(fixture, []byte( + "#!/bin/sh\nprintf x >> "+marker+"\nexec /bin/sh \"$@\"\n", + ), 0o700)) + } + quoted, quoteErr := QuotePOSIXLiteral(fixture) + require.NoError(t, quoteErr) + require.Equal(t, 1, strings.Count(script, interpreter.assignment)) + // Model an isolated target's fixed paths without changing the host filesystem. + script = strings.Replace(script, interpreter.assignment, strings.SplitN( + interpreter.assignment, "=", 2, + )[0]+"="+quoted, 1) + } + return script +} diff --git a/pkg/inject/inject.sh b/pkg/inject/inject.sh index 467de14611..a7977c203a 100644 --- a/pkg/inject/inject.sh +++ b/pkg/inject/inject.sh @@ -7,6 +7,8 @@ INSTALL_PATH="$INSTALL_DIR/$INSTALL_FILENAME" PREFER_DOWNLOAD={{ .PreferDownload }} CHMOD_PATH={{ .ChmodPath }} INSTALLER_MODE=direct +INSTALLER_SHELL=/bin/sh +DOCKERLESS_SHELL=/.dockerless/bin/sh temp_file= command_exists() { @@ -39,17 +41,19 @@ run_installer_script() { installer_code=$1 shift case "$INSTALLER_MODE" in - direct) /bin/sh -c "$installer_code" devsy-install "$@" ;; - sudo) sudo -n -E /bin/sh -c "$installer_code" devsy-install "$@" ;; + direct) "$INSTALLER_SHELL" -c "$installer_code" devsy-install "$@" ;; + sudo) sudo -n -E "$INSTALLER_SHELL" -c "$installer_code" devsy-install "$@" ;; su) # su implementations disagree about forwarding arguments after -c. quote_installer_argument "$installer_code" - su_command="/bin/sh -c $quoted_argument devsy-install" + su_command=" -c $quoted_argument devsy-install" + quote_installer_argument "$INSTALLER_SHELL" + su_command="$quoted_argument$su_command" for installer_argument; do quote_installer_argument "$installer_argument" su_command="$su_command $quoted_argument" done - su -s /bin/sh -c "$su_command" root + su -s "$INSTALLER_SHELL" -c "$su_command" root ;; esac } @@ -164,6 +168,14 @@ download_binary() { } setup_sudo() { + # Dockerless injects before the image filesystem supplies /bin/sh. + if [ "${DOCKERLESS:-}" = "true" ] && [ ! -x "$INSTALLER_SHELL" ] && [ -x "$DOCKERLESS_SHELL" ]; then + INSTALLER_SHELL=$DOCKERLESS_SHELL + fi + if [ ! -x "$INSTALLER_SHELL" ]; then + >&2 echo "Error: Agent installation requires executable /bin/sh (or the bundled Dockerless shell in Dockerless containers)" + return 1 + fi if ! run_installer_script 'mkdir -p -- "$1" && probe=$(mktemp "$1/.devsy-write.XXXXXX") && rm -f -- "$probe"' "$INSTALL_DIR" 2>/dev/null; then if command_exists sudo; then if ! sudo -nl >/dev/null 2>&1; then From e9fef6f3d5513cc6900354f433e020842c3cee88 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 18:19:35 -0600 Subject: [PATCH 08/11] test(inject): cover sudo and su installer dispatch --- docs/issue-1621/README.md | 2 + pkg/inject/installer_modes_test.go | 140 +++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 pkg/inject/installer_modes_test.go diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index abc0a97665..a2b13a1606 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -110,6 +110,8 @@ The corrected script (`61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb09 The actual bundled BusyBox `su` wrapper also preserved empty and hostile literal arguments byte-for-byte and live stdin on both architectures, using the existing root account from a root process. This verifies shell/argument compatibility, not non-root privilege escalation. Both diagnostic containers were removed after validation. +Persistent `TestPrivilegedInstallerModesLiveStream` and `TestPrivilegedInstallerFailureStatus` exercise the real dispatcher in sudo/su mode using strict test doubles. The doubles validate invocation flags and pass the requested code unchanged to a real POSIX shell; markers stay off protocol stdout. Actual `Inject` sequencing checks hostile paths/URLs, exact binary and live NUL-containing bytes, one `done`, and failure status 37. These deterministic tests require no host privilege operation and complement, rather than replace, the real-tool evidence above. + Follow-ups remain precise: investigate Kubernetes staging diagnostics/stream closure in the Dockerless filesystem transition, and provide an explicit diagnostic stream for staged-file cleanup rather than submitting an all-false exec stream request. This change preserves native argv routing and configured interactive shells. SSH provider bootstrapping still requires `/bin/sh` before the core script executes. ## T9 boundary audit and release coordination diff --git a/pkg/inject/installer_modes_test.go b/pkg/inject/installer_modes_test.go new file mode 100644 index 0000000000..78b29a3e49 --- /dev/null +++ b/pkg/inject/installer_modes_test.go @@ -0,0 +1,140 @@ +//nolint:gosec // Generated scripts and privilege-command doubles execute only in private test directories. +package inject + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + testSudoMode = "sudo" + testSuMode = "su" +) + +const installerModeDoubles = ` +sudo() { + [ "$#" -ge 6 ] && [ "$1" = -n ] && [ "$2" = -E ] && + [ "$3" = /bin/sh ] && [ "$4" = -c ] && [ "$6" = devsy-install ] || return 98 + printf 'sudo\n' >> "$INSTALLER_TEST_MARKER" || return 99 + shift 2 + "$@" +} +su() { + [ "$#" -eq 5 ] && [ "$1" = -s ] && [ "$2" = /bin/sh ] && + [ "$3" = -c ] && [ "$5" = root ] || return 98 + printf 'su\n' >> "$INSTALLER_TEST_MARKER" || return 99 + "$2" -c "$4" +} +` + +func installerModeFixture(t *testing.T, script, mode, marker string) string { + t.Helper() + require.Contains(t, []string{testSudoMode, testSuMode}, mode) + const assignment = "INSTALLER_MODE=direct" + require.Equal(t, 1, strings.Count(script, assignment)) + quotedMarker, err := QuotePOSIXLiteral(marker) + require.NoError(t, err) + return "INSTALLER_TEST_MARKER=" + quotedMarker + "\n" + installerModeDoubles + + strings.Replace(script, assignment, "INSTALLER_MODE="+mode, 1) +} + +func (s *BootstrapSuite) TestPrivilegedInstallerModesLiveStream() { + for _, mode := range []string{testSudoMode, testSuMode} { + s.Run(mode, func() { s.checkInstallerModeLiveStream(mode) }) + } +} + +func (s *BootstrapSuite) checkInstallerModeLiveStream(mode string) { + const liveInput = "live-tunnel\x00bytes\n" + root := s.T().TempDir() + marker := filepath.Join(root, "mode-used") + params := bootstrapParams(filepath.Join( + root, "dir '\\\n雪", "agent $(touch substitution-ran)`touch substitution-ran`", + )) + params.DownloadURLs.Base = "https://invalid/'\"`$(touch substitution-ran)\\\r\n雪" + params.Command = fmt.Sprintf("printf '%%s\\n' \"$DEVSY_AGENT_URL\"; head -c %d", len(liveInput)) + rendered, err := GenerateScript(Script, params) + s.Require().NoError(err) + fixture := installerModeFixture(s.T(), rendered, mode, marker) + payload := bytes.Repeat([]byte{0, '\'', '\\', '"', '$', '\n', '\r', 255}, 512) + var rawOutput, stderr syncBuffer + var userOutput bytes.Buffer + executed, err := Inject(s.T().Context(), InjectOptions{ + Exec: func(ctx context.Context, script string, stdin io.Reader, stdout, diagnostics io.Writer) error { + if script != rendered { + return fmt.Errorf("unexpected rendered bootstrap") + } + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", fixture) + cmd.Dir = root + cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, io.MultiWriter( + stdout, + &rawOutput, + ), diagnostics + return cmd.Run() + }, + LocalFile: func(bool) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(payload)), nil }, + ScriptParams: params, + Stdin: strings.NewReader(liveInput), + Stdout: &userOutput, + Stderr: &stderr, + Timeout: 5 * time.Second, + }) + s.Require().NoError(err, stderr.String()) + s.True(executed) + s.Empty(stderr.String()) + s.Equal(params.DownloadURLs.Base+"\n"+liveInput, userOutput.String()) + s.True(strings.HasPrefix(rawOutput.String(), "ping\nARM-"), rawOutput.String()) + s.Equal(1, strings.Count(rawOutput.String(), "done\n")) + s.True(strings.HasSuffix(rawOutput.String(), "done\n"+userOutput.String())) + installed, err := os.ReadFile(params.AgentRemotePath) + s.Require().NoError(err) + s.Equal(payload, installed) + assertInstallerModeUsed(s.T(), marker, mode) + _, err = os.Stat(filepath.Join(root, "substitution-ran")) + s.True(os.IsNotExist(err)) +} + +func (s *BootstrapSuite) TestPrivilegedInstallerFailureStatus() { + for _, mode := range []string{testSudoMode, testSuMode} { + s.Run(mode, func() { + root := s.T().TempDir() + marker := filepath.Join(root, "mode-used") + script, err := GenerateScript(Script, bootstrapParams(filepath.Join(root, "agent"))) + s.Require().NoError(err) + script = installerModeFixture(s.T(), script, mode, marker) + const entry = "\nmain\n" + s.Require().Equal(1, strings.Count(script, entry)) + script = strings.Replace(script, entry, "\nrun_installer_script 'exit \"$1\"' 37\n", 1) + ctx, cancel := context.WithTimeout(s.T().Context(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) + output, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + s.Require().ErrorAs(err, &exitErr) + s.Equal(37, exitErr.ExitCode()) + s.Empty(output) + assertInstallerModeUsed(s.T(), marker, mode) + }) + } +} + +func assertInstallerModeUsed(t *testing.T, marker, mode string) { + t.Helper() + used, err := os.ReadFile(marker) + require.NoError(t, err) + lines := strings.Fields(string(used)) + require.NotEmpty(t, lines) + for _, line := range lines { + require.Equal(t, mode, line) + } +} From a7dbc08e68b84ded40be9de34cbbc0416f21ae53 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 19:38:41 -0600 Subject: [PATCH 09/11] fix(inject): publish readable staged agents --- docs/issue-1621/README.md | 13 ++++++++- docs/issue-1621/matrix.py | 29 +++++++++++++++++-- pkg/inject/dockerless_test.go | 9 +++++- pkg/inject/inject.sh | 2 +- pkg/inject/installer_modes_test.go | 3 ++ pkg/inject/script_test.go | 3 ++ .../troubleshooting/linux-troubleshooting.mdx | 2 ++ 7 files changed, 55 insertions(+), 6 deletions(-) diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index a2b13a1606..560bb61307 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -73,7 +73,8 @@ Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/sour | --- | --- | | Original v1.24.0 `inject.sh` | `c2effffd6bbb9e42146bce22e2393f55f19b88d7b152f59d09c3c1e749bc2e0b` | | Pre-Dockerless-fix `pkg/inject/inject.sh` | `b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3` | -| Dockerless-compatible `pkg/inject/inject.sh` | `61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb097090294` | +| Pre-permissions-fix Dockerless-compatible `pkg/inject/inject.sh` | `61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb097090294` | +| Readable staged-agent `pkg/inject/inject.sh` | `1c2b70523285dcca664a61180e5e56fccd09af4ef08a005cdf9ff2d36e62dcdd` | | `pkg/inject/script.go` | `752c82c5f3df1e60359658bc961c557b35995636dab88618ca8e791e21bf8e3e` | | `pkg/agent/inject.go` | `3ea3bb44cbc37132bde6569b68e0425ea8c91d744c3a6ea94988d99d3e6c61e7` | | Actual Linux agent executable | `5e5853e9e0653830858fe2438b46a7e8eb94a927bbbb4fda3142c2ddbe173bc4` | @@ -114,6 +115,16 @@ Persistent `TestPrivilegedInstallerModesLiveStream` and `TestPrivilegedInstaller Follow-ups remain precise: investigate Kubernetes staging diagnostics/stream closure in the Dockerless filesystem transition, and provide an explicit diagnostic stream for staged-file cleanup rather than submitting an all-false exec stream request. This change preserves native argv routing and configured interactive shells. SSH provider bootstrapping still requires `/bin/sh` before the core script executes. +## Elevated installation permissions + +CodeRabbit identified a staging regression: `mktemp` creates mode `0600`, so adding only executable bits published a root-owned `0711` agent after sudo installation. A non-root user could execute that file but could not reopen its own executable. Publication now sets the staged agent to `0755` before the atomic rename. The chmod-failure path still retains the prior agent and removes only staging files. Persistent direct, sudo/su-dispatcher, and Dockerless-selection tests assert the final mode; failure cases retain the original mode. + +An ordinary disposable Linux ARM64 container reproduced the mechanism with the actual `GenerateScript` and `Inject` APIs, a non-root uid 1001, and real noninteractive sudo into an unwritable root-owned destination. The immutable `61d88ef1...` script published root-owned `0711`; the native synthetic ELF started but failed `os.Open(os.Executable())` with permission denied and exit 42. With script `1c2b7052...`, the identical 2,403,587-byte ELF was root-owned `0755`, reopened and read itself, and completed an exact 4,124-byte binary/live exchange with empty stderr and no error. Both runs requested one payload. The rendered scripts differed only in `chmod -- +x` versus `chmod -- 0755`. + +The synthetic ELF SHA-256 was `8f3d322e43d67c91d924de36054d2a66702720da0ea109242c89595b85af77d3`; successful stdout SHA-256 was `445e35c83283238eb1b2ef4bfcdc4e6a53f9600c7f99136a662e8304d6298a12`. This is an actual permissions and streaming reproduction, not a new full Devsy-agent or whole-product qualification. Ubuntu uutils 0.10.0, GNU coreutils 9.4, and BusyBox 1.38 accepted the fixed chmod syntax. The container, its internal test user and sudo policy were removed; no host accounts or security settings changed. + +The fixed matrix validator now explicitly rejects failed child execution, missing or false execution/duplex indicators, nonempty injection/process errors or stderr, and incorrect payload request counts (cold/replacement one, warm zero). Evidence is written before failure, and host-owned exit/phase metadata cannot be overwritten by child JSON. Controlled validator-only subprocess tests under `python3 -O` verified both failure and baseline diagnostic behavior; they are not runtime-matrix evidence. The retained historical 30-cell result also passed the stricter gate. + ## T9 boundary audit and release coordination Read-only inventory at base core `d3f6e421642575c5072badcf56bf5b0a5606bd41`: diff --git a/docs/issue-1621/matrix.py b/docs/issue-1621/matrix.py index d714f69d7d..636b9f583f 100644 --- a/docs/issue-1621/matrix.py +++ b/docs/issue-1621/matrix.py @@ -8,6 +8,29 @@ import subprocess +def fixed_row_passes(row): + expected_requests = {'cold': 1, 'warm': 0, 'forced-replacement': 1}.get(row.get('phase')) + requests = row.get('localRequests') + return ( + expected_requests is not None + and row.get('harnessExit') == 0 + and row.get('executed') is True + and row.get('duplexExact') is True + and row.get('error') == '' + and row.get('execError') == '' + and row.get('stderr') == '' + and isinstance(requests, int) + and not isinstance(requests, bool) + and requests == expected_requests + ) + + +def require_fixed_rows(rows): + failures = [row for row in rows if not fixed_row_passes(row)] + if failures: + raise SystemExit('SSH matrix failed: ' + json.dumps(failures)) + + def main(): p = argparse.ArgumentParser() p.add_argument('--state', required=True, type=Path) @@ -46,15 +69,15 @@ def main(): '-payload', str(payload), '-input', str(input_file), '-check', 'false' if phase == 'warm' else 'true'] c = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=30) - row = {'shell': shell, 'builtin': builtin, 'phase': phase, - 'harnessExit': c.returncode, **json.loads(c.stdout)} + row = {**json.loads(c.stdout), 'shell': shell, 'builtin': builtin, + 'phase': phase, 'harnessExit': c.returncode} row['rawPrefix'] = row.pop('rawStdout')[:30] row.pop('stdout') rows.append(row) print(json.dumps(row), flush=True) a.output.write_text(json.dumps(rows, indent=2) + '\n') if not a.baseline: - assert all(r['duplexExact'] and not r['error'] and not r['stderr'] for r in rows) + require_fixed_rows(rows) if __name__ == '__main__': diff --git a/pkg/inject/dockerless_test.go b/pkg/inject/dockerless_test.go index d41b5fe1ba..d1c9ce00f6 100644 --- a/pkg/inject/dockerless_test.go +++ b/pkg/inject/dockerless_test.go @@ -71,11 +71,18 @@ func (s *BootstrapSuite) TestInstallerShellSelection() { installed, readErr := os.ReadFile(installPath) s.Require().NoError(readErr) s.Equal(payload, installed) + info, statErr := os.Stat(installPath) + s.Require().NoError(statErr) + if tc.selected == "" { + s.Equal(os.FileMode(0o750), info.Mode().Perm()) + } else { + s.Equal(os.FileMode(0o755), info.Mode().Perm()) + } for _, name := range []string{standardInterpreter, bundledInterpreter} { _, statErr := os.Stat(filepath.Join(root, name+"-used")) s.Equal(name == tc.selected, statErr == nil) } - _, statErr := os.Stat(filepath.Join(root, "substitution-ran")) + _, statErr = os.Stat(filepath.Join(root, "substitution-ran")) s.True(os.IsNotExist(statErr)) }) } diff --git a/pkg/inject/inject.sh b/pkg/inject/inject.sh index a7977c203a..f49b8c103a 100644 --- a/pkg/inject/inject.sh +++ b/pkg/inject/inject.sh @@ -93,7 +93,7 @@ prepare_temp() { publish_binary() { if [ "$CHMOD_PATH" = "true" ]; then - run_installer_script 'chmod -- +x "$1"' "$temp_file" || { + run_installer_script 'chmod -- 0755 "$1"' "$temp_file" || { >&2 echo "Error: Failed to make temporary agent executable" return 1 } diff --git a/pkg/inject/installer_modes_test.go b/pkg/inject/installer_modes_test.go index 78b29a3e49..5f9359ce02 100644 --- a/pkg/inject/installer_modes_test.go +++ b/pkg/inject/installer_modes_test.go @@ -99,6 +99,9 @@ func (s *BootstrapSuite) checkInstallerModeLiveStream(mode string) { installed, err := os.ReadFile(params.AgentRemotePath) s.Require().NoError(err) s.Equal(payload, installed) + info, err := os.Stat(params.AgentRemotePath) + s.Require().NoError(err) + s.Equal(os.FileMode(0o755), info.Mode().Perm()) assertInstallerModeUsed(s.T(), marker, mode) _, err = os.Stat(filepath.Join(root, "substitution-ran")) s.True(os.IsNotExist(err)) diff --git a/pkg/inject/script_test.go b/pkg/inject/script_test.go index 1a210fbc9d..f7bb93c5a9 100644 --- a/pkg/inject/script_test.go +++ b/pkg/inject/script_test.go @@ -135,6 +135,9 @@ func (s *BootstrapSuite) TestHostilePathAndLiveBinaryStream() { installed, err := os.ReadFile(installPath) s.Require().NoError(err) s.Equal(binary, installed) + info, err := os.Stat(installPath) + s.Require().NoError(err) + s.Equal(os.FileMode(0o755), info.Mode().Perm()) _, err = os.Stat(sentinel) s.True(os.IsNotExist(err)) } diff --git a/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx b/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx index 1080194911..c4d0c1ae83 100644 --- a/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx +++ b/sites/docs-devsy-sh/content/docs/troubleshooting/linux-troubleshooting.mdx @@ -19,6 +19,8 @@ Local folders suit early work. Once the devcontainer works, push it to a Git rep The SSH provider supports Bash, Zsh, and Fish login shells on Linux. Agent installation runs through the host's executable `/bin/sh`; interactive terminals and environment probing continue to use the account's configured shell. The workspace container's shell is configured separately. +Dockerless containers are a separate bootstrap case: before their image filesystem is built, core can use the fixed `/.dockerless/bin/sh` bundle when `DOCKERLESS=true`, `/bin/sh` is not executable, and that bundled shell is executable. `/bin/sh` remains preferred when available. This does not change the SSH host prerequisite above. + The host needs standard bootstrap tools, including `mkdir`, `mktemp`, `head` with `-c`, `wc`, `chmod`, `mv`, and `rm`. A download uses `curl` or `wget`; when neither is available, Devsy can transfer the agent over the SSH connection. The agent directory must permit execution and either be writable by the account or allow noninteractive privilege escalation. If startup reports `command not found: sh -c`, check the installed SSH provider version and its release notes for the POSIX bootstrap fix. Changing the account's shell is unnecessary. Both the external OpenSSH client and the built-in SSH transport use the same bootstrap launcher; switching transports does not repair a missing `/bin/sh` or a restricted account. From 83286eb7704afbd29185e4ba07620507648cb7f3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 20:38:12 -0600 Subject: [PATCH 10/11] test(inject): retain failed matrix cell evidence --- docs/issue-1621/README.md | 6 +++ docs/issue-1621/matrix.py | 46 ++++++++++++++++-- docs/issue-1621/test_matrix.py | 88 ++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 docs/issue-1621/test_matrix.py diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md index 560bb61307..bf3d0f4f9a 100644 --- a/docs/issue-1621/README.md +++ b/docs/issue-1621/README.md @@ -125,6 +125,12 @@ The synthetic ELF SHA-256 was `8f3d322e43d67c91d924de36054d2a66702720da0ea109242 The fixed matrix validator now explicitly rejects failed child execution, missing or false execution/duplex indicators, nonempty injection/process errors or stderr, and incorrect payload request counts (cold/replacement one, warm zero). Evidence is written before failure, and host-owned exit/phase metadata cannot be overwritten by child JSON. Controlled validator-only subprocess tests under `python3 -O` verified both failure and baseline diagnostic behavior; they are not runtime-matrix evidence. The retained historical 30-cell result also passed the stricter gate. +## Matrix child-failure evidence + +A later CodeRabbit review identified that a timed-out or malformed child result could stop the matrix before its evidence file was written. A controlled baseline reproduction confirmed that a successful row followed by `TimeoutExpired` escaped with no output file. The helper now records timeouts, failed child startup, invalid/non-object JSON, and missing or wrongly typed output fields as failed cells, retaining partial output and separate child-process stderr. The fixed gate rejects those cells after the complete evidence file is written; baseline runs remain diagnostic. Host-owned exit and diagnostic fields cannot be forged by child JSON. + +Run the persistent helper regressions with `python3 -O -m unittest discover -s docs/issue-1621 -p 'test_matrix.py'`. They simulate child failures in both baseline and fixed entry points, check that earlier and later rows survive, and execute a real local subprocess to verify nonzero exit and stderr override forged JSON metadata. They do not start SSH, containers, or the product and do not replace runtime qualification. Product-source hashes are unchanged by this helper-only follow-up. + ## T9 boundary audit and release coordination Read-only inventory at base core `d3f6e421642575c5072badcf56bf5b0a5606bd41`: diff --git a/docs/issue-1621/matrix.py b/docs/issue-1621/matrix.py index 636b9f583f..3e51b736f4 100644 --- a/docs/issue-1621/matrix.py +++ b/docs/issue-1621/matrix.py @@ -14,6 +14,8 @@ def fixed_row_passes(row): return ( expected_requests is not None and row.get('harnessExit') == 0 + and not row.get('harnessError') + and not row.get('harnessStderr') and row.get('executed') is True and row.get('duplexExact') is True and row.get('error') == '' @@ -31,6 +33,44 @@ def require_fixed_rows(rows): raise SystemExit('SSH matrix failed: ' + json.dumps(failures)) +def captured_text(value): + if isinstance(value, bytes): + return value.decode('utf-8', errors='replace') + return value or '' + + +def run_cell(cmd, env): + exit_code, output, diagnostics, failure = None, '', '', '' + try: + child = subprocess.run(cmd, env=env, capture_output=True, timeout=30) + exit_code = child.returncode + output, diagnostics = captured_text(child.stdout), captured_text(child.stderr) + except subprocess.TimeoutExpired as error: + output, diagnostics = captured_text(error.stdout), captured_text(error.stderr) + failure = f'harness timed out after {error.timeout} seconds' + except OSError as error: + failure = f'harness could not start: {error}' + + result = {} + try: + decoded = json.loads(output) + if not isinstance(decoded, dict): + raise ValueError('harness JSON is not an object') + result = decoded + if not all(isinstance(result.get(key), str) for key in ('rawStdout', 'stdout')): + raise ValueError('harness JSON requires string rawStdout and stdout fields') + except ValueError as error: + failure = '; '.join(filter(None, (failure, str(error)))) + + raw = result.pop('rawStdout', output) + result.pop('stdout', None) + result.update(harnessExit=exit_code, harnessError=failure, + harnessStderr=diagnostics, rawPrefix=(raw if isinstance(raw, str) else output)[:30]) + if failure: + result['harnessOutput'] = output + return result + + def main(): p = argparse.ArgumentParser() p.add_argument('--state', required=True, type=Path) @@ -68,11 +108,7 @@ def main(): '-builtin', builtin, '-dest', dest, '-command', shlex.quote(dest), '-payload', str(payload), '-input', str(input_file), '-check', 'false' if phase == 'warm' else 'true'] - c = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=30) - row = {**json.loads(c.stdout), 'shell': shell, 'builtin': builtin, - 'phase': phase, 'harnessExit': c.returncode} - row['rawPrefix'] = row.pop('rawStdout')[:30] - row.pop('stdout') + row = {**run_cell(cmd, env), 'shell': shell, 'builtin': builtin, 'phase': phase} rows.append(row) print(json.dumps(row), flush=True) a.output.write_text(json.dumps(rows, indent=2) + '\n') diff --git a/docs/issue-1621/test_matrix.py b/docs/issue-1621/test_matrix.py new file mode 100644 index 0000000000..bbcd2c0f15 --- /dev/null +++ b/docs/issue-1621/test_matrix.py @@ -0,0 +1,88 @@ +"""Exercise matrix evidence retention without starting SSH or container fixtures.""" +import contextlib +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import matrix + + +def successful_result(cmd): + result = dict(executed=True, duplexExact=True, error='', execError='', stderr='', + localRequests=0 if cmd[-1] == 'false' else 1, + rawStdout='ping\nARM-true\ndone\n', stdout='live bytes') + return subprocess.CompletedProcess(cmd, 0, json.dumps(result).encode(), b'') + + +class MatrixEvidenceTests(unittest.TestCase): + def test_child_failures_are_saved_for_fixed_and_baseline_runs(self): + failures = { + 'timeout': subprocess.TimeoutExpired([], 30, output=b'partial\xff', stderr=b'timeout diagnostic'), + 'empty': subprocess.CompletedProcess([], 17, b'', b'panic diagnostic'), + 'non-json': subprocess.CompletedProcess([], 0, b'not JSON', b'parse diagnostic'), + 'non-object': subprocess.CompletedProcess([], 0, b'[]', b''), + 'missing-output': subprocess.CompletedProcess([], 0, json.dumps(dict(executed=True, duplexExact=True, error='', execError='', stderr='', localRequests=1)).encode(), b''), + 'wrong-output-type': subprocess.CompletedProcess([], 0, b'{"rawStdout":7,"stdout":null}', b''), + 'not-started': FileNotFoundError('missing harness'), + } + for baseline in (False, True): + for name, failure in failures.items(): + with self.subTest(baseline=baseline, failure=name), tempfile.TemporaryDirectory() as root: + state = Path(root) + (state / 'fixture.json').write_text(json.dumps(dict(bin=root, port=22))) + output = state / 'rows.json' + argv = ['matrix.py', '--state', root, '--core', '/unused/core', + '--provider', '/unused/provider', '--output', str(output)] + if baseline: + argv.append('--baseline') + calls = [] + + def child(cmd, **_kwargs): + calls.append(cmd) + if len(calls) == 2: + if isinstance(failure, Exception): + raise failure + return failure + return successful_result(cmd) + + with patch.object(sys, 'argv', argv), patch.object(matrix.subprocess, 'run', side_effect=child), contextlib.redirect_stdout(io.StringIO()): + if baseline: + matrix.main() + else: + with self.assertRaises(SystemExit): + matrix.main() + rows = json.loads(output.read_text()) + self.assertEqual(len(rows), 6 if baseline else 30) + self.assertTrue(matrix.fixed_row_passes(rows[0])) + self.assertFalse(matrix.fixed_row_passes(rows[1])) + self.assertTrue(matrix.fixed_row_passes(rows[-1])) + self.assertTrue(rows[1]['harnessError']) + self.assertEqual(rows[1]['shell'], 'bash') + self.assertEqual(rows[1]['phase'], 'cold' if baseline else 'warm') + if name == 'timeout': + self.assertIsNone(rows[1]['harnessExit']) + self.assertEqual(rows[1]['rawPrefix'], 'partial\ufffd') + self.assertEqual(rows[1]['harnessStderr'], 'timeout diagnostic') + if name == 'empty': + self.assertEqual(rows[1]['harnessExit'], 17) + self.assertEqual(rows[1]['harnessStderr'], 'panic diagnostic') + + def test_real_child_exit_and_stderr_override_forged_json(self): + forged = dict(executed=True, duplexExact=True, error='', execError='', stderr='', + localRequests=1, rawStdout='ping', stdout='live', + harnessExit=0, harnessError='', harnessStderr='') + code = f'import sys; print({json.dumps(forged)!r}); sys.stderr.write("child diagnostic"); sys.exit(17)' + row = matrix.run_cell([sys.executable, '-c', code], None) + row['phase'] = 'cold' + self.assertEqual(row['harnessExit'], 17) + self.assertEqual(row['harnessStderr'], 'child diagnostic') + self.assertFalse(matrix.fixed_row_passes(row)) + + +if __name__ == '__main__': + unittest.main() From 535444860f9ab70fe9dbc2b344e96bcb032a9a58 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 20 Oct 2026 20:54:08 -0600 Subject: [PATCH 11/11] docs: remove issue-specific verification artifacts --- docs/issue-1621/README.md | 151 ----------------------- docs/issue-1621/inject-harness.go | 191 ------------------------------ docs/issue-1621/matrix.py | 120 ------------------- docs/issue-1621/real-tunnel.go | 163 ------------------------- docs/issue-1621/ssh-fixture.py | 71 ----------- docs/issue-1621/test_matrix.py | 88 -------------- 6 files changed, 784 deletions(-) delete mode 100644 docs/issue-1621/README.md delete mode 100644 docs/issue-1621/inject-harness.go delete mode 100644 docs/issue-1621/matrix.py delete mode 100644 docs/issue-1621/real-tunnel.go delete mode 100644 docs/issue-1621/ssh-fixture.py delete mode 100644 docs/issue-1621/test_matrix.py diff --git a/docs/issue-1621/README.md b/docs/issue-1621/README.md deleted file mode 100644 index bf3d0f4f9a..0000000000 --- a/docs/issue-1621/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# Issue #1621 verification and release coordination - -This evidence concerns the SSH provider plus core agent injection. It does not qualify every provider. No release, merge, or managed activation is authorized by this document. - -Fixture snapshots below retain their original artifact revisions and scope. Later provider PR #152 CI on `e9c4c75e81853bcb11cf707d4aff9a49bb739411` passed all 18 Linux/Bash workspace integration cases with released core v1.24.0, five-platform snapshot packaging, and native Windows stdin tests. This adds bounded whole-CLI and Windows-runner evidence; it does not establish Desktop acceptance, whole-workspace coverage across every login shell, or published artifacts. - -## Actual Linux fixture - -The 2026-10-20 run used disposable Ubuntu 26.04 aarch64 containers on a macOS host, OpenSSH 10.2p1, Bash 5.3.9, Zsh 5.9, Fish 4.2.1, Dash `/bin/sh`, and BusyBox 1.37.0. Five distinct non-root passwd entries used `/bin/{bash,zsh,fish,dash,ash}`. Password authentication, PAM, root login, and MOTD were disabled; public-key authentication used fresh temporary keys. Only an ephemeral localhost port was published. No host accounts, servers, shell configuration, or SSH keys were changed. - -Independent clones tested BusyBox `/bin/sh`, missing `/bin/sh`, and non-executable `/bin/sh`; the shared server retained Dash. Runtime results and credentials remained outside the repository. The fixture image actually used was the cached `mcr.microsoft.com/devcontainers/base@sha256:91eff046d5c12318fc98e59258982b87ba896c467427ed8b5dab5a02107a5db7`. The helper defaults to the cached GHCR base and never pulls images; OS packages are installed inside the disposable container. - -## Baseline and corrected transport - -Original core v1.24.0 (`1f47cd185bd134b45cdce12b947f6c601d76d329`) and original provider v1.1.44 (`e37a6c68a695d9d1d3806b2d42a755750816f60e`) were independently exported with `git archive` and built from immutable temporary sources. Working-tree implementation edits did not enter the baseline builds. - -| Actual passwd shell | Original external SSH | Original built-in SSH | -| --- | --- | --- | -| Bash | Success: `ping`, `ARM-true`, `done`, payload output | Same success | -| Zsh | Exit 127; `main:7: command not found: sh -c`; raw stdout only `ping\n` | Exit 127; raw `ping\n`; remote diagnostic lost | -| Fish | Exit 127; unsupported assignment syntax; no stdout | Exit 127; no stdout; diagnostic lost | - -For successful baseline injection the transferred synthetic executable printed `agent-started\ntunnel-after-handshake\n`. Baseline failure surfaced as `read inject signal: EOF` for Zsh and `read ping: EOF` for Fish. Baseline used original raw injection commands, without a fixture-side POSIX wrapper. - -The corrected matrix used the actual provider worker's executable, not a substitute SSH launcher, plus the real core `GenerateScript` and `Inject` APIs. A synthetic Python executable distinguished protocol/byte verification from real Devsy startup. - -- Five passwd shells × external/built-in SSH × cold/warm/forced replacement: **30/30 passed**. Each returned exactly 4,110 bytes: 4 KiB containing all byte values, `READY\n`, then `LIVE-OK\n`. The host sent the follow-on request only after receiving output. SHA-256: `7085d9188f2d7d361c78e10a5075b746a0004d17a740085b0363dc49c5c14b43`. -- Final `chmod -- +x` core rerun: **30/30 passed** with random 4 KiB including NUL, quotes, dollar, backtick, CR, newline and backslash; hostile literal destination paths in every cell. Exact output SHA-256: `4d77b6416f4dfdf3a243a3cc27c79dced201dc7b01462e104b1b0d53af6ae9a4`. -- Cold/replacement requested one payload; warm requested none. All successful normal cells had empty stderr and error, with raw `ping\nARM-true\ndone\n` or warm `ping\ndone\n` before payload bytes. -- Unchanged original v1.24.0 core with corrected provider: **6/6 passed**, Bash/Zsh/Fish × both transports, random 4 KiB and genuinely live follow-on. This was original core injection with a synthetic executable, not a rebuilt v1.24.0 full agent. -- Provider init: **10/10 passed**, five shells × both transports; writable per-user agent path and `/bin/true` Docker stand-in. This verifies init transport/output checks, not a real remote Docker daemon. - -## Actual Linux failures and recovery - -| Case | Observed result | -| --- | --- | -| BusyBox `/bin/sh` clone, both transports | Exact binary/live exchange passed | -| Missing `/bin/sh`, both transports | Exit 127, diagnostic preserved, no `ping`, no payload request | -| Denied `/bin/sh`, both transports | Exit 126, diagnostic preserved, no `ping`, no payload request | -| Remote stdout/stderr and exit 37, both transports | Exact `ordinary-out`, exact `remote-diagnostic`, exit 37 preserved | -| Sudo denied/restrictive `/root` destination, both transports | Error explicitly requires passwordless sudo; only `ping`; no payload request | -| Real sudo installer, native synthetic ELF, both transports | Root-owned unwritable destination installed successfully; normal non-root execution returned exact binary/live exchange | -| Download failure, then stdin, both transports | Three failed localhost downloads followed by successful exact binary/live exchange; test-local `sleep` shim bypassed retry waits | -| No curl/wget in invocation-local PATH, both transports | Clear diagnostic, then successful stdin binary/live exchange | -| Truncated frame plus no download tools, both transports | Exit 1; raw `ping\nARM-true\n`, no `done`; complete-receive and both-methods-failed diagnostics; pre-existing executable still prints `old-valid` | -| Noisy Zsh startup in isolated clone, both transports | Noise remains visible before command output; core built-in test rejects it with `unexpected start line "noisy-rc"`, executed=false, zero payload requests | - -An independent read-only reviewer ran the actual `quote_installer_argument`/`run_installer_script` functions with Linux util-linux 2.41.3 and BusyBox 1.37.0 `su -s /bin/sh -c ... root`. Both preserved empty strings, apostrophes, backslashes, Unicode, CR/newlines, substitutions, leading dash and trailing newlines as exact NUL-delimited arguments. Both preserved `abcdeLIVE\n` across `head -c 5; printf '|'; cat`, producing `abcde|LIVE\n`. GNU and BusyBox accepted `chmod -- +x`. - -## Real agent and remaining acceptance - -The actual development Linux aarch64 Devsy binary (217,260,362 bytes) was transferred and started as `internal ssh-server --stdio`. Initial Zsh external and Bash cold/warm/replacement tunnels completed `SSH-2.0-Go` and two separate command sessions, returning `first-command` and `live-followon`, without stderr. The real-agent startup matrix passed **10/10**: all five passwd shells × both transports, each with SSH handshake and two sequential commands. Six cells reused an installed executable; Dash/ash × both transports performed cold transfers. Earlier Bash/Zsh runs additionally exercised cold/reuse/forced replacement. Some initial cold development-binary transfers exceeded the original 30-second injection/40-second client bounds; those attempts are recorded harness-limit failures, superseded by the successful 150-second-bound startup matrix. The reusable tunnel harness allows 150 seconds for large development binaries. - -Whole CLI `--ide=none` workspace acceptance: **BLOCKED**. Automatic approval review rejected the proposed privileged Docker-daemon fixture because it broadens the host boundary for container workloads; that action was not delivered or run. A safer capability check in the existing fixture as non-root `fixture-zsh` returned `unshare: Operation not permitted` (exit 1). No Docker socket, Docker client/daemon, or rootlesskit was present. No privileged container, host socket mount, seccomp relaxation, or indirect workaround was attempted. The SSH-only fixture uses `/bin/true` during provider init and does not establish the whole CLI workspace flow. Desktop acceptance: **BLOCKED**; no configured Desktop session/workspace fixture was provided. Native macOS unit/static checks are separate from the actual Linux/OpenSSH results above. Original v1.24.0 full-agent startup is **NOT RUN**; original core injection compatibility is verified above. - -Earlier provider artifact (`final5`) SHA-256: `7e18087de9fbc9137a7a561c6fce200b7fe50d523fd99db08b6e8ce7b27015c7`. The complete random/hostile 30-cell synthetic matrix was rerun against that artifact and passed; exact output SHA-256: `475f5b02c861d431216e56c8f7ccd56e5fada5e0e25c1c7fce5172dcefd5f349`. Earlier transport matrix and real cold transfers used the provider worker's first corrected artifact SHA-256 `b7e79349ee68e8685f03ba98ea03427dcb8cd02a31bea54648a902c1dc322c22`; final5 adds stricter initialization and stdin/context shutdown. A final5 rerun also passed all ten real-agent startup/two-session cells using the installed actual Devsy binary. The staged provider diff digest supplied by its owner was `5d9b03da0b349f36b2f3b539e29a454dd1490afa63def9bec145119e167d51b4`; this is a working-diff identifier, not a release SHA. - -The post-review provider artifact SHA-256 is `39468e46550ddf4a9ea3a1033cae10ef02d0feabcce42ad7b89a7f88f40ba989`, with production-diff fingerprint `fde9ee3eceaba52ab03246f0d59151c7070c967168edbce6b63a04ec1ddef1a0` supplied by its owner. This artifact passed a fresh **30/30** random/hostile synthetic matrix, **10/10** real-agent warm startup/two-session matrix, **6/6** unchanged-v1.24.0-core compatibility matrix, and both actual-agent SIGTERM checks (external 0.46 s, built-in 0.41 s; no local children or remote agent remaining). Final reviewed init and exact stdout/stderr/exit-37 checks passed in all ten shell/transport cells (20 checks). The final random-matrix exact stdout SHA-256 is `b8f99178b99fa86953567a3672e32a7427d140ef372cf563c3d296ad8b495ca2`. The final warm fixture was restored from the same cached base and prepopulated through Docker copy with the identical agent artifact; these are warm-start results, not new cold-transfer claims. The post-review changes address platform-specific redirected stdin and input-copy errors. Windows tests were cross-compiled locally; native Windows execution remains **NOT RUN**. - -Native Linux execution of the actual `TestBootstrapSuite` and unchanged `TestInjectScript_RealShell*` tests passed with `/bin/sh` pointing at BusyBox. Cross-compilation only produced the test executable; these results came from executing it in the Linux clone. - -The first GitHub Ubuntu BusyBox step exposed a test-double portability issue: Ubuntu's BusyBox static standalone applets bypassed PATH-based `sleep` and `chmod` doubles. The original failures were reproduced in a disposable Linux ARM64 container using Ubuntu Noble `busybox-static` `1:1.36.1-6ubuntu3.1`, the same package version as CI. Test-only shell functions now inject the faults in the outer and nested installer shells; successful binary/live-stream tests still execute the unmodified generated script. The chmod fault must emit its marker and the installer error, while download fallback must follow three failed attempts. The corrected bootstrap suite and unchanged real-shell transfer tests passed natively with that package as `/bin/sh`. No production installer or CI assertion was relaxed; final-head GitHub AMD64 execution remains a separate gate. - -A later Ubuntu race run exposed an existing delivery-test observer race: stopping its pending `context.AfterFunc` could discard notification of a real attempt cancellation. The fixture now records the actual context cause after the runtime joins. The expected timeout causes, fresh-source retries, exact source close counts, bounded completion, and uncanceled outer-context assertions remain intact. Production delivery behavior is unchanged. - -The macOS hostile-path fixture previously pipelined tunnel bytes immediately after the binary, before `done`. Independent fragmented-write reproductions showed BSD `head` reading ahead into those premature bytes. Production `Inject` waits for `done` before bridging user stdin. The fixture now exercises that actual API and sequencing, retaining exact binary, URL, raw protocol, live NUL-containing payload, and substitution-sentinel assertions. The existing real-shell transfer test remains unchanged. - -Repeated Ubuntu 26.04 CLI snapshot builds exhausted runner disk quota while compiling or linking concurrent Linux architectures. The Linux build job now serializes those same two targets and allows 45 minutes for cold serial execution and follow-on steps. Read-only disk/cache diagnostics surround the snapshot. No target, test, action pin, permission, cache, or unrelated runner file is removed; macOS and Windows build concurrency is unchanged. Final-head GitHub execution remains the resource-fix gate. - -Working core base: `d3f6e421642575c5072badcf56bf5b0a5606bd41`. Final script/source SHA-256: - -| Input | SHA-256 | -| --- | --- | -| Original v1.24.0 `inject.sh` | `c2effffd6bbb9e42146bce22e2393f55f19b88d7b152f59d09c3c1e749bc2e0b` | -| Pre-Dockerless-fix `pkg/inject/inject.sh` | `b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3` | -| Pre-permissions-fix Dockerless-compatible `pkg/inject/inject.sh` | `61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb097090294` | -| Readable staged-agent `pkg/inject/inject.sh` | `1c2b70523285dcca664a61180e5e56fccd09af4ef08a005cdf9ff2d36e62dcdd` | -| `pkg/inject/script.go` | `752c82c5f3df1e60359658bc961c557b35995636dab88618ca8e791e21bf8e3e` | -| `pkg/agent/inject.go` | `3ea3bb44cbc37132bde6569b68e0425ea8c91d744c3a6ea94988d99d3e6c61e7` | -| Actual Linux agent executable | `5e5853e9e0653830858fe2438b46a7e8eb94a927bbbb4fda3142c2ddbe173bc4` | - -SIGTERM cancellation of the final5 provider with blocked stdin completed in under 0.5 seconds on both transports after observing the actual agent SSH banner. Local SSH children/connections and the remote Devsy stdio agent terminated. The built-in command reports context cancellation and exits 1; external SSH exits 255. An arbitrary remote `exec sleep 1621` survived the closed non-PTY channel; those disposable probes were explicitly removed. Cancellation guarantees local transport/stdin-pump shutdown and observed agent cleanup, not remote arbitrary-process reaping or SIGKILL cleanup. - -## Reproduction - -Require a cached supported Ubuntu image, Docker, Python 3, OpenSSH client and the repository Go toolchain. Use a fresh temporary state directory and fresh container name. Never put state/keys/output under tracked docs. - -```sh -python3 docs/issue-1621/ssh-fixture.py setup --state "$FIXTURE_STATE" --name "$FIXTURE_NAME" -python3 docs/issue-1621/ssh-fixture.py metadata --state "$FIXTURE_STATE" --name "$FIXTURE_NAME" -go build -o "$CORE_HARNESS" docs/issue-1621/inject-harness.go -go build -o "$TUNNEL_HARNESS" docs/issue-1621/real-tunnel.go -python3 docs/issue-1621/matrix.py --state "$FIXTURE_STATE" --core "$CORE_HARNESS" --provider "$PROVIDER_BINARY" --output "$RESULT_FILE" --hostile -PATH="$FIXTURE_STATE/bin:$PATH" "$TUNNEL_HARNESS" -provider "$PROVIDER_BINARY" -payload "$LINUX_AGENT" -port "$FIXTURE_PORT" -host fixture-zsh -builtin true -dest /home/fixture-zsh/real-agent -python3 docs/issue-1621/ssh-fixture.py teardown --state "$FIXTURE_STATE" --name "$FIXTURE_NAME" -``` - -`fixture.json` supplies the selected port and wrapper directory. The wrappers supply only the fixture SSH config, including the `ssh -G` lookup used by the built-in transport. Harnesses have `go:build ignore` and must be built explicitly. For baseline reproduction, export the original core into a fresh temporary directory, copy only `inject-harness.go` there, build it against that export's module, and run `matrix.py --baseline` with the original provider executable. Original-provider stderr contains its original timestamped logger. - -The `forced-replacement` synthetic case explicitly sets the installation condition true; it is not a version-parser test. `real-tunnel.go -check` accepts the actual generated version check when testing version selection. Low-level framing failures can be reproduced with `inject-harness -script-out`, then passing `pong\n100\nabc` and EOF to the provider `COMMAND` while its invocation-local PATH omits curl/wget. Do not modify host binaries or accounts. - -## Dockerless installer compatibility - -Core E2E on `16532a90792a78839335ffb066f95f1163776da4` exposed a separate container boundary. Kubernetes native staging failed first with a broken stdin stream/WebSocket EOF. Its cleanup call supplied no streams, which the Kubernetes API rejects. Those driver/delivery implementations are unchanged from the baseline; the initiating stream failure remains unproven. The subsequent legacy fallback exposed a confirmed regression: the Dockerless bootstrap filesystem has `/.dockerless/bin/sh` but no `/bin/sh`. - -Ordinary disposable execution of `ghcr.io/devsy-org/dockerless@sha256:8910f52719e38ffe986652efa04796d781575797627cdba3dd1ba8f977078303` confirmed both image variants use a static BusyBox shell at that bundled path. The ARM64 manifest is `sha256:1c1e4242bfe3662fe6531f52833b185c1cefc6e0ff01aa4279df3edd3a359663`; AMD64 is `sha256:58621764ce6d3921b68b05b7fb9fdca7250fc17f906e919961ba8725d87c7fd4`. AMD64 executed through existing emulation on the ARM64 host, rather than native AMD64 hardware. - -The immutable pre-fix bootstrap (`b07ab67954708ee0bffac3b76a498374833a0383225bbf8acedf9e71c46d44c3`) failed after `ping`, without requesting a payload. Original v1.24.0 injection succeeded in the same ARM64 image with a synthetic ELF payload and live follow-on. The corrected bootstrap selects the fixed bundled interpreter only when `DOCKERLESS=true`, `/bin/sh` is not executable, and the bundled interpreter is executable. Ordinary targets retain `/bin/sh`, which is preferred even in Dockerless containers when available. Missing prerequisites fail before permission probes or escalation; warm reuse does not require an installer. - -The corrected script (`61d88ef1fd4394f35026d2c57cf3a3e29562385e2e8799a35a7f2fb097090294`) passed six ordinary-image cold/warm/forced synthetic cells across both architectures, including hostile literal paths and exact 4 KiB plus live follow-on bytes. Four negative cells (disabled Dockerless or both interpreters missing, each architecture) failed explicitly without requesting payloads. The source-exact native staging command and cleanup separately passed on both architectures. These process tests do not reproduce Dockerless `start --wait`, its filesystem transition, or Kubernetes exec/WebSocket transport, and do not establish the initiating native failure's cause. No builder, privileged container, host mount, or security override was used. - -The actual bundled BusyBox `su` wrapper also preserved empty and hostile literal arguments byte-for-byte and live stdin on both architectures, using the existing root account from a root process. This verifies shell/argument compatibility, not non-root privilege escalation. Both diagnostic containers were removed after validation. - -Persistent `TestPrivilegedInstallerModesLiveStream` and `TestPrivilegedInstallerFailureStatus` exercise the real dispatcher in sudo/su mode using strict test doubles. The doubles validate invocation flags and pass the requested code unchanged to a real POSIX shell; markers stay off protocol stdout. Actual `Inject` sequencing checks hostile paths/URLs, exact binary and live NUL-containing bytes, one `done`, and failure status 37. These deterministic tests require no host privilege operation and complement, rather than replace, the real-tool evidence above. - -Follow-ups remain precise: investigate Kubernetes staging diagnostics/stream closure in the Dockerless filesystem transition, and provide an explicit diagnostic stream for staged-file cleanup rather than submitting an all-false exec stream request. This change preserves native argv routing and configured interactive shells. SSH provider bootstrapping still requires `/bin/sh` before the core script executes. - -## Elevated installation permissions - -CodeRabbit identified a staging regression: `mktemp` creates mode `0600`, so adding only executable bits published a root-owned `0711` agent after sudo installation. A non-root user could execute that file but could not reopen its own executable. Publication now sets the staged agent to `0755` before the atomic rename. The chmod-failure path still retains the prior agent and removes only staging files. Persistent direct, sudo/su-dispatcher, and Dockerless-selection tests assert the final mode; failure cases retain the original mode. - -An ordinary disposable Linux ARM64 container reproduced the mechanism with the actual `GenerateScript` and `Inject` APIs, a non-root uid 1001, and real noninteractive sudo into an unwritable root-owned destination. The immutable `61d88ef1...` script published root-owned `0711`; the native synthetic ELF started but failed `os.Open(os.Executable())` with permission denied and exit 42. With script `1c2b7052...`, the identical 2,403,587-byte ELF was root-owned `0755`, reopened and read itself, and completed an exact 4,124-byte binary/live exchange with empty stderr and no error. Both runs requested one payload. The rendered scripts differed only in `chmod -- +x` versus `chmod -- 0755`. - -The synthetic ELF SHA-256 was `8f3d322e43d67c91d924de36054d2a66702720da0ea109242c89595b85af77d3`; successful stdout SHA-256 was `445e35c83283238eb1b2ef4bfcdc4e6a53f9600c7f99136a662e8304d6298a12`. This is an actual permissions and streaming reproduction, not a new full Devsy-agent or whole-product qualification. Ubuntu uutils 0.10.0, GNU coreutils 9.4, and BusyBox 1.38 accepted the fixed chmod syntax. The container, its internal test user and sudo policy were removed; no host accounts or security settings changed. - -The fixed matrix validator now explicitly rejects failed child execution, missing or false execution/duplex indicators, nonempty injection/process errors or stderr, and incorrect payload request counts (cold/replacement one, warm zero). Evidence is written before failure, and host-owned exit/phase metadata cannot be overwritten by child JSON. Controlled validator-only subprocess tests under `python3 -O` verified both failure and baseline diagnostic behavior; they are not runtime-matrix evidence. The retained historical 30-cell result also passed the stricter gate. - -## Matrix child-failure evidence - -A later CodeRabbit review identified that a timed-out or malformed child result could stop the matrix before its evidence file was written. A controlled baseline reproduction confirmed that a successful row followed by `TimeoutExpired` escaped with no output file. The helper now records timeouts, failed child startup, invalid/non-object JSON, and missing or wrongly typed output fields as failed cells, retaining partial output and separate child-process stderr. The fixed gate rejects those cells after the complete evidence file is written; baseline runs remain diagnostic. Host-owned exit and diagnostic fields cannot be forged by child JSON. - -Run the persistent helper regressions with `python3 -O -m unittest discover -s docs/issue-1621 -p 'test_matrix.py'`. They simulate child failures in both baseline and fixed entry points, check that earlier and later rows survive, and execute a real local subprocess to verify nonzero exit and stderr override forged JSON metadata. They do not start SSH, containers, or the product and do not replace runtime qualification. Product-source hashes are unchanged by this helper-only follow-up. - -## T9 boundary audit and release coordination - -Read-only inventory at base core `d3f6e421642575c5072badcf56bf5b0a5606bd41`: - -| Boundary | Existing contract / required disposition | -| --- | --- | -| `pkg/devcontainer/config/userenvprobe.go` | Keep configured `GetShell` and `-lic`/`-lc`/`-ic` semantics. Probe intentionally evaluates the user's environment. | -| `pkg/ssh/server/ssh_container.go:getCommand`, `pkg/shell/shell.go:GetShell` | Keep passwd/configured shell for terminal commands and PTY login `-l`; do not force user terminals to POSIX shell. | -| `pkg/agent/delivery/factory.go` | Local Docker uses named-volume delivery; remote Docker uses native `docker cp`/`mkdir`/`chmod` argv. Available `ArgvExec` uses native driver calls; Apple and fallback custom drivers retain `LegacyShellDelivery`. | -| `pkg/agent/delivery/kubernetes.go` | Driver receives explicit argv, including deliberately owned `sh -c` scripts for staging/download/commit; do not wrap argv again as a remote login command. | -| `pkg/devcontainer/setup.go:execSetupSSHServer` | Native `ArgvExecDriver` starts the agent with literal argv; fallback `CommandDevContainer` string path carries raw protocol stdout. | -| Docker `CommandDevContainer` | Native CLI argv already selects `sh -c`; no remote login-shell wrapper needed. | -| Kubernetes `CommandDevContainer` | Root uses `sh -c`; non-root uses `su USER -c STRING`. Follow up separately on that user-shell contract if broadening scope. | -| Apple `CommandDevContainer` | Native container CLI argv explicitly selects `sh -c`; Apple delivery remains shell-based and requires its own actual macOS qualification. | -| Custom `CommandDevContainer` | Passes `DEVCONTAINER_COMMAND`/`DEVCONTAINER_USER` to configured provider command. Provider implementation owns final shell evaluation; audit each provider before making broader claims. | -| External runtime / microsandbox argv | Preserve raw literal argv and streaming; no login-shell wrapper should be added. | - -Core and SSH-provider changes require independently versioned releases. Releasing only core does not repair Fish parsing in the old provider; corrected provider with old core passes this bounded compatibility matrix. Keep one canonical issue, cross-link both PRs and record exact released artifacts before claiming a shipped fix. Rollback must remain independent: revert/pin the provider separately from core, preserve a previously valid agent on failed replacement, and report which transport/version was restored. Release order, published-artifact qualification, full CLI/Desktop acceptance and merge remain pending owner decisions. diff --git a/docs/issue-1621/inject-harness.go b/docs/issue-1621/inject-harness.go deleted file mode 100644 index 8440970b34..0000000000 --- a/docs/issue-1621/inject-harness.go +++ /dev/null @@ -1,191 +0,0 @@ -//go:build ignore - -package main - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "flag" - "fmt" - "io" - "os" - "os/exec" - "sync" - "time" - - "github.com/devsy-org/devsy/pkg/inject" -) - -type harnessConfig struct { - provider, host, port, builtin, dest, check, command string - payload, input, download, scriptOut, prefix string - prefer bool - timeout time.Duration -} - -type liveOutput struct { - out bytes.Buffer - once sync.Once - ready chan struct{} - count int -} - -func (w *liveOutput) Write(p []byte) (int, error) { - n, err := w.out.Write(p) - if w.out.Len() >= w.count+6 { - w.once.Do(func() { close(w.ready) }) - } - return n, err -} - -type injectionRun struct { - config harnessConfig - output *liveOutput - stderr, raw bytes.Buffer - stdin io.Reader - expected []byte - requests int - execError string -} - -func main() { - config := parseHarnessFlags() - if config.scriptOut != "" { - writeGeneratedScript(config) - return - } - ctx, cancel := context.WithTimeout(context.Background(), config.timeout) - defer cancel() - run := &injectionRun{config: config, output: &liveOutput{ready: make(chan struct{})}} - cleanup := run.prepareInput(ctx) - defer cleanup() - ran, err := inject.Inject(ctx, inject.InjectOptions{ - Exec: run.execute, LocalFile: run.openPayload, ScriptParams: config.params(), - Stdin: run.stdin, Stdout: run.output, Stderr: &run.stderr, Timeout: 5 * time.Second, - }) - run.report(ran, err) -} - -func parseHarnessFlags() harnessConfig { - var c harnessConfig - flag.StringVar(&c.provider, "provider", "", "provider executable") - flag.StringVar(&c.host, "host", "fixture-bash", "fixture host alias") - flag.StringVar(&c.port, "port", "22", "SSH port") - flag.StringVar(&c.builtin, "builtin", "false", "builtin SSH") - flag.StringVar(&c.dest, "dest", "/tmp/verify1621/agent", "remote destination") - flag.StringVar(&c.check, "check", "true", "installation condition") - flag.StringVar(&c.command, "command", "/tmp/verify1621/agent", "post-bootstrap command") - flag.StringVar(&c.payload, "payload", "", "binary source") - flag.StringVar(&c.input, "input", "", "4096-byte binary duplex test input") - flag.StringVar(&c.download, "download", "", "download URL") - flag.StringVar(&c.scriptOut, "script-out", "", "write generated script and exit") - flag.StringVar(&c.prefix, "prefix", "", "POSIX prefix for generated script") - flag.BoolVar(&c.prefer, "prefer-download", false, "prefer download before stdin") - flag.DurationVar(&c.timeout, "timeout", 12*time.Second, "overall bound") - flag.Parse() - return c -} - -func (c harnessConfig) params() *inject.Params { - return &inject.Params{ - Command: c.command, AgentRemotePath: c.dest, - DownloadURLs: &inject.DownloadURLs{Amd: c.download, Arm: c.download}, - ExistsCheck: c.check, ShouldChmodPath: true, PreferAgentDownload: c.prefer, - } -} - -func writeGeneratedScript(c harnessConfig) { - script, err := inject.GenerateScript(inject.Script, c.params()) - if err != nil { - panic(err) - } - // #nosec G703 -- The operator explicitly selects the private fixture output file. - if err = os.WriteFile(c.scriptOut, []byte(script), 0o600); err != nil { - panic(err) - } -} - -func (r *injectionRun) prepareInput(ctx context.Context) func() { - r.stdin = bytes.NewReader([]byte("tunnel-after-handshake\n")) - if r.config.input == "" { - return func() {} - } - // #nosec G304 G703 -- The operator explicitly selects the fixture byte input file. - data, err := os.ReadFile(r.config.input) - if err != nil { - panic(err) - } - r.output.count = len(data) - r.expected = append( - append(append([]byte{}, data...), []byte("READY\n")...), - []byte("LIVE-OK\n")...) - reader, writer := io.Pipe() - r.stdin = reader - go sendLiveInput(ctx, writer, data, r.output.ready) - return func() { _ = reader.Close() } -} - -func sendLiveInput(ctx context.Context, writer *io.PipeWriter, data []byte, ready <-chan struct{}) { - defer func() { _ = writer.Close() }() - if _, err := writer.Write(data); err != nil { - return - } - select { - case <-ready: - _, _ = writer.Write([]byte("LIVE-FOLLOWON\n")) - case <-ctx.Done(): - } -} - -//nolint:revive // The callback must match inject.ExecFunc's five-argument signature. -func (r *injectionRun) execute( - ctx context.Context, - script string, - stdin io.Reader, - stdout, stderr io.Writer, -) error { - // #nosec G204 -- This harness deliberately executes the operator-selected provider artifact. - cmd := exec.CommandContext(ctx, r.config.provider, "command") - cmd.Env = append( - os.Environ(), - "COMMAND="+r.config.prefix+script, - "HOST="+r.config.host, - "PORT="+r.config.port, - "USE_BUILTIN_SSH="+r.config.builtin, - "DOCKER_PATH=/bin/true", - "AGENT_PATH="+r.config.dest, - ) - cmd.Stdin = stdin - cmd.Stdout = io.MultiWriter(stdout, &r.raw) - cmd.Stderr = stderr - err := cmd.Run() - if err != nil { - r.execError = err.Error() - } - return err -} - -func (r *injectionRun) openPayload(_ bool) (io.ReadCloser, error) { - r.requests++ - // #nosec G304 G703 -- The operator explicitly selects the agent artifact being tested. - return os.Open(r.config.payload) -} - -func (r *injectionRun) report(ran bool, err error) { - message := "" - if err != nil { - message = err.Error() - } - result := map[string]any{ - "executed": ran, "error": message, "execError": r.execError, - "stdout": r.output.out.String(), "stdoutBytes": r.output.out.Len(), - "stdoutSHA256": fmt.Sprintf("%x", sha256.Sum256(r.output.out.Bytes())), - "duplexExact": r.expected != nil && bytes.Equal(r.expected, r.output.out.Bytes()), - "localRequests": r.requests, "stderr": r.stderr.String(), "rawStdout": r.raw.String(), - } - if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { - panic(err) - } -} diff --git a/docs/issue-1621/matrix.py b/docs/issue-1621/matrix.py deleted file mode 100644 index 3e51b736f4..0000000000 --- a/docs/issue-1621/matrix.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -"""Run generated core injection through the actual SSH-provider executable.""" -import argparse -import json -import os -from pathlib import Path -import shlex -import subprocess - - -def fixed_row_passes(row): - expected_requests = {'cold': 1, 'warm': 0, 'forced-replacement': 1}.get(row.get('phase')) - requests = row.get('localRequests') - return ( - expected_requests is not None - and row.get('harnessExit') == 0 - and not row.get('harnessError') - and not row.get('harnessStderr') - and row.get('executed') is True - and row.get('duplexExact') is True - and row.get('error') == '' - and row.get('execError') == '' - and row.get('stderr') == '' - and isinstance(requests, int) - and not isinstance(requests, bool) - and requests == expected_requests - ) - - -def require_fixed_rows(rows): - failures = [row for row in rows if not fixed_row_passes(row)] - if failures: - raise SystemExit('SSH matrix failed: ' + json.dumps(failures)) - - -def captured_text(value): - if isinstance(value, bytes): - return value.decode('utf-8', errors='replace') - return value or '' - - -def run_cell(cmd, env): - exit_code, output, diagnostics, failure = None, '', '', '' - try: - child = subprocess.run(cmd, env=env, capture_output=True, timeout=30) - exit_code = child.returncode - output, diagnostics = captured_text(child.stdout), captured_text(child.stderr) - except subprocess.TimeoutExpired as error: - output, diagnostics = captured_text(error.stdout), captured_text(error.stderr) - failure = f'harness timed out after {error.timeout} seconds' - except OSError as error: - failure = f'harness could not start: {error}' - - result = {} - try: - decoded = json.loads(output) - if not isinstance(decoded, dict): - raise ValueError('harness JSON is not an object') - result = decoded - if not all(isinstance(result.get(key), str) for key in ('rawStdout', 'stdout')): - raise ValueError('harness JSON requires string rawStdout and stdout fields') - except ValueError as error: - failure = '; '.join(filter(None, (failure, str(error)))) - - raw = result.pop('rawStdout', output) - result.pop('stdout', None) - result.update(harnessExit=exit_code, harnessError=failure, - harnessStderr=diagnostics, rawPrefix=(raw if isinstance(raw, str) else output)[:30]) - if failure: - result['harnessOutput'] = output - return result - - -def main(): - p = argparse.ArgumentParser() - p.add_argument('--state', required=True, type=Path) - p.add_argument('--core', required=True, type=Path) - p.add_argument('--provider', required=True, type=Path) - p.add_argument('--output', required=True, type=Path) - p.add_argument('--baseline', action='store_true') - p.add_argument('--hostile', action='store_true') - a = p.parse_args() - fixture = json.loads((a.state / 'fixture.json').read_text()) - env = dict(os.environ, PATH=fixture['bin'] + ':' + os.environ['PATH']) - payload = a.state / 'duplex-agent' - payload.write_text('''#!/usr/bin/python3 -import sys -r = sys.stdin.buffer -w = sys.stdout.buffer -w.write(r.read(4096) + b"READY\\n") -w.flush() -if r.readline() == b"LIVE-FOLLOWON\\n": - w.write(b"LIVE-OK\\n") - w.flush() -''') - input_file = a.state / 'input.bin' - data = bytearray(os.urandom(4096)) - data[:8] = b'\x00\'"$`\n\r\\' - input_file.write_bytes(data) - rows = [] - for shell in (['bash', 'zsh', 'fish'] if a.baseline else ['bash', 'zsh', 'fish', 'dash', 'ash']): - for builtin in ['false', 'true']: - suffix = f'/literal space\'"$`(quote)\ncarriage\rsemi;{builtin}' if a.hostile else f'/matrix-{builtin}' - dest = f'/home/fixture-{shell}' + suffix + '/agent' - for phase in (['cold'] if a.baseline else ['cold', 'warm', 'forced-replacement']): - cmd = [str(a.core.resolve()), '-provider', str(a.provider.resolve()), - '-host', 'fixture-' + shell, '-port', str(fixture['port']), - '-builtin', builtin, '-dest', dest, '-command', shlex.quote(dest), - '-payload', str(payload), '-input', str(input_file), - '-check', 'false' if phase == 'warm' else 'true'] - row = {**run_cell(cmd, env), 'shell': shell, 'builtin': builtin, 'phase': phase} - rows.append(row) - print(json.dumps(row), flush=True) - a.output.write_text(json.dumps(rows, indent=2) + '\n') - if not a.baseline: - require_fixed_rows(rows) - - -if __name__ == '__main__': - main() diff --git a/docs/issue-1621/real-tunnel.go b/docs/issue-1621/real-tunnel.go deleted file mode 100644 index 1986de8f3a..0000000000 --- a/docs/issue-1621/real-tunnel.go +++ /dev/null @@ -1,163 +0,0 @@ -//go:build ignore - -package main - -import ( - "bytes" - "context" - "encoding/json" - "flag" - "fmt" - "io" - "maps" - "net" - "os" - "os/exec" - "time" - - "github.com/devsy-org/devsy/pkg/inject" - "golang.org/x/crypto/ssh" -) - -type tunnelConfig struct { - provider, host, port, builtin, dest, payload, check string -} - -type tunnelRun struct { - config tunnelConfig - bridge net.Conn - stderr bytes.Buffer - requests int - completed chan map[string]any -} - -func main() { - result := runTunnel(parseTunnelFlags()) - if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { - panic(err) - } -} - -func parseTunnelFlags() tunnelConfig { - var c tunnelConfig - flag.StringVar(&c.provider, "provider", "", "provider executable") - flag.StringVar(&c.host, "host", "fixture-zsh", "fixture host") - flag.StringVar(&c.port, "port", "22", "SSH port") - flag.StringVar(&c.builtin, "builtin", "false", "builtin transport") - flag.StringVar(&c.dest, "dest", "/home/fixture-zsh/real-agent", "agent destination") - flag.StringVar(&c.payload, "payload", "", "actual Linux agent") - flag.StringVar(&c.check, "check", "true", "replacement condition") - flag.Parse() - return c -} - -func runTunnel(config tunnelConfig) map[string]any { - ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) - defer cancel() - client, bridge := net.Pipe() - defer func() { _ = client.Close(); _ = bridge.Close() }() - run := &tunnelRun{config: config, bridge: bridge, completed: make(chan map[string]any, 1)} - go func() { run.completed <- run.inject(ctx) }() - result := map[string]any{"host": config.host, "builtin": config.builtin} - err := run.connect(client, result) - result["tunnelError"] = "" - if err != nil { - result["tunnelError"] = err.Error() - } - cancel() - _ = bridge.Close() - _ = client.Close() - select { - case injection := <-run.completed: - maps.Copy(result, injection) - result["stderr"] = run.stderr.String() - case <-time.After(8 * time.Second): - result["injectWaitTimeout"] = true - } - return result -} - -func (r *tunnelRun) inject(ctx context.Context) map[string]any { - quoted, err := inject.QuotePOSIXLiteral(r.config.dest) - if err != nil { - return map[string]any{"injectError": err.Error()} - } - ran, err := inject.Inject(ctx, inject.InjectOptions{ - Exec: r.execute, LocalFile: r.openPayload, - ScriptParams: &inject.Params{ - AgentRemotePath: r.config.dest, DownloadURLs: &inject.DownloadURLs{}, - ExistsCheck: r.config.check, ShouldChmodPath: true, - Command: "exec " + quoted + " internal ssh-server --stdio --log-level=error", - }, - Stdin: r.bridge, Stdout: r.bridge, Stderr: &r.stderr, Timeout: 150 * time.Second, - }) - message := "" - if err != nil { - message = err.Error() - } - return map[string]any{"executed": ran, "injectError": message, "localRequests": r.requests} -} - -//nolint:revive // The callback must match inject.ExecFunc's five-argument signature. -func (r *tunnelRun) execute( - ctx context.Context, - script string, - stdin io.Reader, - stdout, stderr io.Writer, -) error { - // #nosec G204 -- This harness deliberately executes the operator-selected provider artifact. - cmd := exec.CommandContext(ctx, r.config.provider, "command") - cmd.Env = append( - os.Environ(), - "HOST="+r.config.host, - "PORT="+r.config.port, - "USE_BUILTIN_SSH="+r.config.builtin, - "COMMAND="+script, - "DOCKER_PATH=/bin/true", - "AGENT_PATH="+r.config.dest, - ) - cmd.Stdin = stdin - cmd.Stdout = stdout - cmd.Stderr = stderr - return cmd.Run() -} - -func (r *tunnelRun) openPayload(_ bool) (io.ReadCloser, error) { - r.requests++ - // #nosec G304 G703 -- The operator explicitly selects the Linux agent artifact being tested. - return os.Open(r.config.payload) -} - -func (r *tunnelRun) connect(client net.Conn, result map[string]any) error { - if err := client.SetDeadline(time.Now().Add(150 * time.Second)); err != nil { - return err - } - conn, channels, requests, err := ssh.NewClientConn(client, "fixture-agent", &ssh.ClientConfig{ - User: r.config.host, - // #nosec G106 -- The SSH server is the ephemeral agent just injected into the isolated fixture. - HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: 150 * time.Second, - }) - if err != nil { - return err - } - sshClient := ssh.NewClient(conn, channels, requests) - defer func() { _ = sshClient.Close() }() - result["serverVersion"] = string(conn.ServerVersion()) - return runTunnelCommands(sshClient, result) -} - -func runTunnelCommands(client *ssh.Client, result map[string]any) error { - for index, command := range []string{"printf first-command", "printf live-followon"} { - session, err := client.NewSession() - if err != nil { - return err - } - out, err := session.Output(command) - _ = session.Close() - result[fmt.Sprintf("command%d", index+1)] = string(out) - if err != nil { - return err - } - } - return nil -} diff --git a/docs/issue-1621/ssh-fixture.py b/docs/issue-1621/ssh-fixture.py deleted file mode 100644 index f309c45f3c..0000000000 --- a/docs/issue-1621/ssh-fixture.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Disposable Linux SSH fixture; credentials remain in the private state directory.""" -import argparse -import json -import os -from pathlib import Path -import shlex -import subprocess - - -def run(*args, **kw): - return subprocess.run(args, check=True, text=True, **kw) - - -def main(): - p = argparse.ArgumentParser() - p.add_argument('action', choices=['setup', 'metadata', 'teardown']) - p.add_argument('--state', required=True, type=Path) - p.add_argument('--name', default='devsy-1621-sshd') - p.add_argument('--image', default='ghcr.io/devsy-org/test-images/base@sha256:4bcb1b466771b1ba1ea110e2a27daea2f6093f9527fb75ee59703ec89b5561cb') - p.add_argument('--reuse', action='store_true') - a = p.parse_args() - if a.action == 'teardown': - run('docker', 'rm', '-f', a.name) - print('Container removed; delete private state after workers finish.') - return - if a.action == 'metadata': - run('docker', 'exec', a.name, 'sh', '-c', "uname -sm; /usr/sbin/sshd -V 2>&1; bash --version | head -1; zsh --version; fish --version; busybox | head -1; readlink -f /bin/sh; getent passwd fixture-bash fixture-zsh fixture-fish fixture-dash fixture-ash; /usr/sbin/sshd -T -f /tmp/fixture-sshd.conf | grep -E '^(passwordauthentication|pubkeyauthentication|permitrootlogin|usepam|printmotd|authorizedkeysfile)'") - return - a.state.mkdir(parents=True, exist_ok=True) - os.chmod(a.state, 0o700) - key = a.state / 'id_ed25519' - if not key.exists(): - run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(key)) - if not a.reuse: - run('docker', 'run', '-d', '--pull=never', '--name', a.name, '-p', '127.0.0.1::22', a.image, 'sleep', 'infinity') - run('docker', 'exec', a.name, 'sh', '-c', 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openssh-server bash zsh fish dash busybox python3 sudo') - run('docker', 'cp', str(key) + '.pub', a.name + ':/tmp/fixture.pub') - run('docker', 'exec', a.name, 'sh', '-ec', ''' -mkdir -p /run/sshd -ln -sf /bin/busybox /bin/ash -for shell in bash zsh fish dash ash; do - name=fixture-$shell - id "$name" >/dev/null 2>&1 || useradd -m -s /bin/$shell "$name" - usermod -p '*' "$name" - mkdir -p /home/$name/.ssh - cp /tmp/fixture.pub /home/$name/.ssh/authorized_keys - chown -R $name:$name /home/$name/.ssh - chmod 700 /home/$name/.ssh - chmod 600 /home/$name/.ssh/authorized_keys -done -printf '%s\\n' 'PasswordAuthentication no' 'KbdInteractiveAuthentication no' 'PermitRootLogin no' 'UsePAM no' 'PrintMotd no' 'AuthorizedKeysFile .ssh/authorized_keys' 'PidFile /run/fixture-sshd.pid' 'Subsystem sftp internal-sftp' > /tmp/fixture-sshd.conf -ssh-keygen -A -/usr/sbin/sshd -f /tmp/fixture-sshd.conf -E /tmp/fixture-sshd.log -''') - port = run('docker', 'port', a.name, '22/tcp', capture_output=True).stdout.strip().split(':')[-1] - config = a.state / 'ssh_config' - shells = ['bash', 'zsh', 'fish', 'dash', 'ash'] - config.write_text('Host fixture-*\n HostName 127.0.0.1\n Port ' + port + '\n IdentityFile ' + str(key) + '\n IdentitiesOnly yes\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no\n BatchMode yes\n LogLevel ERROR\n ConnectTimeout 5\n' + ''.join('Host fixture-' + s + '\n User fixture-' + s + '\n' for s in shells)) - bindir = a.state / 'bin' - bindir.mkdir(exist_ok=True) - for command in ['ssh', 'scp']: - wrapper = bindir / command - wrapper.write_text('#!/bin/sh\nexec /usr/bin/' + command + ' -F ' + shlex.quote(str(config)) + ' "$@"\n') - wrapper.chmod(0o700) - (a.state / 'fixture.json').write_text(json.dumps({'container': a.name, 'port': int(port), 'config': str(config), 'bin': str(bindir)}, indent=2) + '\n') - print(json.dumps({'port': int(port), 'state': str(a.state), 'hosts': ['fixture-' + s for s in shells]})) - - -if __name__ == '__main__': - main() diff --git a/docs/issue-1621/test_matrix.py b/docs/issue-1621/test_matrix.py deleted file mode 100644 index bbcd2c0f15..0000000000 --- a/docs/issue-1621/test_matrix.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Exercise matrix evidence retention without starting SSH or container fixtures.""" -import contextlib -import io -import json -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest -from unittest.mock import patch - -import matrix - - -def successful_result(cmd): - result = dict(executed=True, duplexExact=True, error='', execError='', stderr='', - localRequests=0 if cmd[-1] == 'false' else 1, - rawStdout='ping\nARM-true\ndone\n', stdout='live bytes') - return subprocess.CompletedProcess(cmd, 0, json.dumps(result).encode(), b'') - - -class MatrixEvidenceTests(unittest.TestCase): - def test_child_failures_are_saved_for_fixed_and_baseline_runs(self): - failures = { - 'timeout': subprocess.TimeoutExpired([], 30, output=b'partial\xff', stderr=b'timeout diagnostic'), - 'empty': subprocess.CompletedProcess([], 17, b'', b'panic diagnostic'), - 'non-json': subprocess.CompletedProcess([], 0, b'not JSON', b'parse diagnostic'), - 'non-object': subprocess.CompletedProcess([], 0, b'[]', b''), - 'missing-output': subprocess.CompletedProcess([], 0, json.dumps(dict(executed=True, duplexExact=True, error='', execError='', stderr='', localRequests=1)).encode(), b''), - 'wrong-output-type': subprocess.CompletedProcess([], 0, b'{"rawStdout":7,"stdout":null}', b''), - 'not-started': FileNotFoundError('missing harness'), - } - for baseline in (False, True): - for name, failure in failures.items(): - with self.subTest(baseline=baseline, failure=name), tempfile.TemporaryDirectory() as root: - state = Path(root) - (state / 'fixture.json').write_text(json.dumps(dict(bin=root, port=22))) - output = state / 'rows.json' - argv = ['matrix.py', '--state', root, '--core', '/unused/core', - '--provider', '/unused/provider', '--output', str(output)] - if baseline: - argv.append('--baseline') - calls = [] - - def child(cmd, **_kwargs): - calls.append(cmd) - if len(calls) == 2: - if isinstance(failure, Exception): - raise failure - return failure - return successful_result(cmd) - - with patch.object(sys, 'argv', argv), patch.object(matrix.subprocess, 'run', side_effect=child), contextlib.redirect_stdout(io.StringIO()): - if baseline: - matrix.main() - else: - with self.assertRaises(SystemExit): - matrix.main() - rows = json.loads(output.read_text()) - self.assertEqual(len(rows), 6 if baseline else 30) - self.assertTrue(matrix.fixed_row_passes(rows[0])) - self.assertFalse(matrix.fixed_row_passes(rows[1])) - self.assertTrue(matrix.fixed_row_passes(rows[-1])) - self.assertTrue(rows[1]['harnessError']) - self.assertEqual(rows[1]['shell'], 'bash') - self.assertEqual(rows[1]['phase'], 'cold' if baseline else 'warm') - if name == 'timeout': - self.assertIsNone(rows[1]['harnessExit']) - self.assertEqual(rows[1]['rawPrefix'], 'partial\ufffd') - self.assertEqual(rows[1]['harnessStderr'], 'timeout diagnostic') - if name == 'empty': - self.assertEqual(rows[1]['harnessExit'], 17) - self.assertEqual(rows[1]['harnessStderr'], 'panic diagnostic') - - def test_real_child_exit_and_stderr_override_forged_json(self): - forged = dict(executed=True, duplexExact=True, error='', execError='', stderr='', - localRequests=1, rawStdout='ping', stdout='live', - harnessExit=0, harnessError='', harnessStderr='') - code = f'import sys; print({json.dumps(forged)!r}); sys.stderr.write("child diagnostic"); sys.exit(17)' - row = matrix.run_cell([sys.executable, '-c', code], None) - row['phase'] = 'cold' - self.assertEqual(row['harnessExit'], 17) - self.assertEqual(row['harnessStderr'], 'child diagnostic') - self.assertFalse(matrix.fixed_row_passes(row)) - - -if __name__ == '__main__': - unittest.main()