diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index 8243c77..7da3261 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -3,10 +3,15 @@ name: Commit on: pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: commits: name: Check Commits runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read pull-requests: write diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 9314776..99e15b4 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,10 +6,18 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: lint: name: Lint runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: free disk space uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # v2.0.0 diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 8a6ffc2..b83d3f7 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -8,10 +8,15 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check: name: Pre-commit Checks runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d792843..0b9363c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -5,21 +5,27 @@ on: workflow_dispatch: inputs: core_tag: - description: >- - Core release tag to qualify with this provider checkout (for example - v1.25.0-beta.8). Runs the release-qualification job. + description: Core release tag to qualify, for example v1.25.0-beta.8. Leave empty to skip qualification. required: false type: string provider_tag: description: Published SSH provider release to qualify - required: true + required: false default: v1.1.47 type: string +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: name: Integration tests (compatibility core v1.24.0, not release qualification) runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -34,7 +40,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y bash zsh fish openssh-server - name: unit tests - run: go test -race -covermode=atomic . ./cmd/... ./hack/... ./pkg/... + run: go test -race -covermode=atomic . ./cmd/... ./hack/... ./pkg/... ./e2e/qualification/... env: CGO_ENABLED: "1" @@ -56,11 +62,7 @@ jobs: du -sh "$(go env GOCACHE)" "$(go env GOMODCACHE)" - name: provision login shell accounts - run: | - sudo service ssh start - for shell in bash zsh fish; do - sudo useradd --create-home --shell "$(command -v "$shell")" "devsy-$shell" - done + run: go run ./hack/ci provision-accounts - name: integration tests working-directory: ./e2e @@ -71,6 +73,7 @@ jobs: windows-stdin: name: Windows redirected SSH stdin runs-on: windows-latest + timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -90,8 +93,6 @@ jobs: fail-fast: false matrix: core_artifact: [cli, desktop] - permissions: - contents: read env: CORE_TAG: ${{ inputs.core_tag }} CORE_REPO: devsy-org/devsy @@ -121,59 +122,17 @@ jobs: run: sudo apt-get update && sudo apt-get install -y bash zsh fish openssh-server - name: download and verify the released provider - run: | - set -euo pipefail - mkdir -p dist/build_linux_amd64_v1 - asset=devsy-provider-ssh-linux-amd64 - gh release download "$PROVIDER_TAG" --repo "$GITHUB_REPOSITORY" \ - --pattern "$asset" --dir dist/build_linux_amd64_v1 --clobber - gh release download "$PROVIDER_TAG" --repo "$GITHUB_REPOSITORY" \ - --pattern checksums.txt --dir dist --clobber - binary="dist/build_linux_amd64_v1/$asset" - digest="$(sha256sum "$binary" | cut -d' ' -f1)" - published_digest="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$PROVIDER_TAG" \ - --jq ".assets[] | select(.name == \"$asset\") | .digest")" - if [ "$published_digest" != "sha256:$digest" ]; then - echo "::error::downloaded provider sha256:$digest differs from '$published_digest'" - exit 1 - fi - chmod 0755 "$binary" - echo "provider binary sha256: $digest (matches published release digest)" \ - | tee -a "$QUALIFICATION_DIR/pair-verification.txt" >> "$GITHUB_STEP_SUMMARY" + run: go run ./hack/ci verify-provider env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: require the explicit candidate before SSH setup - working-directory: ./e2e - run: | - status=0 - env -u DEVSY_E2E_BINARY go test -v -timeout 300s --ginkgo.v \ - --ginkgo.label-filter=release-qualification \ - --ginkgo.focus='bash builtin=false installs a cold agent by remote download' \ - > "$QUALIFICATION_DIR/missing-binary-guard.log" 2>&1 || status=$? - cat "$QUALIFICATION_DIR/missing-binary-guard.log" - test "$status" -eq 1 - grep -F 'DEVSY_E2E_BINARY is required' "$QUALIFICATION_DIR/missing-binary-guard.log" + run: go run ./hack/ci missing-candidate-guard env: DEVSY_E2E_RELEASE_QUALIFICATION: "true" - - name: provision login shell accounts - run: | - sudo service ssh start - for shell in bash zsh fish; do - sudo useradd --create-home --shell "$(command -v "$shell")" "devsy-$shell" - sudo usermod --append --groups docker "devsy-$shell" - done - sudo useradd --create-home --shell "$(command -v bash)" --groups docker devsy-sudo - policy=/etc/sudoers.d/devsy-e2e-qualification - sudo test ! -e "$policy" - rule="$(mktemp)" - trap 'rm -f "$rule"' EXIT - printf '%s\n' 'devsy-sudo ALL=(root) NOPASSWD: SETENV: /bin/sh -c * devsy-install *' > "$rule" - sudo visudo -cf "$rule" - echo "DEVSY_SUDO_FIXTURE_POLICY=$policy" >> "$GITHUB_ENV" - sudo install -m 0440 -o root -g root "$rule" "$policy" - sudo visudo -cf "$policy" + - name: provision qualification accounts + run: go run ./hack/ci provision-qualification-accounts - name: qualify login shells with the explicit pair working-directory: ./e2e @@ -185,35 +144,12 @@ jobs: DEVSY_E2E_SHELL_MATRIX: "true" DEVSY_E2E_REPORT: ${{ runner.temp }}/qualification/qualification-report.json - - name: remove owned sudo fixture policy - if: always() && env.DEVSY_SUDO_FIXTURE_POLICY != '' - run: sudo rm -f -- "$DEVSY_SUDO_FIXTURE_POLICY" + - name: remove qualification accounts and sudo fixture + if: always() + run: go run ./hack/ci cleanup-qualification-accounts - - name: require a complete, passing report - run: | - jq -e --arg sha "$CORE_SHA256" ' - .pair.coreBinarySha256 == $sha - and (.cells | length == 6) - and ([.cells[] | [.shell, .builtinSsh]] | unique | length == 6) - and all(.cells[]; - .cleanup == "ok" - and (.scenarios | length == 5) - and all(.scenarios[]; .outcome == "passed") - and (.agentDir | contains(" ") and contains("\u0027") - and contains("\\") and contains("$") and contains("\u03bb")) - and .agentSha256.download == $sha - and .agentSha256.stdin == $sha) - and .privilege.user == "devsy-sudo" - and .privilege.userUid > 0 - and .privilege.directoryState == "0:755" - and .privilege.writeDenied == true - and .privilege.agentUid == 0 - and .privilege.agentVersion == .pair.coreVersion - and .privilege.agentSha256 == $sha - and .privilege.liveCommand == true - and .privilege.outcome == "passed" - and .privilege.cleanup == "ok" - ' "$QUALIFICATION_DIR/qualification-report.json" + - name: verify the qualification report + run: go run ./hack/ci verify-report - name: upload qualification report if: always() diff --git a/.golangci.yaml b/.golangci.yaml index 7075100..fa7d76b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -33,14 +33,22 @@ linters: - whitespace exclusions: rules: - - path: hack/ci/qualify\.go + - path: 'hack/ci/(qualify|verify_provider)\.go' linters: - gosec text: 'G302: Expect file permissions to be 0600 or less' - - path: hack/ci/qualify\.go + - path: 'hack/ci/(exec|guard)\.go' linters: - gosec - text: 'G204: Subprocess launched with variable' + text: 'G204|G702' + - path: 'hack/ci/provision_test\.go' + linters: + - gosec + text: 'G703' + - path: 'e2e/qualification/report\.go' + linters: + - gosec + text: 'G304' settings: cyclop: max-complexity: 8 diff --git a/e2e/README.md b/e2e/README.md index 272d9ec..f41393d 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,66 +1,114 @@ -### E2E tests +# E2E tests -#### Prerequisites +These tests run the SSH provider against real login shells and a local sshd. +Run Go commands from `e2e/` unless a step says otherwise. + +## Layout + +### `e2e/qualification` + +Plain Go package named `qualification` with no Ginkgo or Gomega imports. +It is the single source for the report schema and the shell matrix. + +- `SchemaVersion`, `PrivilegeUser`, and the lifecycle scenario names. +- `Transports`: the shell and transport matrix. `Shells()` returns the unique shell names. +- `Pair`, `Cell`, `Scenario`, `Privilege`, and `Report`: the JSON report types. +- `Load` reads a report file. `Report.Validate` checks it. + +### `e2e/tests/integration` + +Ginkgo specs in package `integration`. + +- `labels.go`: label names used by the spec filters. +- `environment.go`: provider and core setup, SSH keys, the login account key, and `providerCommandEnv`. +- `compat_lifecycle.go`: the compatibility lifecycle suite. It uses the default core download. +- `cell.go`: one shell and transport cell. It holds setup, cleanup, and the qualification preparation. +- `lifecycle.go`: the full workspace lifecycle spec for each cell and its scenario checks. +- `provider_command.go`: the provider command stream and exit status specs. +- `privileged_install.go`: the `devsy-sudo` scenario that installs the agent with sudo. +- `pair.go`: the qualification environment gate, the explicit core and provider pair, and the report writer. + +### `hack/ci` + +CI commands. Run them from the repository root with `go run ./hack/ci `. + +- `provision-accounts`: create the `devsy-bash`, `devsy-zsh`, and `devsy-fish` accounts. +- `provision-qualification-accounts`: create the qualification accounts and the sudo fixture. +- `cleanup-qualification-accounts`: remove the qualification accounts and the sudo fixture. +- `qualify-core`: resolve and verify the candidate core. +- `verify-provider`: download the released provider and check its digest. +- `missing-candidate-guard`: fail when the explicit candidate is missing. +- `verify-report`: validate the qualification report. + +## Run the compatibility suite + +The default lifecycle suite downloads core v1.24.0 unless `DEVSY_E2E_BINARY` is set. +That run checks backward compatibility only. It is not release qualification. -Make sure you have ginkgo installed on your local machine: ``` -go get github.com/onsi/ginkgo/ginkgo +go test -v -ginkgo.v -timeout 3600s --ginkgo.label-filter=integration ``` -#### Run all E2E test +## Run the provider command matrix locally + +Use a disposable Linux host or VM. The account command creates real login accounts. + +1. Install the shells and sshd. On Ubuntu: + ``` + sudo apt-get update && sudo apt-get install -y bash zsh fish openssh-server + ``` +2. Create the accounts from the repository root: + ``` + go run ./hack/ci provision-accounts + ``` +3. Start sshd: + ``` + sudo service ssh start + ``` +4. Run the matrix from `e2e/`: + ``` + DEVSY_E2E_SHELL_MATRIX=true go test -v -ginkgo.v -timeout 3600s \ + --ginkgo.label-filter=provider-command + ``` + +## Release qualification + +Qualification checks one explicit core and provider pair. It does not pick a default core. + +Start it by dispatching the `Tests` workflow with `core_tag` (for example `v1.25.0-beta.8`) +and `provider_tag` (default `v1.1.47`). The qualification job only runs when `core_tag` is set. +From a terminal: + ``` -# Install ginkgo and run in this directory -go test -v -ginkgo.v -timeout 3600s +gh workflow run tests.yaml -f core_tag=v1.25.0-beta.8 -f provider_tag=v1.1.47 ``` -#### Release qualification (explicit core/provider pair) +The workflow runs two independent jobs. The `cli` job qualifies the standalone Linux amd64 CLI. +The `desktop` job qualifies the CLI extracted from the Desktop candidate's `Devsy_linux_amd64.deb`. +Each job verifies the published asset digest, runs that binary's `--version`, records its SHA256, +and checks that core `e22be00` and `9c80b29` and provider `d31fed1` are ancestors of the selected +release commits. The provider binary and checksums come from the selected provider release. -The default lifecycle suite downloads core v1.24.0 unless `DEVSY_E2E_BINARY` is set. -That run checks backward compatibility only and is not release qualification. - -The qualification suite requires an explicit `DEVSY_E2E_BINARY`; missing candidate -metadata or host prerequisites fail rather than selecting the compatibility core. -Dispatch the `Tests` workflow with `core_tag` and `provider_tag` (default `v1.1.47`). -Two independent jobs qualify the standalone Linux amd64 CLI and the CLI extracted -from the Desktop candidate's `Devsy_linux_amd64.deb`. Each job verifies the published -asset digest, executes that binary's `--version`, records its SHA256, and checks that -core `e22be00` and `9c80b29` and provider `d31fed1` are ancestors of the selected release commits. -The provider binary and checksums come from the selected provider release. - -The jobs run `release-qualification || provider-command` and upload -`release-qualification--cli` and `release-qualification--desktop` reports. -The report identifies the exact core/provider pair and harness commit, all six -shell/transport cells, five agent scenarios per cell, binary hashes, and cleanup. -All six cells use an agent path containing a space, apostrophe, backslash, dollar -sign, and Unicode character; every remote helper quotes the path literally. -The shell/transport cases come from one shared matrix definition used by both the -bootstrap stream checks and lifecycle qualification. Run qualification in one -process: its fail-closed report state is process-local and does not aggregate across -Ginkgo workers. Adding a shell or transport also requires updating the report gate -and workflow account provisioning; the matrix definition alone is not sufficient. -One additional focused scenario uses the dedicated non-root `devsy-sudo` account -and a root-owned `0755` agent directory. It proves that an ordinary write is -denied, a real sudo installation produces a root-owned candidate agent with the -expected version/checksum, and workspace SSH still executes a live command. -A skipped or incomplete matrix or privilege scenario cannot pass the report gate. - -To run by hand, use a disposable Ubuntu 24.04 Linux amd64 host with local sshd, -Docker, and provisioned `devsy-bash`, `devsy-zsh`, `devsy-fish`, and `devsy-sudo` -accounts in the `docker` group. The release-qualification job is pinned to Ubuntu -24.04 because its temporary sudoers rule uses command-argument wildcards accepted -by classic sudo but rejected by sudo-rs on newer Ubuntu runners. This compatibility -fixture grants the dedicated test account a broad root shell dispatch and is only -for disposable qualification runners; it is not a production least-privilege policy. -Use the workflow's provisioning on that owned host: only -`devsy-sudo` receives the validated, temporary sudoers rule for -`/bin/sh -c * devsy-install *`, with `NOPASSWD` and `SETENV` for the installer's -explicit dispatch. Remove `/etc/sudoers.d/devsy-e2e-qualification` after the run, -including failure paths, as the workflow's `always()` cleanup does. -Run from `e2e/`, after placing the released provider binary at -`dist/build_linux_amd64_v1/devsy-provider-ssh-linux-amd64` and its release checksums -at `dist/checksums.txt` relative to the repository root. Unset `DEVSY_AGENT_URL`, -`DEVSY_AGENT_BINARY`, and `DEVSY_AGENT_PREFER_DOWNLOAD`: inherited overrides can -disable the version check or select the wrong agent transport. +Each job uploads `release-qualification--cli` or `release-qualification--desktop`. +The upload holds `pair-verification.txt`, `qualification-report.json`, and `missing-binary-guard.log`. + +### Why Ubuntu 24.04 + +The release job runs on `ubuntu-24.04` on purpose. Its temporary sudoers rule uses command +argument wildcards. Classic sudo accepts them. sudo-rs on newer Ubuntu runners rejects them. +The rule is a fixture for disposable runners only. It lets the `devsy-sudo` account run a broad +root shell dispatch, `/bin/sh -c * devsy-install *`. It is not a production policy. + +### Run it by hand + +Use a disposable Ubuntu 24.04 Linux amd64 host with sshd and Docker. Provision the accounts first. +The `devsy-bash`, `devsy-zsh`, `devsy-fish`, and `devsy-sudo` accounts must exist and belong to the +`docker` group. Put the released provider binary at +`dist/build_linux_amd64_v1/devsy-provider-ssh-linux-amd64` and its checksums at `dist/checksums.txt`, +both relative to the repository root. + +Unset `DEVSY_AGENT_URL`, `DEVSY_AGENT_BINARY`, and `DEVSY_AGENT_PREFER_DOWNLOAD` first. +Inherited overrides can disable the version check or select the wrong agent transport. ``` DEVSY_E2E_RELEASE_QUALIFICATION=true DEVSY_E2E_SHELL_MATRIX=true \ @@ -73,12 +121,58 @@ go test -v -ginkgo.v -timeout 14400s \ --ginkgo.label-filter='release-qualification || provider-command' ``` -Only disposable fixture accounts are provisioned; never change a user's login shell, -`SHELL`, `/etc/passwd`, or `.zshrc` to make qualification pass. Download and stdin -transfer must deliver the candidate's exact bytes. The matrix retains the existing -command-boundary stream, stderr, and nonzero-exit coverage. +Run qualification in one process. Its fail-closed report state lives in that process. +It does not merge results from Ginkgo parallel nodes, so do not run it with parallel nodes. + +When the candidate binary is missing, the run fails with `DEVSY_E2E_BINARY is required`. +A missing candidate never falls back to the compatibility core. + +When the run ends, run `go run ./hack/ci cleanup-qualification-accounts` to remove the sudo rule. +The workflow does this in an `always()` step. + +### Rules for the matrix + +- Only the fixture accounts are provisioned. Never change a user's login shell, `SHELL`, + `/etc/passwd`, or `.zshrc` to make a run pass. +- Every cell uses an agent path with a space, an apostrophe, a backslash, a dollar sign, + and a Unicode character. Each remote helper quotes that path literally. +- Download and stdin transfer must deliver the candidate's exact bytes. +- The matrix keeps the command-boundary stream, stderr, and nonzero-exit coverage. +- A skipped or incomplete matrix or privilege scenario fails the report gate. + +Each cell runs five agent scenarios. The privilege scenario runs once. It uses the `devsy-sudo` +account and a root-owned `0755` agent directory. It proves that an ordinary write is denied, +that a real sudo install produces a root-owned agent with the expected version and checksum, +and that workspace SSH still runs a live command. + +## Report schema + +The report is JSON. Its top-level `schemaVersion` is `1`, which is `qualification.SchemaVersion`. + +- `pair`: `coreBinary`, `coreTag`, `coreSha`, `coreVersion`, `coreBinarySha256`, `providerRef`, + `providerSha`, `providerBinary`, `providerBinarySha256`, and `harnessSha`. +- `cells`: one entry per shell and transport. Each has `shell`, `user`, `builtinSsh`, `workspace`, + `agentDir`, `loginShell`, `scenarios`, `agentSha256`, and `cleanup`. +- `privilege`: the `devsy-sudo` run. It has `user`, `userUid`, `agentDir`, `directoryState`, + `writeDenied`, `writeError`, `agentUid`, `agentVersion`, `agentSha256`, `liveCommand`, + `outcome`, and `cleanup`. + +`go run ./hack/ci verify-report` loads the report and checks it. The gate requires the pair to +match `CORE_SHA256`, all six cells to be present, every scenario to pass, cleanup to be `ok`, +and the privilege record to pass. A missing or failed item fails the gate. + +## Add a shell or transport + +1. Edit `Transports` in `e2e/qualification`. Each entry is a shell name and whether it uses + built-in SSH. +2. Do not edit the matrix, the report checks, or the account provisioning. They all read + `Transports` and `Shells()`. +3. If the workflow does not install the shell yet, add its package to the `apt-get install` lines + in `tests.yaml`. + +## Scope of the Desktop run -Extracting and qualifying the Desktop-bundled CLI is not native Desktop UI -acceptance, installation/update validation, or a customer-host reproduction. -Stable release publication, supported-platform artifact smokes, reporter-equivalent -Desktop fresh/reopen traces, and customer confirmation remain separate release work. +Extracting and qualifying the Desktop-bundled CLI is not native Desktop UI acceptance, +installation or update validation, or a customer host reproduction. +Stable release publication, supported-platform artifact smokes, reporter-equivalent Desktop +fresh and reopen traces, and customer confirmation remain separate release work. diff --git a/e2e/qualification/matrix.go b/e2e/qualification/matrix.go new file mode 100644 index 0000000..06ea335 --- /dev/null +++ b/e2e/qualification/matrix.go @@ -0,0 +1,48 @@ +package qualification + +const SchemaVersion = 1 + +const PrivilegeUser = "devsy-sudo" + +const ( + ScenarioColdDownload = "installs a cold agent by remote download" + ScenarioCurrentReuse = "reuses the current agent when the workspace is opened again" + ScenarioStaleReplacement = "replaces a deliberately old agent" + ScenarioStdinTransfer = "installs a cold agent by stdin transfer and keeps stdin for the live stream" + ScenarioLoginShellUnchanged = "leaves the account login shell unchanged" +) + +var LifecycleScenarios = []string{ + ScenarioColdDownload, + ScenarioCurrentReuse, + ScenarioStaleReplacement, + ScenarioStdinTransfer, + ScenarioLoginShellUnchanged, +} + +type Transport struct { + Shell string + Builtin bool +} + +var Transports = []Transport{ + {Shell: "bash"}, + {Shell: "bash", Builtin: true}, + {Shell: "zsh"}, + {Shell: "zsh", Builtin: true}, + {Shell: "fish"}, + {Shell: "fish", Builtin: true}, +} + +func Shells() []string { + var shells []string + seen := make(map[string]bool) + for _, transport := range Transports { + if seen[transport.Shell] { + continue + } + seen[transport.Shell] = true + shells = append(shells, transport.Shell) + } + return shells +} diff --git a/e2e/qualification/report.go b/e2e/qualification/report.go new file mode 100644 index 0000000..54018fd --- /dev/null +++ b/e2e/qualification/report.go @@ -0,0 +1,75 @@ +package qualification + +import ( + "encoding/json" + "fmt" + "os" +) + +type Pair struct { + CoreBinary string `json:"coreBinary"` + CoreTag string `json:"coreTag"` + CoreSHA string `json:"coreSha"` + CoreVersion string `json:"coreVersion"` + CoreBinarySHA256 string `json:"coreBinarySha256"` + ProviderRef string `json:"providerRef"` + ProviderSHA string `json:"providerSha"` + ProviderBinary string `json:"providerBinary"` + ProviderBinarySHA256 string `json:"providerBinarySha256"` + HarnessSHA string `json:"harnessSha"` +} + +type Scenario struct { + Name string `json:"name"` + Outcome string `json:"outcome"` +} + +type Cell struct { + Shell string `json:"shell"` + User string `json:"user"` + BuiltinSSH bool `json:"builtinSsh"` + Workspace string `json:"workspace"` + AgentDir string `json:"agentDir"` + LoginShell string `json:"loginShell"` + Scenarios []Scenario `json:"scenarios"` + AgentSHA256 map[string]string `json:"agentSha256"` + CleanupStatus string `json:"cleanup"` +} + +type Privilege struct { + User string `json:"user"` + UserUID uint64 `json:"userUid"` + AgentDir string `json:"agentDir"` + DirectoryState string `json:"directoryState"` + WriteDenied bool `json:"writeDenied"` + WriteError string `json:"writeError"` + AgentUID uint64 `json:"agentUid"` + AgentVersion string `json:"agentVersion"` + AgentSHA256 string `json:"agentSha256"` + LiveCommand bool `json:"liveCommand"` + Outcome string `json:"outcome"` + CleanupStatus string `json:"cleanup"` +} + +type Report struct { + SchemaVersion int `json:"schemaVersion"` + Pair Pair `json:"pair"` + Cells []*Cell `json:"cells"` + Privilege *Privilege `json:"privilege"` +} + +func Load(path string) (Report, error) { + file, err := os.Open(path) + if err != nil { + return Report{}, fmt.Errorf("open report %s: %w", path, err) + } + defer func() { _ = file.Close() }() + + decoder := json.NewDecoder(file) + decoder.DisallowUnknownFields() + var report Report + if err := decoder.Decode(&report); err != nil { + return Report{}, fmt.Errorf("decode report %s: %w", path, err) + } + return report, nil +} diff --git a/e2e/qualification/report_test.go b/e2e/qualification/report_test.go new file mode 100644 index 0000000..17a726a --- /dev/null +++ b/e2e/qualification/report_test.go @@ -0,0 +1,54 @@ +package qualification + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestLoadRoundTrip(t *testing.T) { + want := validReport(testSHA) + data, err := json.MarshalIndent(want, "", " ") + if err != nil { + t.Fatalf("marshal report: %v", err) + } + path := filepath.Join(t.TempDir(), "report.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write report: %v", err) + } + + got, err := Load(path) + if err != nil { + t.Fatalf("load report: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatal("loaded report differs from the written report") + } +} + +func TestLoadRejectsUnknownField(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.json") + data := `{"schemaVersion": 1, "pair": {}, "cells": [], "privilege": null, "extra": true}` + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatalf("write report: %v", err) + } + + if _, err := Load(path); err == nil { + t.Fatal("Load accepted a report with an unknown field") + } +} + +func TestLoadMissingFileNamesPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing.json") + + _, err := Load(path) + if err == nil { + t.Fatal("Load accepted a missing file") + } + if !strings.Contains(err.Error(), path) { + t.Fatalf("error %q does not name the path %q", err, path) + } +} diff --git a/e2e/qualification/validate.go b/e2e/qualification/validate.go new file mode 100644 index 0000000..9b80f83 --- /dev/null +++ b/e2e/qualification/validate.go @@ -0,0 +1,187 @@ +package qualification + +import ( + "errors" + "fmt" + "strings" +) + +const ( + outcomePassed = "passed" + cleanupOK = "ok" +) + +var agentDirLiterals = []string{" ", "'", `\`, "$", "\u03bb"} + +type Expectation struct { + CoreBinarySHA256 string +} + +func (r Report) Validate(want Expectation) error { + if err := r.validateHeader(want); err != nil { + return err + } + if err := r.validateCells(want); err != nil { + return err + } + return r.validatePrivilege(want) +} + +func (r Report) validateHeader(want Expectation) error { + if r.SchemaVersion != SchemaVersion { + return fmt.Errorf("schema version %d, want %d", r.SchemaVersion, SchemaVersion) + } + if want.CoreBinarySHA256 == "" { + return errors.New("expected core binary sha256 is empty") + } + if r.Pair.CoreBinarySHA256 != want.CoreBinarySHA256 { + return fmt.Errorf( + "pair core binary sha256 %q, want %q", + r.Pair.CoreBinarySHA256, want.CoreBinarySHA256, + ) + } + return nil +} + +func (r Report) validateCells(want Expectation) error { + if len(r.Cells) != len(Transports) { + return fmt.Errorf("%d cells, want %d", len(r.Cells), len(Transports)) + } + if err := validateTransports(r.Cells); err != nil { + return err + } + for _, cell := range r.Cells { + if err := validateCell(cell, want); err != nil { + return err + } + } + return nil +} + +func validateTransports(cells []*Cell) error { + seen := make(map[Transport]bool, len(cells)) + for _, cell := range cells { + if cell == nil { + return errors.New("report has a null cell") + } + transport := cell.transport() + if seen[transport] { + return fmt.Errorf("duplicate cell for %s", describe(transport)) + } + seen[transport] = true + } + for _, transport := range Transports { + if !seen[transport] { + return fmt.Errorf("missing cell for %s", describe(transport)) + } + } + return nil +} + +func validateCell(cell *Cell, want Expectation) error { + label := describe(cell.transport()) + if cell.CleanupStatus != cleanupOK { + return fmt.Errorf("%s: cleanup %q, want %q", label, cell.CleanupStatus, cleanupOK) + } + if err := validateScenarios(cell.Scenarios); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + if err := validateAgentDir(cell.AgentDir); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + if err := validateAgentDigests(cell.AgentSHA256, want.CoreBinarySHA256); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + return nil +} + +func validateScenarios(scenarios []Scenario) error { + if len(scenarios) != len(LifecycleScenarios) { + return fmt.Errorf("%d scenarios, want %d", len(scenarios), len(LifecycleScenarios)) + } + for i, scenario := range scenarios { + if scenario.Name != LifecycleScenarios[i] { + return fmt.Errorf("scenario %d is %q, want %q", i, scenario.Name, LifecycleScenarios[i]) + } + if scenario.Outcome != outcomePassed { + return fmt.Errorf("scenario %q outcome %q, want %q", + scenario.Name, scenario.Outcome, outcomePassed) + } + } + return nil +} + +func validateAgentDir(agentDir string) error { + for _, literal := range agentDirLiterals { + if !strings.Contains(agentDir, literal) { + return fmt.Errorf("agent dir %q lacks %q", agentDir, literal) + } + } + return nil +} + +func validateAgentDigests(digests map[string]string, want string) error { + for _, source := range []string{"download", "stdin"} { + if digests[source] != want { + return fmt.Errorf("agent sha256 for %s is %q, want %q", source, digests[source], want) + } + } + return nil +} + +func (r Report) validatePrivilege(want Expectation) error { + privilege := r.Privilege + if privilege == nil { + return errors.New("report has no privilege section") + } + if err := validatePrivilegeUser(privilege); err != nil { + return err + } + return validatePrivilegeAgent(privilege, r.Pair.CoreVersion, want.CoreBinarySHA256) +} + +func validatePrivilegeUser(privilege *Privilege) error { + if privilege.User != PrivilegeUser { + return fmt.Errorf("privilege user %q, want %q", privilege.User, PrivilegeUser) + } + if privilege.UserUID == 0 { + return errors.New("privilege user has uid 0") + } + if privilege.DirectoryState != "0:755" { + return fmt.Errorf("privilege directory state %q, want \"0:755\"", privilege.DirectoryState) + } + if !privilege.WriteDenied { + return errors.New("privilege user write to the agent directory was not denied") + } + return nil +} + +func validatePrivilegeAgent(privilege *Privilege, version, sha string) error { + if privilege.AgentUID != 0 { + return fmt.Errorf("privilege agent uid %d, want 0", privilege.AgentUID) + } + if privilege.AgentVersion != version { + return fmt.Errorf("privilege agent version %q, want %q", privilege.AgentVersion, version) + } + if privilege.AgentSHA256 != sha { + return fmt.Errorf("privilege agent sha256 %q, want %q", privilege.AgentSHA256, sha) + } + if !privilege.LiveCommand { + return errors.New("privilege live command did not run") + } + if privilege.Outcome != outcomePassed { + return fmt.Errorf("privilege outcome %q, want %q", privilege.Outcome, outcomePassed) + } + if privilege.CleanupStatus != cleanupOK { + return fmt.Errorf("privilege cleanup %q, want %q", privilege.CleanupStatus, cleanupOK) + } + return nil +} + +func (c *Cell) transport() Transport { + return Transport{Shell: c.Shell, Builtin: c.BuiltinSSH} +} + +func describe(transport Transport) string { + return fmt.Sprintf("shell %s builtinSsh %t", transport.Shell, transport.Builtin) +} diff --git a/e2e/qualification/validate_test.go b/e2e/qualification/validate_test.go new file mode 100644 index 0000000..62e9f85 --- /dev/null +++ b/e2e/qualification/validate_test.go @@ -0,0 +1,164 @@ +package qualification + +import ( + "strings" + "testing" +) + +const testSHA = "5b1c9e0d" + +const ( + testAgentDir = "/tmp/devsy agent'\\$\u03bb" + staleSHA = "other" + outcomeFailed = "failed" +) + +func validReport(sha string) Report { + cells := make([]*Cell, 0, len(Transports)) + for _, transport := range Transports { + cells = append(cells, &Cell{ + Shell: transport.Shell, + User: "devsy-user", + BuiltinSSH: transport.Builtin, + Workspace: "workspace", + AgentDir: testAgentDir, + LoginShell: "/bin/" + transport.Shell, + Scenarios: passedScenarios(), + AgentSHA256: map[string]string{"download": sha, "stdin": sha}, + CleanupStatus: cleanupOK, + }) + } + return Report{ + SchemaVersion: SchemaVersion, + Pair: Pair{CoreVersion: "1.2.3", CoreBinarySHA256: sha}, + Cells: cells, + Privilege: &Privilege{ + User: PrivilegeUser, + UserUID: 1001, + AgentDir: "/opt/devsy-agent", + DirectoryState: "0:755", + WriteDenied: true, + WriteError: "permission denied", + AgentUID: 0, + AgentVersion: "1.2.3", + AgentSHA256: sha, + LiveCommand: true, + Outcome: outcomePassed, + CleanupStatus: cleanupOK, + }, + } +} + +func passedScenarios() []Scenario { + scenarios := make([]Scenario, 0, len(LifecycleScenarios)) + for _, name := range LifecycleScenarios { + scenarios = append(scenarios, Scenario{Name: name, Outcome: outcomePassed}) + } + return scenarios +} + +func TestValidReportPasses(t *testing.T) { + if err := validReport(testSHA).Validate(Expectation{CoreBinarySHA256: testSHA}); err != nil { + t.Fatalf("valid report rejected: %v", err) + } +} + +type violation struct { + name string + mutate func(r *Report, want *Expectation) +} + +func agentDirWithout(literal string) func(*Report, *Expectation) { + return func(r *Report, _ *Expectation) { + r.Cells[0].AgentDir = strings.ReplaceAll(r.Cells[0].AgentDir, literal, "") + } +} + +func privilegeViolation(mutate func(*Privilege)) func(*Report, *Expectation) { + return func(r *Report, _ *Expectation) { mutate(r.Privilege) } +} + +func cellViolation(mutate func(*Cell)) func(*Report, *Expectation) { + return func(r *Report, _ *Expectation) { mutate(r.Cells[0]) } +} + +var headerViolations = []violation{ + {"wrong schema version", func(r *Report, _ *Expectation) { + r.SchemaVersion = SchemaVersion + 1 + }}, + {"empty expectation sha", func(r *Report, want *Expectation) { + r.Pair.CoreBinarySHA256 = "" + want.CoreBinarySHA256 = "" + }}, + {"pair sha differs from expectation", func(r *Report, _ *Expectation) { + r.Pair.CoreBinarySHA256 = staleSHA + }}, + {"missing cell", func(r *Report, _ *Expectation) { + r.Cells = r.Cells[:len(r.Cells)-1] + }}, + {"duplicate transport", func(r *Report, _ *Expectation) { + r.Cells[1].Shell = r.Cells[0].Shell + r.Cells[1].BuiltinSSH = r.Cells[0].BuiltinSSH + }}, +} + +var cellViolations = []violation{ + {"wrong cleanup", cellViolation(func(c *Cell) { c.CleanupStatus = outcomeFailed })}, + {"scenario failed", cellViolation(func(c *Cell) { c.Scenarios[0].Outcome = outcomeFailed })}, + {"scenario order wrong", cellViolation(func(c *Cell) { + c.Scenarios[0], c.Scenarios[1] = c.Scenarios[1], c.Scenarios[0] + })}, + {"agent dir missing space", agentDirWithout(" ")}, + {"agent dir missing apostrophe", agentDirWithout("'")}, + {"agent dir missing backslash", agentDirWithout(`\`)}, + {"agent dir missing dollar", agentDirWithout("$")}, + {"agent dir missing lambda", agentDirWithout("\u03bb")}, + { + "download sha mismatch", + cellViolation(func(c *Cell) { c.AgentSHA256["download"] = staleSHA }), + }, + {"stdin sha mismatch", cellViolation(func(c *Cell) { c.AgentSHA256["stdin"] = staleSHA })}, +} + +var privilegeViolations = []violation{ + {"nil privilege", func(r *Report, _ *Expectation) { r.Privilege = nil }}, + {"privilege user wrong", privilegeViolation(func(p *Privilege) { p.User = "root" })}, + {"root uid user", privilegeViolation(func(p *Privilege) { p.UserUID = 0 })}, + { + "wrong directory state", + privilegeViolation(func(p *Privilege) { p.DirectoryState = "0:777" }), + }, + {"write not denied", privilegeViolation(func(p *Privilege) { p.WriteDenied = false })}, + {"agent uid not 0", privilegeViolation(func(p *Privilege) { p.AgentUID = 1000 })}, + {"agent version mismatch", privilegeViolation(func(p *Privilege) { p.AgentVersion = "0.0.1" })}, + { + "privilege agent sha mismatch", + privilegeViolation(func(p *Privilege) { p.AgentSHA256 = staleSHA }), + }, + {"live command false", privilegeViolation(func(p *Privilege) { p.LiveCommand = false })}, + { + "privilege outcome failed", + privilegeViolation(func(p *Privilege) { p.Outcome = outcomeFailed }), + }, + { + "privilege cleanup not ok", + privilegeViolation(func(p *Privilege) { p.CleanupStatus = outcomeFailed }), + }, +} + +func TestValidateRejectsViolations(t *testing.T) { + var all []violation + all = append(all, headerViolations...) + all = append(all, cellViolations...) + all = append(all, privilegeViolations...) + for _, tt := range all { + t.Run(tt.name, func(t *testing.T) { + report := validReport(testSHA) + want := Expectation{CoreBinarySHA256: testSHA} + tt.mutate(&report, &want) + if err := report.Validate(want); err == nil { + t.Fatal("Validate accepted the report") + } + }) + } +} diff --git a/e2e/tests/integration/login_shells_workspace.go b/e2e/tests/integration/cell.go similarity index 67% rename from e2e/tests/integration/login_shells_workspace.go rename to e2e/tests/integration/cell.go index 03e1722..eae67d7 100644 --- a/e2e/tests/integration/login_shells_workspace.go +++ b/e2e/tests/integration/cell.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" "github.com/devsy-org/devsy/e2e/framework" "github.com/kballard/go-shellquote" "github.com/onsi/ginkgo/v2" @@ -32,7 +33,6 @@ const ( pollTimeout = 2 * time.Minute pollInterval = 5 * time.Second - // Debug lines the core CLI logs for the agent transport it selected. logStdinTransfer = "inject binary" logRemoteDownload = "prefer downloading agent from URL" logHostSSHServerStarted = "starting ssh server: stdio=true" @@ -50,10 +50,8 @@ func prepareQualification() { }) } -// shellCell is one login shell and SSH implementation pair with its own Devsy home, -// provider, workspace, and remote agent directory. type shellCell struct { - pair qualifiedPair + pair qualification.Pair shell string user string builtin bool @@ -63,7 +61,7 @@ type shellCell struct { agentPath string rootAgentDir bool loginShell string - report *cellReport + report *qualification.Cell } func newShellCell(shell string, builtin bool) *shellCell { @@ -102,25 +100,12 @@ func (c *shellCell) mustDevsy(timeout time.Duration, env []string, args ...strin return output } -// remote runs a POSIX script on the account through the provider under test. -// External SSH keeps helper commands independent of the transport being qualified. func (c *shellCell) remote(script string) (string, error) { - home, err := os.UserHomeDir() - framework.ExpectNoError(err) ctx, cancel := context.WithTimeout(context.Background(), remoteTimeout) defer cancel() // #nosec G204 -- The provider binary is the artifact under test. cmd := exec.CommandContext(ctx, providerPath, "command") - cmd.Env = append( - os.Environ(), - "HOST="+c.user+"@127.0.0.1", - "PORT=22", - "USE_BUILTIN_SSH=false", - "EXTRA_FLAGS="+shellquote.Join("-i", filepath.Join(home, ".ssh", "id_rsa")), - "AGENT_PATH="+c.agentPath, - "DOCKER_PATH=docker", - "COMMAND="+script, - ) + cmd.Env = providerCommandEnv(c.user, false, c.agentPath, script) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr if err := cmd.Run(); err != nil { @@ -157,8 +142,6 @@ func (c *shellCell) expectWorkspaceSSH() { }).WithTimeout(pollTimeout).WithPolling(pollInterval).Should(gomega.ContainSubstring(marker)) } -// hostBootstrapOutput excludes the separate injection that delivers the agent -// from the remote SSH host into the devcontainer. func hostBootstrapOutput(output string) string { end := strings.Index(output, logHostSSHServerStarted) gomega.Expect(end).To(gomega.BeNumerically(">", 0), "remote SSH server did not start") @@ -173,7 +156,6 @@ func (c *shellCell) agentSHA256() string { return strings.Fields(c.mustRemote("sha256sum < " + shellquote.Join(c.agentPath)))[0] } -// agentInode changes on every atomic replacement and survives a reused agent. func (c *shellCell) agentInode() string { return c.mustRemote(shellquote.Join("stat", "-c", "%i", "--", c.agentPath)) } @@ -187,7 +169,6 @@ func (c *shellCell) expectCandidateAgentInstalled() { } func (c *shellCell) installStaleAgent() { - // A prior agent process can still hold the installed executable open. staged := shellquote.Join(c.agentPath + ".stale") c.mustRemote(fmt.Sprintf( "mkdir -p -- %s && printf '#!/bin/sh\\necho %s\\n' > %s "+ @@ -245,7 +226,6 @@ func (c *shellCell) cleanup() { } func (c *shellCell) removeAgentDirectory() error { - // Only the per-cell directory is removed, never a shared agent path. if !strings.HasPrefix(c.agentDir, remoteAgentDirRoot) || len(c.agentDir) == len(remoteAgentDirRoot) { return nil @@ -257,44 +237,13 @@ func (c *shellCell) removeAgentDirectory() error { return err } -var _ = ginkgo.Describe( - "SSH login shell full workspace lifecycle", - ginkgo.Label("release-qualification"), - func() { - for _, cell := range shellTransports { - describeShellCell(cell.shell, cell.builtin) - } - }, -) - -func describeShellCell(shell string, builtin bool) { - ginkgo.Describe(shell+" builtin="+strconv.FormatBool(builtin), ginkgo.Ordered, func() { - cell := newShellCell(shell, builtin) - ginkgo.BeforeAll(cell.setup) - ginkgo.ReportAfterEach(cell.recordScenario) - ginkgo.It("installs a cold agent by remote download", cell.expectColdDownload) - ginkgo.It( - "reuses the current agent when the workspace is opened again", - cell.expectCurrentAgentReuse, - ) - ginkgo.It("replaces a deliberately old agent", cell.expectStaleAgentReplacement) - ginkgo.It( - "installs a cold agent by stdin transfer and keeps stdin for the live stream", - cell.expectStdinTransfer, - ) - ginkgo.It("leaves the account login shell unchanged", func() { - gomega.Expect(cell.passwdEntry()).To(gomega.Equal(cell.loginShell)) - }) - }) -} - func (c *shellCell) setup() { if !qualificationEnabled() { ginkgo.Skip(envQualification + " is not set to true") } prepareQualification() c.pair = requireQualifiedPair() - c.report = &cellReport{ + c.report = &qualification.Cell{ Shell: c.shell, User: c.user, BuiltinSSH: c.builtin, @@ -337,64 +286,8 @@ func (c *shellCell) recordScenario(report ginkgo.SpecReport) { report.State.Is(types.SpecStateSkipped|types.SpecStatePending) { return } - c.report.Scenarios = append(c.report.Scenarios, scenarioResult{ + c.report.Scenarios = append(c.report.Scenarios, qualification.Scenario{ Name: report.LeafNodeText, Outcome: report.State.String(), }) } - -func (c *shellCell) expectColdDownload() { - output := c.up() - hostOutput := hostBootstrapOutput(output) - gomega.Expect(hostOutput).To(gomega.ContainSubstring(logRemoteDownload)) - gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) - c.expectCandidateAgentInstalled() - c.report.AgentSHA256["download"] = c.agentSHA256() - c.expectWorkspaceSSH() -} - -func (c *shellCell) expectCurrentAgentReuse() { - inode := c.agentInode() - output := c.up() - hostOutput := hostBootstrapOutput(output) - gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) - gomega.Expect(c.agentInode()).To(gomega.Equal(inode)) - c.expectCandidateAgentInstalled() - c.expectWorkspaceSSH() -} - -func (c *shellCell) expectStaleAgentReplacement() { - c.installStaleAgent() - staleInode := c.agentInode() - output := c.up() - hostOutput := hostBootstrapOutput(output) - gomega.Expect(hostOutput).To(gomega.ContainSubstring(logRemoteDownload)) - gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) - gomega.Expect(c.agentInode()).NotTo(gomega.Equal(staleInode)) - c.expectCandidateAgentInstalled() - c.expectWorkspaceSSH() -} - -func (c *shellCell) expectStdinTransfer() { - c.mustRemote(shellquote.Join("rm", "-rf", "--", c.agentDir)) - stdinMode := []string{envAgentPreferDownload + "=false"} - output := c.up(stdinMode...) - hostOutput := hostBootstrapOutput(output) - gomega.Expect(hostOutput).To(gomega.ContainSubstring(logStdinTransfer)) - c.expectCandidateAgentInstalled() - c.report.AgentSHA256["stdin"] = c.agentSHA256() - c.expectWorkspaceSSH() - - inode := c.agentInode() - output = c.up(stdinMode...) - hostOutput = hostBootstrapOutput(output) - gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) - gomega.Expect(c.agentInode()).To(gomega.Equal(inode)) - c.expectWorkspaceSSH() -} - -func sshPrivateKeyPath() string { - home, err := os.UserHomeDir() - framework.ExpectNoError(err) - return filepath.Join(home, ".ssh", "id_rsa") -} diff --git a/e2e/tests/integration/compat_lifecycle.go b/e2e/tests/integration/compat_lifecycle.go new file mode 100644 index 0000000..2f2355e --- /dev/null +++ b/e2e/tests/integration/compat_lifecycle.go @@ -0,0 +1,155 @@ +package integration + +import ( + "bytes" + "os" + "os/exec" + "strings" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe( + "devsy provider ssh lifecycle suite (compatibility core "+compatCoreVersion+" unless overridden)", + ginkgo.Label(labelIntegration), + ginkgo.Ordered, + func() { + ginkgo.BeforeAll(func() { + setupProvider() + setupSSHKeys() + setupDevsyCLI() + }) + + ginkgo.It("should generate valid provider.yaml", func() { + data, err := os.ReadFile("../dist/provider.yaml") + framework.ExpectNoError(err) + framework.ExpectNotEqual(len(data), 0) + }) + + ginkgo.It("should have required provider options", func() { + data, err := os.ReadFile("../dist/provider.yaml") + framework.ExpectNoError(err) + + content := string(data) + framework.ExpectEqual(strings.Contains(content, "HOST"), true) + framework.ExpectEqual(strings.Contains(content, "AGENT_PATH"), true) + framework.ExpectEqual(strings.Contains(content, "DOCKER_PATH"), true) + }) + ginkgo.It("should install provider with devsy", func() { + cmd := exec.Command( + "bin/devsy", + "provider", + "add", + "../dist/provider.yaml", + "-o", + "HOST=127.0.0.1", + ) + err := cmd.Run() + framework.ExpectNoError(err) + }) + + ginkgo.It("should fail the init", func() { + cmd := exec.Command(providerPath, "init") + cmd.Env = localProviderEnv("1234", "ls") + err := cmd.Run() + framework.ExpectError(err) + }) + + ginkgo.It("should run the init", func() { + cmd := exec.Command(providerPath, "init") + cmd.Env = localProviderEnv("22", "ls") + err := cmd.Run() + framework.ExpectNoError(err) + }) + + ginkgo.It("should run a command", func() { + cmd := exec.Command(providerPath, "command") + cmd.Env = localProviderEnv("22", "ls") + err := cmd.Run() + framework.ExpectNoError(err) + }) + + ginkgo.It("should run a command and verify the output", func() { + cmd := exec.Command("ls", "/") + controlOutput, err := cmd.Output() + framework.ExpectNoError(err) + + cmd = exec.Command(providerPath, "command") + cmd.Env = localProviderEnv("22", "ls /") + output, err := cmd.Output() + framework.ExpectNoError(err) + gomega.Expect(output).To(gomega.Equal(controlOutput)) + }) + + ginkgo.It("should run a multiline command and verify the output", func() { + cmd := exec.Command("echo", `line1 +line2 +line3`) + controlOutput, err := cmd.Output() + framework.ExpectNoError(err) + + script := "echo line1\necho line2\necho line3" + ginkgo.GinkgoT().Setenv("COMMAND", script) + + cmd = exec.Command(providerPath, "command") + cmd.Env = localProviderEnv("22", script) + output, err := cmd.CombinedOutput() + framework.ExpectNoError(err) + gomega.Expect(output).To(gomega.Equal(controlOutput)) + }) + + ginkgo.It("should run a failing command and fail", func() { + cmd := exec.Command(providerPath, "command") + cmd.Env = localProviderEnv("22", "not-a-command") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + framework.ExpectError(err) + gomega.Expect(cmd.ProcessState).NotTo(gomega.BeNil()) + gomega.Expect(cmd.ProcessState.ExitCode()).To(gomega.Equal(127)) + gomega.Expect(stdout.Bytes()).To(gomega.BeEmpty()) + gomega.Expect(stderr.Bytes()).NotTo(gomega.BeEmpty()) + gomega.Expect(stderr.String()).To(gomega.ContainSubstring("not-a-command")) + }) + + ginkgo.It("should run devsy up", func() { + cmd := exec.Command("bin/devsy", "workspace", "up", "--debug", "--ide=none", "../") + err := cmd.Run() + framework.ExpectNoError(err) + }) + + ginkgo.It("should run commands to workspace via ssh", func() { + cmd := exec.Command("ssh", "devsy-provider-ssh.devsy", "echo", "test") + output, err := cmd.Output() + framework.ExpectNoError(err) + gomega.Expect(output).To(gomega.Equal([]byte("test\n"))) + }) + + ginkgo.It("should cleanup devsy workspace", func() { + cmd := exec.Command( + "bin/devsy", + "workspace", + "delete", + "--debug", + "--force", + "devsy-provider-ssh", + ) + err := cmd.Run() + framework.ExpectNoError(err) + }) + }, +) + +func localProviderEnv(port, command string) []string { + return append( + os.Environ(), + "AGENT_PATH=/tmp/devsy/agent", + "COMMAND="+command, + "DOCKER_PATH=docker", + "HOST=localhost", + "PORT="+port, + "USE_BUILTIN_SSH=false", + ) +} diff --git a/e2e/tests/integration/environment.go b/e2e/tests/integration/environment.go new file mode 100644 index 0000000..6f749bd --- /dev/null +++ b/e2e/tests/integration/environment.go @@ -0,0 +1,163 @@ +package integration + +import ( + "bytes" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/kballard/go-shellquote" + "github.com/onsi/ginkgo/v2" +) + +const providerPath = "../dist/build_linux_amd64_v1/devsy-provider-ssh-linux-amd64" + +const compatCoreVersion = "v1.24.0" + +func setupProvider() { + cmd := exec.Command("go", "run", "hack/provider/main.go", "0.0.0") + cmd.Dir = "../" + + projectRoot, err := filepath.Abs("../") + framework.ExpectNoError(err) + + distPath := filepath.Join(projectRoot, "dist") + cmd.Env = append(os.Environ(), "PROJECT_ROOT="+distPath) + output, err := cmd.Output() + framework.ExpectNoError(err) + + err = os.WriteFile("../dist/provider.yaml", output, 0o600) + framework.ExpectNoError(err) +} + +func setupSSHKeys() { + homeDir, err := os.UserHomeDir() + framework.ExpectNoError(err) + + sshDir := filepath.Join(homeDir, ".ssh") + err = os.MkdirAll(sshDir, 0o700) + framework.ExpectNoError(err) + + sshKeyPath := filepath.Join(sshDir, "id_rsa") + _, err = os.Stat(sshKeyPath) // #nosec G703 -- SSH key path is safely constructed + if err != nil && os.IsNotExist(err) { + ginkgo.GinkgoWriter.Println("generating ssh keys") + // #nosec G204,G702 -- SSH key path is safely constructed + cmd := exec.Command("ssh-keygen", "-q", "-t", "rsa", "-N", "", "-f", sshKeyPath) + err = cmd.Run() + framework.ExpectNoError(err) + } + framework.ExpectNoError(err) + + cmd := exec.Command("ssh-keygen", "-y", "-f", sshKeyPath) // #nosec G204,G702 + publicKey, err := cmd.Output() + framework.ExpectNoError(err) + + authorizedKeysPath := filepath.Join(homeDir, ".ssh", "authorized_keys") + existing, err := os.ReadFile(authorizedKeysPath) // #nosec G304,G703 + if err != nil && !os.IsNotExist(err) { + framework.ExpectNoError(err) + } + if !bytes.Contains(existing, bytes.TrimSpace(publicKey)) { + // #nosec G304,G703 -- authorized_keys path is safely constructed + f, err := os.OpenFile(authorizedKeysPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + framework.ExpectNoError(err) + + defer func() { _ = f.Close() }() + _, err = f.Write(publicKey) + framework.ExpectNoError(err) + } +} + +func sshPrivateKeyPath() string { + home, err := os.UserHomeDir() + framework.ExpectNoError(err) + return filepath.Join(home, ".ssh", "id_rsa") +} + +func installLoginAccountKey(user string) { + home, err := os.UserHomeDir() + framework.ExpectNoError(err) + sshDir := "/home/" + user + "/.ssh" + for _, args := range [][]string{ + {"install", "-d", "-m", "700", "-o", user, "-g", user, sshDir}, + { + "install", "-m", "600", "-o", user, "-g", user, + filepath.Join(home, ".ssh", "id_rsa.pub"), sshDir + "/authorized_keys", + }, + } { + // #nosec G204 -- Install only the controlled fixture key into provisioned test accounts. + output, err := exec.Command("sudo", args...).CombinedOutput() + framework.ExpectNoError(err, string(output)) + } +} + +func setupDevsyCLI() { + if qualificationEnabled() { + requireQualifiedPair() + } + + if local := os.Getenv(envCoreBinary); local != "" { + // #nosec G304,G703 -- Test runner explicitly selects the local core binary. + data, err := os.ReadFile(local) + framework.ExpectNoError(err) + framework.ExpectNoError(os.MkdirAll("bin", 0o750)) + // #nosec G306,G703 -- Local core binary must be executable. + framework.ExpectNoError(os.WriteFile("bin/devsy", data, 0o700)) + return + } + + downloadCompatCore() +} + +func downloadCompatCore() { + ginkgo.GinkgoWriter.Printf("compatibility run against core %s\n", compatCoreVersion) + client := &http.Client{Timeout: time.Second * 30} + resp, err := client.Get( + "https://github.com/devsy-org/devsy/releases/download/" + compatCoreVersion + + "/devsy-linux-amd64", + ) + framework.ExpectNoError(err) + framework.ExpectEqual(resp.StatusCode, http.StatusOK) + defer func() { _ = resp.Body.Close() }() + + binDir := "bin/" + err = os.MkdirAll(binDir, 0o755) // #nosec G301 -- bin directory is safely constructed + framework.ExpectNoError(err) + + binPath := filepath.Join(binDir, "devsy") + // #nosec G304,G302 -- path is safely constructed, needs execute permissions + out, err := os.OpenFile(binPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) + framework.ExpectNoError(err) + + _, err = io.Copy(out, resp.Body) + framework.ExpectNoError(err) + + err = out.Close() + framework.ExpectNoError(err) + + verifyCmd := exec.Command( + binPath, + "--version", + ) // #nosec G204,G702 -- path is safely constructed + err = verifyCmd.Run() + framework.ExpectNoError(err) +} + +func providerCommandEnv(user string, builtin bool, agentPath, script string) []string { + return append( + os.Environ(), + "HOST="+user+"@127.0.0.1", + "PORT=22", + "USE_BUILTIN_SSH="+strconv.FormatBool(builtin), + "EXTRA_FLAGS="+shellquote.Join("-i", sshPrivateKeyPath()), + "AGENT_PATH="+agentPath, + "DOCKER_PATH=docker", + "COMMAND="+script, + ) +} diff --git a/e2e/tests/integration/integration.go b/e2e/tests/integration/integration.go deleted file mode 100644 index a1a64e5..0000000 --- a/e2e/tests/integration/integration.go +++ /dev/null @@ -1,305 +0,0 @@ -package integration - -import ( - "bytes" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/devsy-org/devsy/e2e/framework" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -const providerPath = "../dist/build_linux_amd64_v1/devsy-provider-ssh-linux-amd64" - -func setupProvider() { - cmd := exec.Command("go", "run", "hack/provider/main.go", "0.0.0") - cmd.Dir = "../" - - projectRoot, err := filepath.Abs("../") - framework.ExpectNoError(err) - - distPath := filepath.Join(projectRoot, "dist") - cmd.Env = append(os.Environ(), "PROJECT_ROOT="+distPath) - output, err := cmd.Output() - framework.ExpectNoError(err) - - err = os.WriteFile("../dist/provider.yaml", output, 0o600) - framework.ExpectNoError(err) -} - -func setupSSHKeys() { - homeDir, err := os.UserHomeDir() - framework.ExpectNoError(err) - - sshDir := filepath.Join(homeDir, ".ssh") - err = os.MkdirAll(sshDir, 0o700) - framework.ExpectNoError(err) - - sshKeyPath := filepath.Join(sshDir, "id_rsa") - _, err = os.Stat(sshKeyPath) // #nosec G703 -- SSH key path is safely constructed - if err != nil && os.IsNotExist(err) { - ginkgo.GinkgoWriter.Println("generating ssh keys") - // #nosec G204,G702 -- SSH key path is safely constructed - cmd := exec.Command("ssh-keygen", "-q", "-t", "rsa", "-N", "", "-f", sshKeyPath) - err = cmd.Run() - framework.ExpectNoError(err) - } - framework.ExpectNoError(err) - - cmd := exec.Command("ssh-keygen", "-y", "-f", sshKeyPath) // #nosec G204,G702 - publicKey, err := cmd.Output() - framework.ExpectNoError(err) - - authorizedKeysPath := filepath.Join(homeDir, ".ssh", "authorized_keys") - existing, err := os.ReadFile(authorizedKeysPath) // #nosec G304,G703 - if err != nil && !os.IsNotExist(err) { - framework.ExpectNoError(err) - } - if !bytes.Contains(existing, bytes.TrimSpace(publicKey)) { - // #nosec G304,G703 -- authorized_keys path is safely constructed - f, err := os.OpenFile(authorizedKeysPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) - framework.ExpectNoError(err) - - defer func() { _ = f.Close() }() - _, err = f.Write(publicKey) - framework.ExpectNoError(err) - } -} - -// compatCoreVersion is the core release the lifecycle suite downloads when no -// binary override is supplied. It checks backward compatibility only. -const compatCoreVersion = "v1.24.0" - -func setupDevsyCLI() { - if qualificationEnabled() { - requireQualifiedPair() - } - - if local := os.Getenv(envCoreBinary); local != "" { - // #nosec G304,G703 -- Test runner explicitly selects the local core binary. - data, err := os.ReadFile(local) - framework.ExpectNoError(err) - framework.ExpectNoError(os.MkdirAll("bin", 0o750)) - // #nosec G306,G703 -- Local core binary must be executable. - framework.ExpectNoError(os.WriteFile("bin/devsy", data, 0o700)) - return - } - - downloadCompatCore() -} - -func downloadCompatCore() { - ginkgo.GinkgoWriter.Printf("compatibility run against core %s\n", compatCoreVersion) - client := &http.Client{Timeout: time.Second * 30} - resp, err := client.Get( - "https://github.com/devsy-org/devsy/releases/download/" + compatCoreVersion + - "/devsy-linux-amd64", - ) - framework.ExpectNoError(err) - framework.ExpectEqual(resp.StatusCode, http.StatusOK) - defer func() { _ = resp.Body.Close() }() - - binDir := "bin/" - err = os.MkdirAll(binDir, 0o755) // #nosec G301 -- bin directory is safely constructed - framework.ExpectNoError(err) - - binPath := filepath.Join(binDir, "devsy") - // #nosec G304,G302 -- path is safely constructed, needs execute permissions - out, err := os.OpenFile(binPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) - framework.ExpectNoError(err) - - _, err = io.Copy(out, resp.Body) - framework.ExpectNoError(err) - - err = out.Close() - framework.ExpectNoError(err) - - verifyCmd := exec.Command( - binPath, - "--version", - ) // #nosec G204,G702 -- path is safely constructed - err = verifyCmd.Run() - framework.ExpectNoError(err) -} - -var _ = ginkgo.Describe( - "devsy provider ssh lifecycle suite (compatibility core "+compatCoreVersion+" unless overridden)", - ginkgo.Label("integration"), - ginkgo.Ordered, - func() { - ginkgo.BeforeAll(func() { - setupProvider() - setupSSHKeys() - setupDevsyCLI() - }) - - ginkgo.It("should generate valid provider.yaml", func() { - data, err := os.ReadFile("../dist/provider.yaml") - framework.ExpectNoError(err) - framework.ExpectNotEqual(len(data), 0) - }) - - ginkgo.It("should have required provider options", func() { - data, err := os.ReadFile("../dist/provider.yaml") - framework.ExpectNoError(err) - - content := string(data) - framework.ExpectEqual(strings.Contains(content, "HOST"), true) - framework.ExpectEqual(strings.Contains(content, "AGENT_PATH"), true) - framework.ExpectEqual(strings.Contains(content, "DOCKER_PATH"), true) - }) - ginkgo.It("should install provider with devsy", func() { - cmd := exec.Command( - "bin/devsy", - "provider", - "add", - "../dist/provider.yaml", - "-o", - "HOST=127.0.0.1", - ) - err := cmd.Run() - framework.ExpectNoError(err) - }) - - ginkgo.It("should fail the init", func() { - cmd := exec.Command(providerPath, "init") - cmd.Env = append(cmd.Environ(), []string{ - "AGENT_PATH=/tmp/devsy/agent", - "COMMAND=ls", - "DOCKER_PATH=docker", - "HOST=localhost", - "PORT=1234", - "USE_BUILTIN_SSH=false", - }...) - err := cmd.Run() - framework.ExpectError(err) - }) - - ginkgo.It("should run the init", func() { - cmd := exec.Command(providerPath, "init") - cmd.Env = append(cmd.Environ(), []string{ - "AGENT_PATH=/tmp/devsy/agent", - "COMMAND=ls", - "DOCKER_PATH=docker", - "HOST=localhost", - "PORT=22", - "USE_BUILTIN_SSH=false", - }...) - err := cmd.Run() - framework.ExpectNoError(err) - }) - - ginkgo.It("should run a command", func() { - cmd := exec.Command(providerPath, "command") - cmd.Env = append(cmd.Environ(), []string{ - "AGENT_PATH=/tmp/devsy/agent", - "COMMAND=ls", - "DOCKER_PATH=docker", - "HOST=localhost", - "PORT=22", - "USE_BUILTIN_SSH=false", - }...) - err := cmd.Run() - framework.ExpectNoError(err) - }) - - ginkgo.It("should run a command and verify the output", func() { - cmd := exec.Command("ls", "/") - controlOutput, err := cmd.Output() - framework.ExpectNoError(err) - - cmd = exec.Command(providerPath, "command") - cmd.Env = append(cmd.Environ(), []string{ - "AGENT_PATH=/tmp/devsy/agent", - "COMMAND=ls /", - "DOCKER_PATH=docker", - "HOST=localhost", - "PORT=22", - "USE_BUILTIN_SSH=false", - }...) - output, err := cmd.Output() - framework.ExpectNoError(err) - gomega.Expect(output).To(gomega.Equal(controlOutput)) - }) - - ginkgo.It("should run a multiline command and verify the output", func() { - cmd := exec.Command("echo", `line1 -line2 -line3`) - controlOutput, err := cmd.Output() - framework.ExpectNoError(err) - - ginkgo.GinkgoT().Setenv("COMMAND", `echo line1 -echo line2 -echo line3`) - - cmd = exec.Command(providerPath, "command") - cmd.Env = append(cmd.Environ(), []string{ - "AGENT_PATH=/tmp/devsy/agent", - "DOCKER_PATH=docker", - "COMMAND=" + `echo line1 -echo line2 -echo line3`, - "HOST=localhost", - "PORT=22", - "USE_BUILTIN_SSH=false", - }...) - output, err := cmd.CombinedOutput() - framework.ExpectNoError(err) - gomega.Expect(output).To(gomega.Equal(controlOutput)) - }) - - ginkgo.It("should run a failing command and fail", func() { - cmd := exec.Command(providerPath, "command") - cmd.Env = append(cmd.Environ(), []string{ - "AGENT_PATH=/tmp/devsy/agent", - "COMMAND=not-a-command", - "DOCKER_PATH=docker", - "HOST=localhost", - "PORT=22", - "USE_BUILTIN_SSH=false", - }...) - var stdout, stderr bytes.Buffer - cmd.Stdout, cmd.Stderr = &stdout, &stderr - err := cmd.Run() - framework.ExpectError(err) - gomega.Expect(cmd.ProcessState).NotTo(gomega.BeNil()) - gomega.Expect(cmd.ProcessState.ExitCode()).To(gomega.Equal(127)) - gomega.Expect(stdout.Bytes()).To(gomega.BeEmpty()) - gomega.Expect(stderr.Bytes()).NotTo(gomega.BeEmpty()) - gomega.Expect(stderr.String()).To(gomega.ContainSubstring("not-a-command")) - }) - - ginkgo.It("should run devsy up", func() { - cmd := exec.Command("bin/devsy", "workspace", "up", "--debug", "--ide=none", "../") - err := cmd.Run() - framework.ExpectNoError(err) - }) - - ginkgo.It("should run commands to workspace via ssh", func() { - cmd := exec.Command("ssh", "devsy-provider-ssh.devsy", "echo", "test") - output, err := cmd.Output() - framework.ExpectNoError(err) - gomega.Expect(output).To(gomega.Equal([]byte("test\n"))) - }) - - ginkgo.It("should cleanup devsy workspace", func() { - cmd := exec.Command( - "bin/devsy", - "workspace", - "delete", - "--debug", - "--force", - "devsy-provider-ssh", - ) - err := cmd.Run() - framework.ExpectNoError(err) - }) - }, -) diff --git a/e2e/tests/integration/labels.go b/e2e/tests/integration/labels.go new file mode 100644 index 0000000..15ac267 --- /dev/null +++ b/e2e/tests/integration/labels.go @@ -0,0 +1,7 @@ +package integration + +const ( + labelIntegration = "integration" + labelProviderCommand = "provider-command" + labelReleaseQualification = "release-qualification" +) diff --git a/e2e/tests/integration/lifecycle.go b/e2e/tests/integration/lifecycle.go new file mode 100644 index 0000000..6022d7e --- /dev/null +++ b/e2e/tests/integration/lifecycle.go @@ -0,0 +1,87 @@ +package integration + +import ( + "strconv" + + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" + "github.com/kballard/go-shellquote" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe( + "SSH login shell full workspace lifecycle", + ginkgo.Label(labelReleaseQualification), + func() { + for _, cell := range qualification.Transports { + describeShellCell(cell.Shell, cell.Builtin) + } + }, +) + +func describeShellCell(shell string, builtin bool) { + ginkgo.Describe(shell+" builtin="+strconv.FormatBool(builtin), ginkgo.Ordered, func() { + cell := newShellCell(shell, builtin) + ginkgo.BeforeAll(cell.setup) + ginkgo.ReportAfterEach(cell.recordScenario) + ginkgo.It(qualification.ScenarioColdDownload, cell.expectColdDownload) + ginkgo.It(qualification.ScenarioCurrentReuse, cell.expectCurrentAgentReuse) + ginkgo.It(qualification.ScenarioStaleReplacement, cell.expectStaleAgentReplacement) + ginkgo.It(qualification.ScenarioStdinTransfer, cell.expectStdinTransfer) + ginkgo.It(qualification.ScenarioLoginShellUnchanged, cell.expectLoginShellUnchanged) + }) +} + +func (c *shellCell) expectColdDownload() { + output := c.up() + hostOutput := hostBootstrapOutput(output) + gomega.Expect(hostOutput).To(gomega.ContainSubstring(logRemoteDownload)) + gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) + c.expectCandidateAgentInstalled() + c.report.AgentSHA256["download"] = c.agentSHA256() + c.expectWorkspaceSSH() +} + +func (c *shellCell) expectCurrentAgentReuse() { + inode := c.agentInode() + output := c.up() + hostOutput := hostBootstrapOutput(output) + gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) + gomega.Expect(c.agentInode()).To(gomega.Equal(inode)) + c.expectCandidateAgentInstalled() + c.expectWorkspaceSSH() +} + +func (c *shellCell) expectStaleAgentReplacement() { + c.installStaleAgent() + staleInode := c.agentInode() + output := c.up() + hostOutput := hostBootstrapOutput(output) + gomega.Expect(hostOutput).To(gomega.ContainSubstring(logRemoteDownload)) + gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) + gomega.Expect(c.agentInode()).NotTo(gomega.Equal(staleInode)) + c.expectCandidateAgentInstalled() + c.expectWorkspaceSSH() +} + +func (c *shellCell) expectStdinTransfer() { + c.mustRemote(shellquote.Join("rm", "-rf", "--", c.agentDir)) + stdinMode := []string{envAgentPreferDownload + "=false"} + output := c.up(stdinMode...) + hostOutput := hostBootstrapOutput(output) + gomega.Expect(hostOutput).To(gomega.ContainSubstring(logStdinTransfer)) + c.expectCandidateAgentInstalled() + c.report.AgentSHA256["stdin"] = c.agentSHA256() + c.expectWorkspaceSSH() + + inode := c.agentInode() + output = c.up(stdinMode...) + hostOutput = hostBootstrapOutput(output) + gomega.Expect(hostOutput).NotTo(gomega.ContainSubstring(logStdinTransfer)) + gomega.Expect(c.agentInode()).To(gomega.Equal(inode)) + c.expectWorkspaceSSH() +} + +func (c *shellCell) expectLoginShellUnchanged() { + gomega.Expect(c.passwdEntry()).To(gomega.Equal(c.loginShell)) +} diff --git a/e2e/tests/integration/login_shells.go b/e2e/tests/integration/login_shells.go deleted file mode 100644 index 4c7a602..0000000 --- a/e2e/tests/integration/login_shells.go +++ /dev/null @@ -1,80 +0,0 @@ -package integration - -import ( - "bytes" - "fmt" - "os" - "os/exec" - "path/filepath" - "strconv" - - "github.com/devsy-org/devsy/e2e/framework" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -var _ = ginkgo.Describe( - "SSH login shells", - ginkgo.Label("integration", "provider-command"), - func() { - ginkgo.BeforeEach(func() { - if os.Getenv("DEVSY_E2E_SHELL_MATRIX") != "true" { - ginkgo.Skip("requires provisioned Linux login shell accounts") - } - setupSSHKeys() - }) - for _, cell := range shellTransports { - ginkgo.It( - fmt.Sprintf( - "preserves bootstrap streams with %s builtin=%t", - cell.shell, - cell.builtin, - ), - func() { - home, err := os.UserHomeDir() - framework.ExpectNoError(err) - user := "devsy-" + cell.shell - installLoginAccountKey(user) - payload := bytes.Repeat([]byte{0, 1, 2, 255, '\n', '\'', '\\'}, 512) - cmd := exec.Command(providerPath, "command") - cmd.Env = append( - os.Environ(), - "HOST="+user+"@127.0.0.1", - "PORT=22", - "USE_BUILTIN_SSH="+strconv.FormatBool(cell.builtin), - "EXTRA_FLAGS=-i "+filepath.Join(home, ".ssh", "id_rsa"), - "AGENT_PATH=/tmp/devsy/agent", - "DOCKER_PATH=docker", - "COMMAND=printf 'READY\\n'\ncat\nprintf 'diagnostic' >&2\nexit 37", - ) - cmd.Stdin = bytes.NewReader(payload) - var stdout, stderr bytes.Buffer - cmd.Stdout, cmd.Stderr = &stdout, &stderr - err = cmd.Run() - gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(cmd.ProcessState.ExitCode()).To(gomega.Equal(37)) - gomega.Expect(stdout.Bytes()). - To(gomega.Equal(append([]byte("READY\n"), payload...))) - gomega.Expect(stderr.String()).To(gomega.ContainSubstring("diagnostic")) - }, - ) - } - }, -) - -func installLoginAccountKey(user string) { - home, err := os.UserHomeDir() - framework.ExpectNoError(err) - sshDir := "/home/" + user + "/.ssh" - for _, args := range [][]string{ - {"install", "-d", "-m", "700", "-o", user, "-g", user, sshDir}, - { - "install", "-m", "600", "-o", user, "-g", user, - filepath.Join(home, ".ssh", "id_rsa.pub"), sshDir + "/authorized_keys", - }, - } { - // #nosec G204 -- Install only the controlled fixture key into provisioned test accounts. - output, err := exec.Command("sudo", args...).CombinedOutput() - framework.ExpectNoError(err, string(output)) - } -} diff --git a/e2e/tests/integration/qualification.go b/e2e/tests/integration/pair.go similarity index 63% rename from e2e/tests/integration/qualification.go rename to e2e/tests/integration/pair.go index 23a779e..5cdbecb 100644 --- a/e2e/tests/integration/qualification.go +++ b/e2e/tests/integration/pair.go @@ -12,6 +12,7 @@ import ( "strings" "sync" + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" "github.com/devsy-org/devsy/e2e/framework" "github.com/onsi/ginkgo/v2" ) @@ -28,8 +29,6 @@ const ( defaultReportPath = "qualification-report.json" - // Any of these changes how the core selects the agent transport or disables the - // remote version check, so a qualification run must not inherit them. envAgentURL = "DEVSY_AGENT_URL" envAgentBinary = "DEVSY_AGENT_BINARY" envAgentPreferDownload = "DEVSY_AGENT_PREFER_DOWNLOAD" @@ -37,85 +36,17 @@ const ( devVersion = "v0.0.0" ) -// qualifiedPair identifies the exact core artifact and provider build under test. -type qualifiedPair struct { - CoreBinary string `json:"coreBinary"` - CoreTag string `json:"coreTag"` - CoreSHA string `json:"coreSha"` - CoreVersion string `json:"coreVersion"` - CoreBinarySHA256 string `json:"coreBinarySha256"` - ProviderRef string `json:"providerRef"` - ProviderSHA string `json:"providerSha"` - ProviderBinary string `json:"providerBinary"` - ProviderBinarySHA256 string `json:"providerBinarySha256"` - HarnessSHA string `json:"harnessSha"` -} - -type scenarioResult struct { - Name string `json:"name"` - Outcome string `json:"outcome"` -} - -type shellTransport struct { - shell string - builtin bool -} - -var shellTransports = []shellTransport{ - {shell: "bash"}, - {shell: "bash", builtin: true}, - {shell: "zsh"}, - {shell: "zsh", builtin: true}, - {shell: "fish"}, - {shell: "fish", builtin: true}, -} - -type cellReport struct { - Shell string `json:"shell"` - User string `json:"user"` - BuiltinSSH bool `json:"builtinSsh"` - Workspace string `json:"workspace"` - AgentDir string `json:"agentDir"` - LoginShell string `json:"loginShell"` - Scenarios []scenarioResult `json:"scenarios"` - AgentSHA256 map[string]string `json:"agentSha256"` - CleanupStatus string `json:"cleanup"` -} - -type privilegeReport struct { - User string `json:"user"` - UserUID uint64 `json:"userUid"` - AgentDir string `json:"agentDir"` - DirectoryState string `json:"directoryState"` - WriteDenied bool `json:"writeDenied"` - WriteError string `json:"writeError"` - AgentUID uint64 `json:"agentUid"` - AgentVersion string `json:"agentVersion"` - AgentSHA256 string `json:"agentSha256"` - LiveCommand bool `json:"liveCommand"` - Outcome string `json:"outcome"` - CleanupStatus string `json:"cleanup"` -} - -type qualificationReport struct { - Pair qualifiedPair `json:"pair"` - Cells []*cellReport `json:"cells"` - Privilege *privilegeReport `json:"privilege"` -} - var ( qualificationMu sync.Mutex - qualificationState *qualifiedPair - currentReport = &qualificationReport{} + qualificationState *qualification.Pair + currentReport = &qualification.Report{SchemaVersion: qualification.SchemaVersion} ) func qualificationEnabled() bool { return os.Getenv(envQualification) == "true" } -// requireQualifiedPair fails instead of skipping: a release row that silently -// falls back to a downloaded core would qualify the wrong pair. -func requireQualifiedPair() qualifiedPair { +func requireQualifiedPair() qualification.Pair { qualificationMu.Lock() defer qualificationMu.Unlock() if qualificationState != nil { @@ -124,7 +55,7 @@ func requireQualifiedPair() qualifiedPair { requireQualificationEnvironment() - pair := qualifiedPair{ + pair := qualification.Pair{ CoreBinary: requiredEnv(envCoreBinary, "the candidate core CLI binary"), CoreVersion: requiredEnv(envCoreVersion, "the expected `devsy --version` output"), CoreTag: requiredEnv(envCoreTag, "the core release tag under test"), @@ -206,13 +137,13 @@ func gitOutput(args ...string) string { return strings.TrimSpace(string(output)) } -func registerCellReport(report *cellReport) { +func registerCellReport(report *qualification.Cell) { qualificationMu.Lock() defer qualificationMu.Unlock() currentReport.Cells = append(currentReport.Cells, report) } -func registerPrivilegeReport(report *privilegeReport) { +func registerPrivilegeReport(report *qualification.Privilege) { qualificationMu.Lock() defer qualificationMu.Unlock() currentReport.Privilege = report diff --git a/e2e/tests/integration/login_shells_sudo.go b/e2e/tests/integration/privileged_install.go similarity index 87% rename from e2e/tests/integration/login_shells_sudo.go rename to e2e/tests/integration/privileged_install.go index 1f0cfac..eb09d09 100644 --- a/e2e/tests/integration/login_shells_sudo.go +++ b/e2e/tests/integration/privileged_install.go @@ -6,6 +6,7 @@ import ( "os/exec" "strconv" + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" "github.com/devsy-org/devsy/e2e/framework" "github.com/kballard/go-shellquote" "github.com/onsi/ginkgo/v2" @@ -14,7 +15,7 @@ import ( var _ = ginkgo.Describe( "SSH installer privilege transition", - ginkgo.Label("release-qualification"), + ginkgo.Label(labelReleaseQualification), func() { ginkgo.It( "installs an executable agent through real sudo for a non-root user", @@ -29,13 +30,12 @@ func qualifySudoInstallation() { } prepareQualification() cell := newShellCell("bash", false) - cell.user = "devsy-sudo" + cell.user = qualification.PrivilegeUser cell.rootAgentDir = true - result := &privilegeReport{User: cell.user, AgentDir: cell.agentDir, Outcome: "failed"} + result := &qualification.Privilege{User: cell.user, AgentDir: cell.agentDir, Outcome: "failed"} registerPrivilegeReport(result) ginkgo.DeferCleanup(func() { cell.finishPrivilegeReport(result) }) ginkgo.DeferCleanup(func() { - // Root directory creation can fail before setup registers its cleanup. if cell.report == nil { framework.ExpectNoError(cell.removeAgentDirectory()) } @@ -62,7 +62,7 @@ func qualifySudoInstallation() { result.Outcome = "passed" } -func (c *shellCell) requireSudoPreconditions(result *privilegeReport) { +func (c *shellCell) requireSudoPreconditions(result *qualification.Privilege) { var err error result.UserUID, err = strconv.ParseUint(c.mustRemote("id -u"), 10, 32) framework.ExpectNoError(err) @@ -78,7 +78,7 @@ func (c *shellCell) requireSudoPreconditions(result *privilegeReport) { result.WriteDenied = true } -func (c *shellCell) finishPrivilegeReport(result *privilegeReport) { +func (c *shellCell) finishPrivilegeReport(result *qualification.Privilege) { if c.report != nil { result.CleanupStatus = c.report.CleanupStatus } diff --git a/e2e/tests/integration/provider_command.go b/e2e/tests/integration/provider_command.go new file mode 100644 index 0000000..98f20c5 --- /dev/null +++ b/e2e/tests/integration/provider_command.go @@ -0,0 +1,59 @@ +package integration + +import ( + "bytes" + "fmt" + "os" + "os/exec" + + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe( + "SSH login shells", + ginkgo.Label(labelIntegration, labelProviderCommand), + func() { + ginkgo.BeforeEach(func() { + if os.Getenv("DEVSY_E2E_SHELL_MATRIX") != "true" { + ginkgo.Skip("requires provisioned Linux login shell accounts") + } + setupSSHKeys() + }) + for _, transport := range qualification.Transports { + ginkgo.It( + fmt.Sprintf( + "preserves bootstrap streams with %s builtin=%t", + transport.Shell, + transport.Builtin, + ), + func() { + expectBootstrapStreams(transport.Shell, transport.Builtin) + }, + ) + } + }, +) + +func expectBootstrapStreams(shell string, builtin bool) { + user := "devsy-" + shell + installLoginAccountKey(user) + payload := bytes.Repeat([]byte{0, 1, 2, 255, '\n', '\'', '\\'}, 512) + cmd := exec.Command(providerPath, "command") + cmd.Env = providerCommandEnv( + user, + builtin, + "/tmp/devsy/agent", + "printf 'READY\\n'\ncat\nprintf 'diagnostic' >&2\nexit 37", + ) + cmd.Stdin = bytes.NewReader(payload) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(cmd.ProcessState.ExitCode()).To(gomega.Equal(37)) + gomega.Expect(stdout.Bytes()). + To(gomega.Equal(append([]byte("READY\n"), payload...))) + gomega.Expect(stderr.String()).To(gomega.ContainSubstring("diagnostic")) +} diff --git a/hack/ci/dispatcher_test.go b/hack/ci/dispatcher_test.go new file mode 100644 index 0000000..316571e --- /dev/null +++ b/hack/ci/dispatcher_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "errors" + "strings" + "testing" +) + +func withCommands(t *testing.T, replacement map[string]func() error) { + t.Helper() + original := commands + commands = replacement + t.Cleanup(func() { commands = original }) +} + +func TestRunRejectsMissingOrExtraArguments(t *testing.T) { + withCommands(t, map[string]func() error{"only": func() error { return nil }}) + for _, args := range [][]string{nil, {}, {"only", "extra"}} { + var stderr bytes.Buffer + if code := run(args, &stderr); code != 2 { + t.Fatalf("run(%v) = %d, want 2", args, code) + } + if !strings.Contains(stderr.String(), "usage: go run ./hack/ci ") { + t.Fatalf("usage message missing command list: %q", stderr.String()) + } + } +} + +func TestRunRejectsUnknownCommand(t *testing.T) { + withCommands(t, map[string]func() error{ + "alpha": func() error { return nil }, + "beta": func() error { return nil }, + }) + var stderr bytes.Buffer + if code := run([]string{"gamma"}, &stderr); code != 2 { + t.Fatalf("run unknown = %d, want 2", code) + } + if !strings.Contains(stderr.String(), `unknown command "gamma"`) || + !strings.Contains(stderr.String(), "alpha|beta") { + t.Fatalf("unexpected message: %q", stderr.String()) + } +} + +func TestRunMapsHandlerResultToExitCode(t *testing.T) { + withCommands(t, map[string]func() error{ + "passes": func() error { return nil }, + "fails": func() error { return errors.New("boom") }, + }) + var stderr bytes.Buffer + if code := run([]string{"passes"}, &stderr); code != 0 || stderr.Len() != 0 { + t.Fatalf("passes = %d with stderr %q", code, stderr.String()) + } + if code := run([]string{"fails"}, &stderr); code != 1 || + !strings.Contains(stderr.String(), "boom") { + t.Fatalf("fails = %d with stderr %q", code, stderr.String()) + } +} + +func TestEveryCommandIsRegisteredWithAHandler(t *testing.T) { + for name, handler := range commands { + if handler == nil { + t.Fatalf("command %q has no handler", name) + } + } +} diff --git a/hack/ci/exec.go b/hack/ci/exec.go new file mode 100644 index 0000000..9594be8 --- /dev/null +++ b/hack/ci/exec.go @@ -0,0 +1,89 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +var allowedTools = map[string]struct{}{ + "dpkg-deb": {}, + "gh": {}, + "git": {}, + "install": {}, + "rm": {}, + "service": {}, + "sha256sum": {}, + "sudo": {}, + "test": {}, + "useradd": {}, + "usermod": {}, + "visudo": {}, +} + +func command(name string, args ...string) (string, error) { + if _, ok := allowedTools[name]; !ok { + return "", fmt.Errorf("unsupported command %q", name) + } + cmd := exec.Command(name, args...) + cmd.Stderr = os.Stderr + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) + } + return string(output), nil +} + +func requireEnv(names ...string) (map[string]string, error) { + values := make(map[string]string, len(names)) + var missing []string + for _, name := range names { + value := os.Getenv(name) + if value == "" { + missing = append(missing, name) + } + values[name] = value + } + if len(missing) > 0 { + return nil, fmt.Errorf("missing required environment: %s", strings.Join(missing, ", ")) + } + return values, nil +} + +func appendLines(path string, lines []string) error { + if path == "" { + return fmt.Errorf("output path is required") + } + directory, name := filepath.Split(filepath.Clean(path)) + root, err := os.OpenRoot(directory) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + file, err := root.OpenFile(name, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + for _, line := range lines { + if _, err := fmt.Fprintln(file, line); err != nil { + return err + } + } + return nil +} + +func parseSHA256Output(output string) (string, error) { + fields := strings.Fields(output) + if len(fields) < 2 || len(fields[0]) != sha256.Size*2 { + return "", fmt.Errorf("invalid sha256sum output %q", output) + } + if _, err := hex.DecodeString(fields[0]); err != nil { + return "", fmt.Errorf("invalid sha256sum output %q", output) + } + return fields[0], nil +} diff --git a/hack/ci/guard.go b/hack/ci/guard.go new file mode 100644 index 0000000..8249ce7 --- /dev/null +++ b/hack/ci/guard.go @@ -0,0 +1,93 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" +) + +const ( + guardModuleDir = "e2e" + guardLogName = "missing-binary-guard.log" + guardMessage = "DEVSY_E2E_BINARY is required" + envCoreBinary = "DEVSY_E2E_BINARY" + envReleaseQualified = "DEVSY_E2E_RELEASE_QUALIFICATION" +) + +func requireExplicitCandidate() error { + env, err := requireEnv("QUALIFICATION_DIR") + if err != nil { + return err + } + output, exitCode, err := runMissingBinaryGuard() + if err != nil { + return err + } + logPath := filepath.Join(env["QUALIFICATION_DIR"], guardLogName) + if err := os.WriteFile(logPath, output, 0o600); err != nil { + return err + } + if _, err := os.Stdout.Write(output); err != nil { + return err + } + return evaluateGuardResult(exitCode, string(output)) +} + +func runMissingBinaryGuard() ([]byte, int, error) { + cmd := exec.Command( + "go", + "test", + "-v", + "-timeout", + "300s", + "--ginkgo.v", + "--ginkgo.label-filter=release-qualification", + "--ginkgo.focus="+missingBinaryFocus(), + ) + cmd.Dir = guardModuleDir + cmd.Env = guardEnvironment() + output, err := cmd.CombinedOutput() + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + return output, exitErr.ExitCode(), nil + } + if err != nil { + return nil, 0, fmt.Errorf("run missing binary guard: %w", err) + } + return output, 0, nil +} + +func missingBinaryFocus() string { + transport := qualification.Transports[0] + return fmt.Sprintf( + "%s builtin=%t %s", + transport.Shell, + transport.Builtin, + qualification.ScenarioColdDownload, + ) +} + +func guardEnvironment() []string { + environ := os.Environ() + env := make([]string, 0, len(environ)+1) + for _, entry := range environ { + if !strings.HasPrefix(entry, envCoreBinary+"=") { + env = append(env, entry) + } + } + return append(env, envReleaseQualified+"=true") +} + +func evaluateGuardResult(exitCode int, output string) error { + if exitCode != 1 { + return fmt.Errorf("missing binary guard exited with %d, expected 1", exitCode) + } + if !strings.Contains(output, guardMessage) { + return fmt.Errorf("missing binary guard output does not contain %q", guardMessage) + } + return nil +} diff --git a/hack/ci/main.go b/hack/ci/main.go index c39e4c8..4b44cf8 100644 --- a/hack/ci/main.go +++ b/hack/ci/main.go @@ -2,16 +2,53 @@ package main import ( "fmt" + "io" "os" + "sort" + "strings" ) +var commands = map[string]func() error{ + "cleanup-qualification-accounts": cleanupQualificationAccounts, + "missing-candidate-guard": requireExplicitCandidate, + "provision-accounts": provisionAccounts, + "provision-qualification-accounts": provisionQualificationAccounts, + "qualify-core": qualifyCore, + "verify-provider": verifyProvider, + "verify-report": verifyReport, +} + func main() { - if len(os.Args) != 2 || os.Args[1] != "qualify-core" { - fmt.Fprintln(os.Stderr, "usage: go run ./hack/ci qualify-core") - os.Exit(2) + os.Exit(run(os.Args[1:], os.Stderr)) +} + +func run(args []string, stderr io.Writer) int { + if len(args) != 1 { + _, _ = fmt.Fprintf(stderr, "usage: go run ./hack/ci <%s>\n", commandNames()) + return 2 } - if err := qualifyCore(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + handler, ok := commands[args[0]] + if !ok { + _, _ = fmt.Fprintf( + stderr, + "unknown command %q; expected one of %s\n", + args[0], + commandNames(), + ) + return 2 + } + if err := handler(); err != nil { + _, _ = fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +func commandNames() string { + names := make([]string, 0, len(commands)) + for name := range commands { + names = append(names, name) } + sort.Strings(names) + return strings.Join(names, "|") } diff --git a/hack/ci/provision.go b/hack/ci/provision.go new file mode 100644 index 0000000..e2def27 --- /dev/null +++ b/hack/ci/provision.go @@ -0,0 +1,180 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" +) + +const ( + sudoersPolicyPath = "/etc/sudoers.d/devsy-e2e-qualification" + policyEnv = "DEVSY_SUDO_FIXTURE_POLICY" + qualificationGroup = "docker" + privilegeShell = "bash" +) + +type provisioner struct { + run func(name string, args ...string) (string, error) + lookPath func(file string) (string, error) + appendEnv func(lines []string) error + tempDir string +} + +func newProvisioner() *provisioner { + return &provisioner{ + run: command, + lookPath: exec.LookPath, + appendEnv: appendGitHubEnv, + tempDir: os.TempDir(), + } +} + +func appendGitHubEnv(lines []string) error { + return appendLines(os.Getenv("GITHUB_ENV"), lines) +} + +func provisionAccounts() error { + return newProvisioner().provisionLogins() +} + +func provisionQualificationAccounts() error { + return newProvisioner().provisionQualification() +} + +func cleanupQualificationAccounts() error { + return newProvisioner().removeSudoersPolicy() +} + +func (p *provisioner) provisionLogins() error { + if err := p.startSSH(); err != nil { + return err + } + return p.createLoginAccounts() +} + +func (p *provisioner) provisionQualification() error { + if err := p.startSSH(); err != nil { + return err + } + if err := p.createLoginAccounts(qualificationGroup); err != nil { + return err + } + if err := p.createSudoAccount(); err != nil { + return err + } + return p.installSudoersPolicy() +} + +func (p *provisioner) sudo(args ...string) error { + _, err := p.run("sudo", args...) + return err +} + +func (p *provisioner) startSSH() error { + if err := p.sudo("service", "ssh", "start"); err != nil { + return fmt.Errorf("start ssh service: %w", err) + } + return nil +} + +func (p *provisioner) createLoginAccounts(groups ...string) error { + for _, shell := range qualification.Shells() { + account := "devsy-" + shell + if err := p.useradd(account, shell); err != nil { + return err + } + if len(groups) == 0 { + continue + } + if err := p.addGroups(account, groups); err != nil { + return err + } + } + return nil +} + +func (p *provisioner) createSudoAccount() error { + return p.useradd(qualification.PrivilegeUser, privilegeShell, "--groups", qualificationGroup) +} + +func (p *provisioner) useradd(account, shell string, extra ...string) error { + path, err := p.lookPath(shell) + if err != nil { + return fmt.Errorf("resolve shell %s: %w", shell, err) + } + args := append([]string{"useradd", "--create-home", "--shell", path}, extra...) + if err := p.sudo(append(args, account)...); err != nil { + return fmt.Errorf("create account %s: %w", account, err) + } + return nil +} + +func (p *provisioner) addGroups(account string, groups []string) error { + list := strings.Join(groups, ",") + if err := p.sudo("usermod", "--append", "--groups", list, account); err != nil { + return fmt.Errorf("add %s to groups %s: %w", account, list, err) + } + return nil +} + +func (p *provisioner) installSudoersPolicy() error { + if err := p.sudo("test", "!", "-e", sudoersPolicyPath); err != nil { + return fmt.Errorf("sudoers policy %s must not exist: %w", sudoersPolicyPath, err) + } + ruleFile, err := p.writeSudoersRule() + if err != nil { + return err + } + defer func() { _ = os.Remove(ruleFile) }() + if err := p.sudo("visudo", "-cf", ruleFile); err != nil { + return fmt.Errorf("validate sudoers rule: %w", err) + } + if err := p.appendEnv([]string{policyEnv + "=" + sudoersPolicyPath}); err != nil { + return fmt.Errorf("export %s: %w", policyEnv, err) + } + install := []string{"install", "-m", "0440", "-o", "root", "-g", "root"} + if err := p.sudo(append(install, ruleFile, sudoersPolicyPath)...); err != nil { + return fmt.Errorf("install sudoers policy: %w", err) + } + if err := p.sudo("visudo", "-cf", sudoersPolicyPath); err != nil { + return fmt.Errorf("validate installed sudoers policy: %w", err) + } + return nil +} + +func (p *provisioner) writeSudoersRule() (string, error) { + file, err := os.CreateTemp(p.tempDir, "devsy-sudoers-") + if err != nil { + return "", fmt.Errorf("create sudoers temp file: %w", err) + } + path := file.Name() + _, writeErr := file.WriteString(sudoersRule() + "\n") + if err := errors.Join(writeErr, file.Close()); err != nil { + _ = os.Remove(path) + return "", fmt.Errorf("write sudoers rule: %w", err) + } + return path, nil +} + +func (p *provisioner) removeSudoersPolicy() error { + policy := os.Getenv(policyEnv) + if policy == "" { + return nil + } + if policy != sudoersPolicyPath { + return fmt.Errorf("refusing to remove %s: only %s is allowed", policy, sudoersPolicyPath) + } + if err := p.sudo("rm", "-f", "--", policy); err != nil { + return fmt.Errorf("remove sudoers policy: %w", err) + } + return nil +} + +func sudoersRule() string { + return qualification.PrivilegeUser + + " ALL=(root) NOPASSWD: SETENV: /bin/sh -c * devsy-install *" +} diff --git a/hack/ci/provision_test.go b/hack/ci/provision_test.go new file mode 100644 index 0000000..5b94ca3 --- /dev/null +++ b/hack/ci/provision_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "fmt" + "os" + "regexp" + "slices" + "strings" + "testing" +) + +const ( + startSSHCall = "sudo service ssh start" + testPolicyLine = "devsy-sudo ALL=(root) NOPASSWD: SETENV: /bin/sh -c * devsy-install *\n" + testTempCall = "/devsy-sudoers-N" +) + +var tempRuleName = regexp.MustCompile(`[^\s]*/devsy-sudoers-[^\s]+`) + +type recorder struct { + calls []string + failOn string + rule string + env []string +} + +func (r *recorder) run(name string, args ...string) (string, error) { + raw := strings.Join(append([]string{name}, args...), " ") + call := tempRuleName.ReplaceAllString(raw, testTempCall) + r.calls = append(r.calls, call) + if isTempValidation(name, args) { + content, err := os.ReadFile(args[2]) + if err != nil { + return "", err + } + r.rule = string(content) + } + if call == r.failOn { + return "", fmt.Errorf("%s: exit status 1", call) + } + return "", nil +} + +func isTempValidation(name string, args []string) bool { + return name == "sudo" && len(args) == 3 && args[0] == "visudo" && args[1] == "-cf" && + args[2] != sudoersPolicyPath +} + +func (r *recorder) lookPath(name string) (string, error) { + return "/bin/" + name, nil +} + +func (r *recorder) appendEnv(lines []string) error { + r.env = append(r.env, lines...) + return nil +} + +func newTestProvisioner(r *recorder, dir string) *provisioner { + return &provisioner{ + run: r.run, + lookPath: r.lookPath, + appendEnv: r.appendEnv, + tempDir: dir, + } +} + +func assertCalls(t *testing.T, got, want []string) { + t.Helper() + if !slices.Equal(got, want) { + t.Fatalf("calls:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +var ( + loginCalls = []string{ + startSSHCall, + "sudo useradd --create-home --shell /bin/bash devsy-bash", + "sudo useradd --create-home --shell /bin/zsh devsy-zsh", + "sudo useradd --create-home --shell /bin/fish devsy-fish", + } + qualificationCalls = []string{ + startSSHCall, + "sudo useradd --create-home --shell /bin/bash devsy-bash", + "sudo usermod --append --groups docker devsy-bash", + "sudo useradd --create-home --shell /bin/zsh devsy-zsh", + "sudo usermod --append --groups docker devsy-zsh", + "sudo useradd --create-home --shell /bin/fish devsy-fish", + "sudo usermod --append --groups docker devsy-fish", + "sudo useradd --create-home --shell /bin/bash --groups docker devsy-sudo", + "sudo test ! -e /etc/sudoers.d/devsy-e2e-qualification", + "sudo visudo -cf " + testTempCall, + "sudo install -m 0440 -o root -g root " + testTempCall + + " /etc/sudoers.d/devsy-e2e-qualification", + "sudo visudo -cf /etc/sudoers.d/devsy-e2e-qualification", + } +) + +func TestProvisionLoginsRunsSSHStartThenOneAccountPerShell(t *testing.T) { + r := &recorder{} + p := newTestProvisioner(r, t.TempDir()) + + if err := p.provisionLogins(); err != nil { + t.Fatalf("provisionLogins() error = %v", err) + } + assertCalls(t, r.calls, loginCalls) +} + +func TestProvisionQualificationRunsFullSequence(t *testing.T) { + r := &recorder{} + p := newTestProvisioner(r, t.TempDir()) + + if err := p.provisionQualification(); err != nil { + t.Fatalf("provisionQualification() error = %v", err) + } + assertCalls(t, r.calls, qualificationCalls) + if r.rule != testPolicyLine { + t.Fatalf("sudoers rule = %q, want %q", r.rule, testPolicyLine) + } + wantEnv := []string{"DEVSY_SUDO_FIXTURE_POLICY=/etc/sudoers.d/devsy-e2e-qualification"} + if !slices.Equal(r.env, wantEnv) { + t.Fatalf("env lines = %q, want %q", r.env, wantEnv) + } +} + +func TestProvisionQualificationStopsAtFailedUsermod(t *testing.T) { + r := &recorder{failOn: "sudo usermod --append --groups docker devsy-bash"} + p := newTestProvisioner(r, t.TempDir()) + + err := p.provisionQualification() + if err == nil { + t.Fatal("provisionQualification() error = nil, want usermod failure") + } + if !strings.Contains(err.Error(), "usermod") { + t.Fatalf("error %q does not mention usermod", err) + } + assertCalls(t, r.calls, qualificationCalls[:3]) +} + +func TestProvisionQualificationRejectsExistingPolicyBeforeVisudo(t *testing.T) { + r := &recorder{failOn: "sudo test ! -e /etc/sudoers.d/devsy-e2e-qualification"} + p := newTestProvisioner(r, t.TempDir()) + + if err := p.provisionQualification(); err == nil { + t.Fatal("provisionQualification() error = nil, want existing policy failure") + } + for _, call := range r.calls { + if strings.Contains(call, "visudo") || strings.Contains(call, "install") { + t.Fatalf("call %q ran after the policy existence check failed", call) + } + } + if len(r.env) != 0 { + t.Fatalf("env lines = %q, want none", r.env) + } +} + +func TestProvisionQualificationRemovesTempRule(t *testing.T) { + dir := t.TempDir() + p := newTestProvisioner(&recorder{}, dir) + + if err := p.provisionQualification(); err != nil { + t.Fatalf("provisionQualification() error = %v", err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("temp dir holds %d leftover entries, want none", len(entries)) + } +} + +func TestCleanupQualificationAccountsRemovesOnlyTheExactPolicy(t *testing.T) { + const policy = "/etc/sudoers.d/devsy-e2e-qualification" + tests := []struct { + name string + env string + wantErr bool + want []string + }{ + {name: "unset is a no-op", env: ""}, + {name: "foreign path is refused", env: "/etc/passwd", wantErr: true}, + {name: "exact path is removed", env: policy, want: []string{"sudo rm -f -- " + policy}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(policyEnv, tt.env) + r := &recorder{} + p := newTestProvisioner(r, t.TempDir()) + + err := p.removeSudoersPolicy() + if (err != nil) != tt.wantErr { + t.Fatalf("removeSudoersPolicy() error = %v, wantErr %v", err, tt.wantErr) + } + assertCalls(t, r.calls, tt.want) + }) + } +} + +func TestProvisionLoginsStopsWhenSSHStartFails(t *testing.T) { + r := &recorder{failOn: startSSHCall} + p := newTestProvisioner(r, t.TempDir()) + + err := p.provisionLogins() + if err == nil || !strings.Contains(err.Error(), "service") { + t.Fatalf("provisionLogins() error = %v, want ssh start failure", err) + } + assertCalls(t, r.calls, loginCalls[:1]) +} diff --git a/hack/ci/qualify.go b/hack/ci/qualify.go index a167f5b..b4f2bcd 100644 --- a/hack/ci/qualify.go +++ b/hack/ci/qualify.go @@ -1,13 +1,9 @@ package main import ( - "crypto/sha256" - "encoding/hex" "fmt" "io" "os" - "os/exec" - "path/filepath" "regexp" "strings" ) @@ -277,29 +273,6 @@ func writeEvidence(i qualificationInput, result qualificationResult) error { return appendLines(os.Getenv("GITHUB_STEP_SUMMARY"), evidence) } -func appendLines(path string, lines []string) error { - if path == "" { - return fmt.Errorf("output path is required") - } - directory, name := filepath.Split(filepath.Clean(path)) - root, err := os.OpenRoot(directory) - if err != nil { - return err - } - defer func() { _ = root.Close() }() - file, err := root.OpenFile(name, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) - if err != nil { - return err - } - defer func() { _ = file.Close() }() - for _, line := range lines { - if _, err := fmt.Fprintln(file, line); err != nil { - return err - } - } - return nil -} - func validateAncestry(statuses []string) error { if len(statuses) != len(coreFixes) { return fmt.Errorf( @@ -327,39 +300,6 @@ func validateAssetDigest(published, digest string) error { return nil } -func parseSHA256Output(output string) (string, error) { - fields := strings.Fields(output) - if len(fields) < 2 || len(fields[0]) != sha256.Size*2 { - return "", fmt.Errorf("invalid sha256sum output %q", output) - } - if _, err := hex.DecodeString(fields[0]); err != nil { - return "", fmt.Errorf("invalid sha256sum output %q", output) - } - return fields[0], nil -} - -func command(name string, args ...string) (string, error) { - var cmd *exec.Cmd - switch name { - case "gh": - cmd = exec.Command("gh", args...) - case "git": - cmd = exec.Command("git", args...) - case "sha256sum": - cmd = exec.Command("sha256sum", args...) - case "dpkg-deb": - cmd = exec.Command("dpkg-deb", args...) - default: - return "", fmt.Errorf("unsupported command %q", name) - } - cmd.Stderr = os.Stderr - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) - } - return string(output), nil -} - func validTag(tag string) bool { return coreTagPattern.MatchString(tag) } diff --git a/hack/ci/verify_provider.go b/hack/ci/verify_provider.go new file mode 100644 index 0000000..4450332 --- /dev/null +++ b/hack/ci/verify_provider.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + providerAssetName = "devsy-provider-ssh-linux-amd64" + providerBinaryDir = "dist/build_linux_amd64_v1" + providerDistDir = "dist" + providerChecksums = "checksums.txt" +) + +type providerRelease struct { + tag string + repo string + qualificationDir string +} + +func verifyProvider() error { + env, err := requireEnv("PROVIDER_TAG", "QUALIFICATION_DIR", "GITHUB_REPOSITORY") + if err != nil { + return err + } + release := providerRelease{ + tag: env["PROVIDER_TAG"], + repo: env["GITHUB_REPOSITORY"], + qualificationDir: env["QUALIFICATION_DIR"], + } + return release.verify() +} + +func (r providerRelease) verify() error { + binary := filepath.Join(providerBinaryDir, providerAssetName) + if err := r.download(); err != nil { + return err + } + digest, err := providerDigest(binary) + if err != nil { + return err + } + if err := r.requirePublishedDigest(digest); err != nil { + return err + } + if err := os.Chmod(binary, 0o700); err != nil { + return err + } + evidence := []string{ + "provider binary sha256: " + digest + " (matches published release digest)", + } + pairPath := filepath.Join(r.qualificationDir, "pair-verification.txt") + if err := appendLines(pairPath, evidence); err != nil { + return err + } + return appendLines(os.Getenv("GITHUB_STEP_SUMMARY"), evidence) +} + +func (r providerRelease) download() error { + if err := os.MkdirAll(providerBinaryDir, 0o750); err != nil { + return err + } + if _, err := command( + "gh", + "release", + "download", + r.tag, + "--repo", + r.repo, + "--pattern", + providerAssetName, + "--dir", + providerBinaryDir, + "--clobber", + ); err != nil { + return fmt.Errorf("download provider binary: %w", err) + } + if _, err := command( + "gh", + "release", + "download", + r.tag, + "--repo", + r.repo, + "--pattern", + providerChecksums, + "--dir", + providerDistDir, + "--clobber", + ); err != nil { + return fmt.Errorf("download provider checksums: %w", err) + } + return nil +} + +func (r providerRelease) requirePublishedDigest(digest string) error { + published, err := command( + "gh", + "api", + "repos/"+r.repo+"/releases/tags/"+r.tag, + "--jq", + ".assets[] | select(.name == \""+providerAssetName+"\") | .digest", + ) + if err != nil { + return fmt.Errorf("read published provider digest: %w", err) + } + return validateAssetDigest(strings.TrimSpace(published), digest) +} + +func providerDigest(binary string) (string, error) { + output, err := command("sha256sum", binary) + if err != nil { + return "", err + } + return parseSHA256Output(output) +} diff --git a/hack/ci/verify_report.go b/hack/ci/verify_report.go new file mode 100644 index 0000000..b82e436 --- /dev/null +++ b/hack/ci/verify_report.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "path/filepath" + + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" +) + +const reportFileName = "qualification-report.json" + +func verifyReport() error { + env, err := requireEnv("QUALIFICATION_DIR", "CORE_SHA256") + if err != nil { + return err + } + report, err := qualification.Load(filepath.Join(env["QUALIFICATION_DIR"], reportFileName)) + if err != nil { + return fmt.Errorf("load qualification report: %w", err) + } + want := qualification.Expectation{CoreBinarySHA256: env["CORE_SHA256"]} + if err := report.Validate(want); err != nil { + return fmt.Errorf("validate qualification report: %w", err) + } + return nil +} diff --git a/hack/ci/verify_test.go b/hack/ci/verify_test.go new file mode 100644 index 0000000..72ba6a5 --- /dev/null +++ b/hack/ci/verify_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/devsy-org/devsy-provider-ssh/e2e/qualification" +) + +const ( + testCoreSHA = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + testAgentDir = "/tmp/devsy-e2e-verify space'\\$-\u03bb" + testUserName = "devsy-user" + testVersion = "v1.0.0" +) + +func TestEvaluateGuardResult(t *testing.T) { + tests := []struct { + name string + exitCode int + output string + wantErr bool + }{ + { + name: "exit 1 with message passes", + exitCode: 1, + output: guardMessage, + wantErr: false, + }, + { + name: "exit 0 with message fails", + exitCode: 0, + output: guardMessage, + wantErr: true, + }, + { + name: "exit 1 without message fails", + exitCode: 1, + output: "--- FAIL: TestRelease", + wantErr: true, + }, + { + name: "exit 2 with message fails", + exitCode: 2, + output: guardMessage, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := evaluateGuardResult(tt.exitCode, tt.output) + if (err != nil) != tt.wantErr { + t.Fatalf("evaluateGuardResult(%d, %q) error = %v, want error %t", + tt.exitCode, tt.output, err, tt.wantErr) + } + }) + } +} + +func TestVerifyReportRequiresEnvironment(t *testing.T) { + t.Setenv("QUALIFICATION_DIR", "") + t.Setenv("CORE_SHA256", "") + err := verifyReport() + if err == nil || !strings.Contains(err.Error(), "missing required environment") { + t.Fatalf("verifyReport() error = %v, want missing required environment", err) + } +} + +func TestVerifyReportAcceptsValidReport(t *testing.T) { + t.Setenv("QUALIFICATION_DIR", writeTestReport(t, validTestReport())) + t.Setenv("CORE_SHA256", testCoreSHA) + if err := verifyReport(); err != nil { + t.Fatalf("verifyReport() error = %v", err) + } +} + +func TestVerifyReportRejectsOtherCoreBinary(t *testing.T) { + t.Setenv("QUALIFICATION_DIR", writeTestReport(t, validTestReport())) + t.Setenv("CORE_SHA256", strings.Repeat("0", 64)) + if err := verifyReport(); err == nil { + t.Fatal("verifyReport() accepted a report built for another core binary") + } +} + +func writeTestReport(t *testing.T, report qualification.Report) string { + t.Helper() + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, reportFileName), data, 0o600); err != nil { + t.Fatal(err) + } + return dir +} + +func validTestReport() qualification.Report { + report := qualification.Report{SchemaVersion: qualification.SchemaVersion} + report.Pair = qualification.Pair{ + CoreTag: testVersion, + CoreVersion: testVersion, + CoreBinarySHA256: testCoreSHA, + } + for _, transport := range qualification.Transports { + report.Cells = append(report.Cells, validTestCell(transport)) + } + report.Privilege = validTestPrivilege() + return report +} + +func validTestCell(transport qualification.Transport) *qualification.Cell { + scenarios := make([]qualification.Scenario, 0, len(qualification.LifecycleScenarios)) + for _, name := range qualification.LifecycleScenarios { + scenarios = append(scenarios, qualification.Scenario{Name: name, Outcome: "passed"}) + } + return &qualification.Cell{ + Shell: transport.Shell, + User: testUserName, + BuiltinSSH: transport.Builtin, + Workspace: "workspace", + AgentDir: testAgentDir, + LoginShell: "/bin/" + transport.Shell, + Scenarios: scenarios, + AgentSHA256: map[string]string{"download": testCoreSHA, "stdin": testCoreSHA}, + CleanupStatus: "ok", + } +} + +func validTestPrivilege() *qualification.Privilege { + return &qualification.Privilege{ + User: qualification.PrivilegeUser, + UserUID: 1001, + AgentDir: testAgentDir, + DirectoryState: "0:755", + WriteDenied: true, + AgentUID: 0, + AgentVersion: testVersion, + AgentSHA256: testCoreSHA, + LiveCommand: true, + Outcome: "passed", + CleanupStatus: "ok", + } +}