diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6a2a054..2b36724 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,16 +19,16 @@ permissions: jobs: tests: - # Threshold 0 is the bootstrap case of test-shell.yml: there is no - # tests/coverage.sh on this branch yet, so the job passes with a notice - # and the check exists, which is what lets main be protected from the - # first day. It is raised to the measured number when the tests land, - # and only ever raised (decision 0006). This job produces the check - # "tests / coverage", the required status on main. The job id is part - # of that name, so renaming it silently detaches the protection rule. + # tests/coverage.sh measures three shell files (COVERAGE.md): the + # first boot library, the verification hook this layer adds and the + # boot test's library, all three at 100 percent. The threshold is the + # measured number and is only ever raised (decision 0006). This job + # produces the check "tests / coverage", the required status on main. + # The job id is part of that name, so renaming it silently detaches + # the protection rule. uses: keel-linux/.github/.github/workflows/test-shell.yml@main with: - threshold: 0 + threshold: 100 appliance: # The boot test (org-plan section 1): the postgresql layer is fetched from # https://mirror.keellinux.org/layers, verified, assembled into a diff --git a/COVERAGE.md b/COVERAGE.md new file mode 100644 index 0000000..ce6a18c --- /dev/null +++ b/COVERAGE.md @@ -0,0 +1,67 @@ +# Coverage + +Standard: decisions 0003 (90 percent per repository, 95 for code the +project writes) and 0004 (bats plus kcov for shell; a build and a boot on +LXC as the acceptance test of a recipe, docs/org-plan.md section 1). + +## Measured 2026-09-27 + +| File | Test | Lines | Note | +| --- | --- | --- | --- | +| overlay/usr/lib/inithooks/lib/postgresql.sh | tests/postgresql.bats (11 tests) | 100 percent (41/41) under kcov | every function and every branch | +| overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify | tests/hook.bats (10 tests) | 100 percent (15/15) under kcov | every path, including the two failures that matter | +| tests/lib/boot-test-lib.sh | tests/boot-test.bats (37 tests) | 100 percent (141/141) under kcov | argument parsing, address discovery, deadlines, the database, module, Webmin and diff verdicts | +| conf.d/main | the build | integration only | build time script, 0004 pragmatic limits | +| tests/boot-test.sh | itself | integration only | the thin main of the acceptance test: keel and LXC as root | +| overlay ... firstboot.d/35pgsqlpass | common | not this repository | the hook that sets the password belongs to the `common` fork and is used, not rewritten | + +Total over the three measured shell files: **100 percent (197/197)**, +58 bats tests. `tests/coverage.sh` fails below `COVERAGE_THRESHOLD`, which +the workflow sets to 100, the measured number. It is only ever raised +(decision 0006). + + $ COVERAGE_THRESHOLD=100 tests/coverage.sh + kcov line coverage (threshold 100 percent): + 100.00 41/41 postgresql.sh + 100.00 15/15 36pgsqlverify + 100.00 141/141 boot-test-lib.sh + +## What the hook tests cover + +The hook is executed for real against scratch directories, with PATH stubs +for `systemctl`, `pg_isready` and `psql` under a scratch `INITHOOKS_PATH` +whose `lib` is a symlink to the real library, so kcov measures the file the +layer ships. The `psql` stub answers the probe only for the password the +description declared, the way a server does. No test needs root, a +database or a network. + +What they are about is the half of the password defect that is this +layer's: the declared password is proved against the database, a password +the database refuses fails the hook, an answer that is not the probe's +fails it, nothing declared fails naming the field and never prompts, a +`PGSQL_PASS` in the conf is not a password and is ignored, `APP_DB_USER` +renames the role and a name that would need quoting is refused before +anything runs, and the cluster is started and waited for on the address the +client will use. + +## The appliance gate + +`appliance / build-and-boot` runs through the organization's +`test-appliance.yml` on the self-hosted `keel-lxc` runner, which fetches +the published layer from `https://mirror.keellinux.org/layers`, verifies +it, assembles it, boots it in LXC and runs `tests/boot-test.sh`. Nothing is +built there. + +State on 2026-09-27: **the layer is not published yet, so the job skips +with a notice and passes.** It becomes a required status on `main` the day +the first `postgresql` layer reaches the mirror. + +## Plan + +- Publish the layer, then require `appliance / build-and-boot` on `main`. +- Measure `conf.d/main`. A build time script that runs inside a chroot as + root is the case decision 0003 splits, and what is left here after the + logic moved to `lib/postgresql.sh` is SQL, three assertions about the + cluster configuration and the apt calls. +- `35pgsqlpass` and `bin/pgsqlconf.py` belong to the `common` fork and are + measured there; `common`'s own COVERAGE.md carries that plan. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..ea76424 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,97 @@ +# Tests + +What a test means for a layer recipe is written in `COVERAGE.md`: the +recipe builds, the result boots in an LXC container, its first boot +completes headless from an instance description, the database accepts the +password that description declared, the panel has the module for it, and +the machine matches the description. + +## Layout + +- `boot-test.sh`: the boot test. `test-appliance.yml` (reusable workflow of + `keel-linux/.github`) runs it on the self-hosted LXC runner after pulling + the layers from `https://mirror.keellinux.org/layers` and checking them + with `keel verify`. It is the thin main: assemble, mark the tree as a + container, install the description, the secrets and the conf, start the + container, wait, connect to the database, check Webmin, `keel diff`. It + builds nothing, so it needs no fab, deck or buildtasks. +- `lib/boot-test-lib.sh`: the logic (argument parsing, address discovery + from `lxc-info`, waiting with a deadline, the secret files, the client + call, the database, module and Webmin verdicts, the diff verdict), as + functions with no side effects, per decision 0004. Same shape as the one + in keel-core, keel-nodebb and keel-mariadb. +- `boot-test.bats`: unit tests of that library. `lxc-info` is a stub first + in `PATH`; the clock and `sleep` are functions. No root, no network, no + LXC, no database. +- `postgresql.bats`: unit tests of + `overlay/usr/lib/inithooks/lib/postgresql.sh`, the logic behind the + first boot hook `36pgsqlverify`. +- `hook.bats`: `36pgsqlverify` itself, run for real against scratch + directories with every system command stubbed. The hook that sets the + password, `35pgsqlpass`, belongs to the `common` fork and is tested + there; this layer uses it rather than rewriting it. +- `coverage.sh`: runs the bats suite under kcov and fails when any measured + file is below `COVERAGE_THRESHOLD` (default 100). +- `instance.yaml`: the description the test container boots from. It + declares `secrets.db_password` from a file, which is the point of the + test. + +## Unit tests and coverage + +Debian packages `bats` (1.11) and `kcov` (43); no root: + + bats tests/postgresql.bats + bats tests/hook.bats + bats tests/boot-test.bats + COVERAGE_THRESHOLD=100 tests/coverage.sh + +`COVERAGE_DIR=coverage tests/coverage.sh` keeps the kcov reports. + +## The boot test by hand + +Needs root, `keel` on `PATH`, LXC (`lxc-start`, `lxc-info`, `lxc-attach`, +`lxc-stop`), `curl`, and a bridge with IPv6 router advertisements or +DHCPv6. + + tests/boot-test.sh postgresql --layers-dir https://mirror.keellinux.org/layers \ + --bridge lxcbr0 + +`--layers-dir` is a directory or an http(s) URL, so on the build host it is +`/mnt/builds/layers` and on a runner it is the mirror. The other useful +options are `--bridge`, `--cache-dir`, `--lxc-path`, `--name`, `--timeout` +and `--keep` (leaves the container running; then `lxc-attach -n `). +`tests/boot-test.sh --help` lists them all. + +What it does, in order: + +1. `keel pull` and `keel assemble` the chain (core, postgresql) into + `//rootfs`. +2. Creates `var/lib/turnkey-info/inithooks.service/lxc` in the rootfs, the + marker `bt-container` writes and the one `keel inspect` reads to call + the machine a container (`network.managed_by: host`). +3. Writes a random `root_password` and `db_password` under + `etc/keel/secrets` (mode 0600) and installs `tests/instance.yaml` at + `etc/keel/instance.yaml` and `etc/inithooks.yaml`. +4. Renders the description into the rootfs `etc/inithooks.conf` with + `keel spec apply`, from a copy whose secret references point inside the + rootfs. Without the conf the first boot is not headless: `30rootpass` + would have nothing declared and no terminal, and `36pgsqlverify` + would fail naming the field the description has to declare. +5. Writes an LXC config for that rootfs on the bridge and starts the + container. +6. Waits for a global IPv6 address (`lxc-info -i`), then for the first boot + to finish: `RUN_FIRSTBOOT=false` in the rootfs copy of + `/etc/default/inithooks`, and then confconsole or an SSH banner. +7. **Connects to the database.** `psql --username=postgres --host=::1 + --port=5432 --no-password --command='SELECT 1'` inside the container, + with the declared password in `PGPASSWORD`. `--no-password` makes psql + fail rather than prompt, so a password that did not arrive is an error + and not a hung test. The password comes from the secret + file the test wrote in step 3 and from nowhere else, so a row back is + the proof that the declarative path carried it end to end. A listening + port would prove nothing: the database listens whatever password it + ended up with. +8. Checks `webmin-postgresql` is installed and that Webmin answers over IPv6 on + 12321. +9. Runs `keel diff --root --spec tests/instance.yaml`; exit 0 or + 13 (no drift) passes. diff --git a/tests/boot-test.bats b/tests/boot-test.bats new file mode 100644 index 0000000..152b59e --- /dev/null +++ b/tests/boot-test.bats @@ -0,0 +1,359 @@ +#!/usr/bin/env bats +# Unit tests of tests/lib/boot-test-lib.sh: argument parsing, address +# discovery from lxc-info, waiting with a deadline, the secret files, the +# database, Webmin and module verdicts, and the diff verdict. Nothing here +# needs root, a network, a database or LXC: lxc-info is a stub first in +# PATH, the clock and sleep are functions. + +bats_require_minimum_version 1.5.0 + +setup() { + load lib/boot-test-lib.sh + STUBS=$(mktemp -d) + PATH=$STUBS:$PATH +} + +teardown() { + rm -rf "$STUBS" +} + +stub_lxc_info() { + # stub_lxc_info OUTPUT: lxc-info prints OUTPUT and records its arguments + printf '#!/bin/bash\necho "$*" >> "%s/lxc-info.calls"\ncat <<"OUT"\n%s\nOUT\n' "$STUBS" "$1" > "$STUBS/lxc-info" + chmod +x "$STUBS/lxc-info" +} + +# argument parsing + +@test "parse_args: the appliance alone takes every default" { + bt_parse_args postgresql + [ "$BT_APPLIANCE" = postgresql ] + [ "$BT_TIMEOUT" = 900 ] + [ "$BT_INTERVAL" = 5 ] + [ "$BT_BRIDGE" = br0 ] + [ "$BT_LAYERS_DIR" = /mnt/builds/layers ] + [ "$BT_CACHE_DIR" = /var/cache/keel/layers ] + [ "$BT_LXC_PATH" = /var/lib/lxc ] + [ -z "$BT_SPEC" ] + [ "$BT_KEEP" = 0 ] + [ "$BT_NAME" = keel-postgresql-boot-test ] + [ "$BT_ROOTFS" = /var/lib/lxc/keel-postgresql-boot-test/rootfs ] +} + +@test "parse_args: every option is read" { + bt_parse_args --timeout 60 --interval 2 --bridge lxcbr0 --layers-dir /l \ + --cache-dir /c --lxc-path /x --name lamp-run-7 --spec /s.yaml --keep lamp + [ "$BT_APPLIANCE" = lamp ] + [ "$BT_TIMEOUT" = 60 ] + [ "$BT_INTERVAL" = 2 ] + [ "$BT_BRIDGE" = lxcbr0 ] + [ "$BT_LAYERS_DIR" = /l ] + [ "$BT_CACHE_DIR" = /c ] + [ "$BT_LXC_PATH" = /x ] + [ "$BT_SPEC" = /s.yaml ] + [ "$BT_KEEP" = 1 ] + [ "$BT_NAME" = lamp-run-7 ] + [ "$BT_ROOTFS" = /x/lamp-run-7/rootfs ] +} + +@test "parse_args: --name is checked as a container name" { + run bt_parse_args postgresql --name "Run 7" + [ "$status" -eq 1 ] + [[ $output == *"is not a container name"* ]] + run bt_parse_args postgresql --name -lead + [ "$status" -eq 1 ] +} + +@test "is_container_name" { + bt_is_container_name keel-postgresql-ci-36255612491-1 + bt_is_container_name 7 + run ! bt_is_container_name "keel core" + run ! bt_is_container_name -x + run ! bt_is_container_name "" +} + +@test "parse_args: the appliance is required" { + run bt_parse_args --keep + [ "$status" -eq 1 ] + [[ $output == *"APPLIANCE is required"* ]] +} + +@test "parse_args: one appliance at a time" { + run bt_parse_args postgresql lamp + [ "$status" -eq 1 ] + [[ $output == *"one appliance at a time"* ]] +} + +@test "parse_args: the keel- prefix and upper case are rejected" { + run bt_parse_args keel-postgresql + [ "$status" -eq 1 ] + [[ $output == *"not an appliance name"* ]] + run bt_parse_args NodeBB + [ "$status" -eq 1 ] +} + +@test "parse_args: an unknown option fails" { + run bt_parse_args postgresql --verbose + [ "$status" -eq 1 ] + [[ $output == *"unknown option --verbose"* ]] +} + +@test "parse_args: timeout and interval must be positive integers" { + run bt_parse_args postgresql --timeout 0 + [ "$status" -eq 1 ] + [[ $output == *"--timeout needs a positive number"* ]] + run bt_parse_args postgresql --interval abc + [ "$status" -eq 1 ] + run bt_parse_args postgresql --timeout + [ "$status" -eq 1 ] +} + +@test "parse_args: an option with a value refuses an empty one" { + run bt_parse_args postgresql --bridge + [ "$status" -eq 1 ] + [[ $output == *"--bridge needs a value"* ]] + run bt_parse_args postgresql --spec "" + [ "$status" -eq 1 ] +} + +@test "parse_args: --help prints the usage and returns 2" { + run bt_parse_args --help + [ "$status" -eq 2 ] + [[ ${lines[0]} == "usage: tests/boot-test.sh APPLIANCE"* ]] + [[ $output == *"--keep"* ]] + run bt_parse_args -h + [ "$status" -eq 2 ] +} + +@test "is_positive_int and is_appliance_name" { + bt_is_positive_int 1 + bt_is_positive_int 900 + run ! bt_is_positive_int 0 + run ! bt_is_positive_int 07 + run ! bt_is_positive_int -5 + run ! bt_is_positive_int "" + bt_is_appliance_name nginx-php-fastcgi + run ! bt_is_appliance_name keel-core + run ! bt_is_appliance_name 9core + run ! bt_is_appliance_name "" +} + +# address discovery + +@test "is_global_ipv6: global and ULA yes, link local, loopback, multicast, IPv4 no" { + bt_is_global_ipv6 2001:db8:1::10 + bt_is_global_ipv6 fd00:1::10 + bt_is_global_ipv6 2001:DB8::1 + run ! bt_is_global_ipv6 fe80::216:3eff:fe00:1 + run ! bt_is_global_ipv6 FEBF::1 + run ! bt_is_global_ipv6 ::1 + run ! bt_is_global_ipv6 ff02::1 + run ! bt_is_global_ipv6 192.0.2.10 + run ! bt_is_global_ipv6 "" +} + +@test "global_ipv6: picks the first global address out of lxc-info output" { + output=$(printf 'IP: fe80::216:3eff:fe00:1\nIP: 192.0.2.10\nIP: 2001:db8:1::10\nIP: 2001:db8:1::11\n' | bt_global_ipv6) + [ "$output" = 2001:db8:1::10 ] +} + +@test "global_ipv6: ignores lines that are not addresses" { + output=$(printf 'Name: keel-postgresql-boot-test\nState: RUNNING\nPID: 4242\nIP: fd00::10\nLink: veth0\n' | bt_global_ipv6) + [ "$output" = fd00::10 ] +} + +@test "global_ipv6: returns 1 while only link local or IPv4 addresses exist" { + run bt_global_ipv6 <<< $'IP: fe80::1\nIP: 192.0.2.10' + [ "$status" -eq 1 ] + [ -z "$output" ] + run bt_global_ipv6 < /dev/null + [ "$status" -eq 1 ] +} + +@test "container_ipv6: calls lxc-info with the lxcpath and the name" { + stub_lxc_info $'Name: keel-postgresql-boot-test\nIP: fe80::1\nIP: 2001:db8:1::10' + output=$(bt_container_ipv6 keel-postgresql-boot-test /var/lib/lxc) + [ "$output" = 2001:db8:1::10 ] + [ "$(cat "$STUBS/lxc-info.calls")" = "-P /var/lib/lxc -n keel-postgresql-boot-test -i" ] +} + +@test "container_ipv6: fails when lxc-info has no global address yet" { + stub_lxc_info $'Name: keel-postgresql-boot-test\nState: RUNNING' + run bt_container_ipv6 keel-postgresql-boot-test /var/lib/lxc + [ "$status" -eq 1 ] +} + +# timeouts + +fake_clock() { echo "$FAKE_NOW"; } +fake_sleep() { FAKE_NOW=$(( FAKE_NOW + $1 )); echo "sleep $1" >> "$STUBS/sleeps"; } +succeed_on_third() { CALLS=$(( CALLS + 1 )); [ "$CALLS" -ge 3 ]; } +never() { return 1; } + +@test "deadline_passed" { + run ! bt_deadline_passed 100 30 129 + bt_deadline_passed 100 30 130 + bt_deadline_passed 100 30 500 +} + +@test "now: the default clock is epoch seconds and BT_CLOCK replaces it" { + [[ $(bt_now) =~ ^[0-9]{10}$ ]] + BT_CLOCK=fake_clock FAKE_NOW=42 + [ "$(bt_now)" = 42 ] +} + +@test "wait_for: polls at the interval until the command succeeds" { + BT_CLOCK=fake_clock BT_SLEEP=fake_sleep FAKE_NOW=1000 CALLS=0 + bt_wait_for 60 5 "three calls" succeed_on_third + [ "$CALLS" -eq 3 ] + [ "$(cat "$STUBS/sleeps")" = $'sleep 5\nsleep 5' ] +} + +@test "wait_for: gives up with a message once the timeout has passed" { + # shellcheck disable=SC2034 # read by bt_now and bt_wait_for + BT_CLOCK=fake_clock BT_SLEEP=fake_sleep FAKE_NOW=1000 + run bt_wait_for 12 5 "something that never happens" never + [ "$status" -eq 1 ] + [[ $output == *"timeout after 12s waiting for something that never happens"* ]] + [ "$(wc -l < "$STUBS/sleeps")" -eq 3 ] +} + +# readiness and verdicts + +@test "is_ssh_banner" { + bt_is_ssh_banner "SSH-2.0-OpenSSH_10.0p2 Debian-7" + run ! bt_is_ssh_banner "HTTP/1.1 400 Bad Request" + run ! bt_is_ssh_banner "" +} + +@test "firstboot_done_in: RUN_FIRSTBOOT=false in the rootfs copy of /etc/default/inithooks" { + printf 'INITHOOKS_CONF=/etc/inithooks.conf\nRUN_FIRSTBOOT=false\n' > "$STUBS/done" + printf 'RUN_FIRSTBOOT=true\n' > "$STUBS/pending" + bt_firstboot_done_in "$STUBS/done" + run ! bt_firstboot_done_in "$STUBS/pending" + run ! bt_firstboot_done_in "$STUBS/missing" +} + +@test "lxc_config: names the container, the rootfs and the bridge" { + output=$(bt_lxc_config keel-postgresql-boot-test /var/lib/lxc/keel-postgresql-boot-test/rootfs br0) + [[ $output == *"lxc.uts.name = keel-postgresql-boot-test"* ]] + [[ $output == *"lxc.rootfs.path = dir:/var/lib/lxc/keel-postgresql-boot-test/rootfs"* ]] + [[ $output == *"lxc.net.0.link = br0"* ]] + [[ $output == *"lxc.net.0.type = veth"* ]] +} + +@test "spec_targets: both paths the first boot reads, under the rootfs" { + output=$(bt_spec_targets /r) + [ "$output" = $'/r/etc/keel/instance.yaml\n/r/etc/inithooks.yaml' ] +} + +@test "secret_targets: the two secret files the spec references" { + output=$(bt_secret_targets /r) + [ "$output" = $'/r/etc/keel/secrets/root_password\n/r/etc/keel/secrets/db_password' ] +} + +@test "webmin_verdict: the login page or a challenge is an answer" { + run bt_webmin_verdict 200 + [ "$status" -eq 0 ] + [[ $output == *"webmin answered 200 on port 12321"* ]] + run bt_webmin_verdict 401 + [ "$status" -eq 0 ] + run bt_webmin_verdict 000 + [ "$status" -eq 1 ] + [[ $output == *"not 200 or 401"* ]] + run bt_webmin_verdict 502 + [ "$status" -eq 1 ] + run bt_webmin_verdict + [ "$status" -eq 1 ] + [[ $output == *"answered ''"* ]] +} + +@test "module_verdict: the database module must be installed on the machine" { + run bt_module_verdict webmin-postgresql "install ok installed" + [ "$status" -eq 0 ] + [[ $output == *"webmin-postgresql is installed"* ]] + run bt_module_verdict webmin-postgresql "install ok unpacked" + [ "$status" -eq 1 ] + [[ $output == *"is 'install ok unpacked'"* ]] + run bt_module_verdict webmin-postgresql + [ "$status" -eq 1 ] + [[ $output == *"is ''"* ]] +} + +@test "db_verdict: the probe answer passes, anything else is a refusal" { + run bt_db_verdict 1 + [ "$status" -eq 0 ] + [[ $output == *"postgres authenticated on [::1]:5432 with the declared password"* ]] + run bt_db_verdict $'\n1\n' + [ "$status" -eq 0 ] + run bt_db_verdict "psql: error: connection failed: password authentication failed" + [ "$status" -eq 1 ] + [[ $output == *"did not reach the database"* ]] + run bt_db_verdict + [ "$status" -eq 1 ] + [[ $output == *"answered ''"* ]] +} + +@test "db_client_argv: the client call, with no password on the line" { + output=$(bt_db_client_argv postgres ::1 5432) + [ "$output" = $'psql\n--username=postgres\n--host=::1\n--port=5432\n--dbname=postgres\n--no-password\n--tuples-only\n--no-align\n--quiet\n--command=SELECT 1' ] + [[ $output != *"PGPASSWORD"* ]] + [[ $output == *"--no-password"* ]] +} + +@test "db_client_argv: another database can be named" { + output=$(bt_db_client_argv postgres ::1 5432 template1) + [[ $output == *"--dbname=template1"* ]] +} + +@test "db_client_argv: an empty user, host, database or a bad port fails" { + run ! bt_db_client_argv "" ::1 5432 + run ! bt_db_client_argv postgres "" 5432 + run ! bt_db_client_argv postgres ::1 "" + run ! bt_db_client_argv postgres ::1 fivefourthreetwo + run ! bt_db_client_argv postgres ::1 5432 "" +} + +@test "spec_in_rootfs: secret references are pointed inside the rootfs" { + printf 'secrets:\n root_password:\n file: /etc/keel/secrets/root_password\ntls:\n acme:\n enabled: false\n' > "$STUBS/spec" + output=$(bt_spec_in_rootfs "$STUBS/spec" /r/rootfs) + [[ $output == *"file: /r/rootfs/etc/keel/secrets/root_password"* ]] + [[ $output == *"enabled: false"* ]] + [[ $output != *"file: /etc/keel"* ]] +} + +@test "random_password: 24 alphanumeric characters from the random source" { + output=$(bt_random_password) + [[ $output =~ ^[A-Za-z0-9]{24}$ ]] + printf 'ab!!cd%%%%efghijklmnopqrstuvwxyz0123456789' > "$STUBS/random" + output=$(BT_RANDOM_SOURCE=$STUBS/random bt_random_password) + [ "$output" = abcdefghijklmnopqrstuvwx ] +} + +@test "random_password: a source too poor to fill the password fails loudly" { + printf '!!!!short!!!!' > "$STUBS/poor" + BT_RANDOM_SOURCE="$STUBS/poor" + run bt_random_password + [ "$status" -eq 1 ] + [[ $output == *"gave only 5 usable characters"* ]] +} + +@test "diff_verdict: 0 and 13 pass, everything else fails with a message" { + run bt_diff_verdict 0 + [ "$status" -eq 0 ] + [ "$output" = "keel diff: no drift" ] + run bt_diff_verdict 13 + [ "$status" -eq 0 ] + [[ $output == *"could not be observed offline"* ]] + run bt_diff_verdict 14 + [ "$status" -eq 1 ] + [[ $output == *"drift found"* ]] + run bt_diff_verdict 2 + [ "$status" -eq 1 ] + [[ $output == *"unreadable or invalid (exit 2)"* ]] + run bt_diff_verdict 3 + [ "$status" -eq 1 ] + run bt_diff_verdict 127 + [ "$status" -eq 1 ] + [[ $output == *"failed with exit 127"* ]] +} diff --git a/tests/boot-test.sh b/tests/boot-test.sh new file mode 100755 index 0000000..47d54cf --- /dev/null +++ b/tests/boot-test.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Boot test of this layer (org-plan section 1), modelled on the one in +# keel-nodebb and keel-mariadb: assemble the published layer chain into an LXC rootfs, boot +# it headless from tests/instance.yaml, wait for the first boot to finish, +# then prove the declarative path end to end. The instance description +# declares secrets.db_password from a file; nothing is configured by hand; +# and the test connects to the database as a client, with that password, to +# say whether it arrived. A listening port would prove nothing here: the +# database listens whatever password it ended up with. +# +# It also checks what the panel offers, because batteries included is a +# property of this distribution: the Webmin module for this database is +# installed and Webmin answers over IPv6 on 12321. +# +# Called by the reusable workflow test-appliance.yml after keel pull and +# keel verify; runnable by hand as root on any host with LXC, see +# tests/README.md. It builds nothing: the layers come from the mirror or +# from a directory bt-layer wrote, so the test needs no fab, deck or +# buildtasks. The logic lives in tests/lib/boot-test-lib.sh and is unit +# tested; this file is the thin main that touches the system. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=lib/boot-test-lib.sh +source "$here/lib/boot-test-lib.sh" + +bt_parse_args "$@" || { rc=$?; [ "$rc" -eq 2 ] && exit 0; exit 1; } +BT_SPEC=${BT_SPEC:-$here/instance.yaml} +if [ "$(id -u)" -ne 0 ]; then + echo "boot-test: must run as root (keel assemble, lxc-start)" >&2 + exit 1 +fi +for tool in keel lxc-start lxc-info lxc-attach lxc-stop curl; do + command -v "$tool" >/dev/null || { echo "boot-test: $tool not found" >&2; exit 1; } +done + +container_dir=$BT_LXC_PATH/$BT_NAME +log() { printf '%s boot-test: %s\n' "$(date -u +%H:%M:%S)" "$*"; } +lxc() { "lxc-$1" -P "$BT_LXC_PATH" -n "$BT_NAME" "${@:2}"; } + +cleanup() { + local rc=$? + if [ "$rc" -ne 0 ] && [ -r "$BT_ROOTFS/var/log/inithooks.log" ]; then + log "last lines of the container's inithooks log:" + tail -n 40 "$BT_ROOTFS/var/log/inithooks.log" + fi + if [ "$BT_KEEP" -eq 1 ]; then + log "keeping $BT_NAME under $BT_LXC_PATH (--keep); lxc-attach -P $BT_LXC_PATH -n $BT_NAME" + return + fi + lxc stop -k >/dev/null 2>&1 || true + rm -rf "$container_dir" +} +trap cleanup EXIT + +# 1. Assemble the chain from the layers the build host published. +log "assembling $BT_APPLIANCE from $BT_LAYERS_DIR into $BT_ROOTFS" +lxc stop -k >/dev/null 2>&1 || true +rm -rf "$container_dir" +mkdir -p "$BT_ROOTFS" +keel pull "$BT_APPLIANCE" --source "$BT_LAYERS_DIR" --cache-dir "$BT_CACHE_DIR" --non-interactive +keel assemble "$BT_APPLIANCE" --rootfs "$BT_ROOTFS" --cache-dir "$BT_CACHE_DIR" --non-interactive + +# 2. The container marker, the instance description, the secrets it +# references and the conf the first boot hooks read. The marker under +# /var/lib/turnkey-info is what bt-container writes and what inspect +# reads to call the machine a container (managed_by: host); the conf is +# what makes the first boot headless, and without it 30rootpass and +# 35pgsqlpass wait on a dialog forever. +log "installing the description, the secrets and the conf into $BT_ROOTFS" +install -D -m 0644 /dev/null "$BT_ROOTFS/var/lib/turnkey-info/inithooks.service/lxc" +install -d -m 0700 "$BT_ROOTFS/etc/keel/secrets" +for target in $(bt_secret_targets "$BT_ROOTFS"); do + bt_random_password > "$target" + chmod 0600 "$target" +done +for target in $(bt_spec_targets "$BT_ROOTFS"); do + install -D -m 0600 "$BT_SPEC" "$target" +done +bt_spec_in_rootfs "$BT_SPEC" "$BT_ROOTFS" > "$container_dir/instance-host.yaml" +keel spec apply --spec "$container_dir/instance-host.yaml" \ + --conf "$BT_ROOTFS/etc/inithooks.conf" --non-interactive + +# The declared password, as the description names it. Everything below +# uses this and nothing else: no value is read out of the container. +declared_password=$(cat "$BT_ROOTFS/etc/keel/secrets/db_password") + +# 3. Boot. +bt_lxc_config "$BT_NAME" "$BT_ROOTFS" "$BT_BRIDGE" > "$container_dir/config" +log "starting $BT_NAME on bridge $BT_BRIDGE" +lxc start -d + +# 4. A global IPv6 address from the bridge. +bt_wait_for "$BT_TIMEOUT" "$BT_INTERVAL" "a global IPv6 address on $BT_NAME" \ + bt_container_ipv6 "$BT_NAME" "$BT_LXC_PATH" > /dev/null +addr=$(bt_container_ipv6 "$BT_NAME" "$BT_LXC_PATH") +log "container address $addr" + +# 5. First boot finished: 98finalize has cleared RUN_FIRSTBOOT and the +# machine answers, on the console (confconsole's usage screen) or on +# SSH. The answer alone is not enough: sshd is up long before the hooks +# are done, so the flag is what says the first boot ended. +usage_screen() { + lxc attach -- pgrep -f confconsole > /dev/null 2>&1 +} +ssh_answers() { + local banner + banner=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && read -r -t 5 line <&3 && printf "%s" "$line"' \ + "$addr" "$BT_SSH_PORT" 2>/dev/null) || return 1 + bt_is_ssh_banner "$banner" +} +first_boot_done() { + bt_firstboot_done_in "$BT_ROOTFS/etc/default/inithooks" || return 1 + usage_screen || ssh_answers +} +bt_wait_for "$BT_TIMEOUT" "$BT_INTERVAL" "the first boot of $BT_NAME to finish" \ + first_boot_done +log "first boot finished; ssh root@$addr" + +# 6. What the first boot hook reported, quoted here so a failure below is +# read next to it. +log "the database lines of the container's inithooks log:" +grep -E '35pgsqlpass|36pgsqlverify|PostgreSQL' "$BT_ROOTFS/var/log/inithooks.log" || true + +# 7. The declarative path, end to end: a client connection to the database +# with the password the description declared. The client runs inside the +# container over TCP on [::1], because the cluster listens on the +# loopback of both families and nowhere else, and the password reaches +# it in the environment so it never appears in the container's process +# list. --no-password makes psql fail rather than prompt, so a password +# that did not arrive is an error and not a hung test. +mapfile -t client < <(bt_db_client_argv "$BT_DB_USER" "$BT_DB_HOST" "$BT_DB_PORT") +log "connecting as $BT_DB_USER on [$BT_DB_HOST]:$BT_DB_PORT with the declared password" +answer=$(lxc attach --set-var "PGPASSWORD=$declared_password" -- "${client[@]}") \ + || { echo "boot-test: the database refused the declared password" >&2; exit 1; } +bt_db_verdict "$answer" + +# 8. The panel core carries, with the module this layer adds to it. +status=$(lxc attach -- dpkg-query -W -f '${Status}' "$BT_WEBMIN_MODULE" 2>/dev/null || true) +bt_module_verdict "$BT_WEBMIN_MODULE" "$status" +code="" +webmin_answers() { + code=$(curl -6 -k -s -o /dev/null -w '%{http_code}' \ + "https://[$addr]:$BT_WEBMIN_PORT/" || true) + [ "$code" = 200 ] || [ "$code" = 401 ] +} +bt_wait_for "$BT_TIMEOUT" "$BT_INTERVAL" "webmin on https://[$addr]:$BT_WEBMIN_PORT/" \ + webmin_answers +bt_webmin_verdict "$code" + +# 9. No drift between the declared description and the booted root. +set +e +keel diff --root "$BT_ROOTFS" --spec "$BT_SPEC" +code=$? +set -e +bt_diff_verdict "$code" +log "$BT_APPLIANCE boot test passed" diff --git a/tests/coverage.sh b/tests/coverage.sh new file mode 100755 index 0000000..3beedf1 --- /dev/null +++ b/tests/coverage.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Line coverage of the shell this layer writes, measured with kcov over the +# bats suite (decision 0004). The measured files are the first boot library +# (lib/postgresql.sh), the first boot hook this layer adds +# (firstboot.d/36pgsqlverify) and the logic of the boot test +# (tests/lib/boot-test-lib.sh); the 95 percent bar of decision 0003 applies +# to all three and all three are at 100. Exits 1 below the threshold, 2 when +# a tool is missing. tests/boot-test.sh is the thin main that runs keel and +# LXC as root and is exercised by the container run in test-appliance.yml, +# not measured here; conf.d/main is a build time script and is exercised by +# the build. firstboot.d/35pgsqlpass is not measured here either: it belongs +# to common, which this layer uses rather than rewrites. +# +# tests/coverage.sh [THRESHOLD] +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +threshold="${1:-${COVERAGE_THRESHOLD:-100}}" + +for tool in kcov bats python3; do + if ! command -v "$tool" >/dev/null; then + echo "$tool not found (apt-get install $tool)" >&2 + exit 2 + fi +done + +report="${COVERAGE_DIR:-$(mktemp -d)}" +# The include pattern is the whitelist, so no exclude pattern is needed; an +# exclude of /tests/ would drop tests/lib/boot-test-lib.sh with it. +kcov --include-pattern=/lib/postgresql.sh,/firstboot.d/36pgsqlverify,/tests/lib/boot-test-lib.sh \ + "$report" bats "$here" + +json="$(find "$report" -mindepth 2 -maxdepth 2 -name coverage.json -not -path "*/kcov-merged/*" | head -1)" +echo +echo "kcov line coverage (threshold $threshold percent):" +awk -F'"' -v threshold="$threshold" ' + /^ *\{"file":/ { + n = split($4, parts, "/") + printf "%7.2f %s/%s %s", $8, $12, $16, parts[n] + if ($8 + 0 < threshold) { printf " BELOW THRESHOLD"; below = 1 } + printf "\n" + seen = 1 + } + END { + if (!seen) { print "no file measured"; exit 1 } + exit below + }' "$json" diff --git a/tests/hook.bats b/tests/hook.bats new file mode 100644 index 0000000..7f664a1 --- /dev/null +++ b/tests/hook.bats @@ -0,0 +1,143 @@ +#!/usr/bin/env bats +# The first boot hook firstboot.d/36pgsqlverify (decision 0004): every path +# it takes runs for real against scratch directories, and every command +# that would touch the system (systemctl, pg_isready, psql) is a PATH stub +# that records its arguments. Nothing here needs root, a database or a +# network. +# +# What these tests are about is the half of the password defect that is +# this layer's: common's 35pgsqlpass already reads DB_PASS and sets the +# role, and nothing said whether the database would accept it. keel diff +# never compares secrets, on either side, so on an appliance whose whole +# purpose is the database a password that did not arrive would have been +# silent. + +bats_require_minimum_version 1.5.0 + +setup() { + ROOT="$BATS_TEST_DIRNAME/.." + HOOK="$ROOT/overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify" + scratch="$BATS_TEST_TMPDIR/hook" + mkdir -p "$scratch/bin" "$scratch/inithooks/bin" + # the library is the real file, not a copy: kcov measures the one the + # layer ships, and a copy per test would be measured as its own + # uncovered file + ln -s "$(cd "$ROOT/overlay/usr/lib/inithooks/lib" && pwd)" "$scratch/inithooks/lib" + + export INITHOOKS_DEFAULT="$scratch/default-inithooks" + export INITHOOKS_CONF="$scratch/inithooks.conf" + export CALLS="$scratch/calls" + export PGSQL_SLEEP=: + + cat > "$INITHOOKS_DEFAULT" <> "$CALLS"' + stub pg_isready 'echo "pg_isready $*" >> "$CALLS"' + # the client the hook proves the password with: it answers the probe + # only for the password the description declared, the way a server does + stub psql 'echo "psql $* PGPASSWORD=$PGPASSWORD" >> "$CALLS" +if [ "$PGPASSWORD" = "s3cret-from-the-description" ]; then echo 1; else +echo "psql: error: connection failed: password authentication failed" >&2; exit 2; fi' + PATH="$scratch/bin:$PATH" +} + +stub() { + printf '#!/bin/sh\n%s\n' "$2" > "$scratch/bin/$1" + chmod +x "$scratch/bin/$1" +} + +DECLARED_PASS=s3cret-from-the-description + +# write_conf [EXTRA_LINE...]: the conf a declared description renders to +write_conf() { + printf 'export DB_PASS=%s\n' "$DECLARED_PASS" > "$INITHOOKS_CONF" + printf '%s\n' "$@" >> "$INITHOOKS_CONF" +} + +@test "the declared password is proved against the database" { + write_conf + run "$HOOK" + [ "$status" -eq 0 ] + grep -q -- "psql --username=postgres --host=::1 --port=5432 --dbname=postgres" "$CALLS" + grep -q -- "PGPASSWORD=$DECLARED_PASS" "$CALLS" + [[ "$output" == *"set from DB_PASS (${#DECLARED_PASS} characters)"* ]] + [[ "$output" == *"verified on [::1]:5432"* ]] + [[ "$output" != *"$DECLARED_PASS"* ]] +} + +@test "a password the database refuses fails the hook" { + write_conf + DECLARED_PASS=not-the-one-the-server-has + write_conf + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"cannot authenticate on [::1]:5432 with the declared password"* ]] +} + +@test "a server that answers something else fails the hook" { + write_conf + stub psql 'echo "surprise"' + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"answered 'surprise', not '1'"* ]] +} + +@test "nothing declared: the hook names the field, and never prompts" { + printf 'export HOSTNAME=db\n' > "$INITHOOKS_CONF" + run "$HOOK" < /dev/null + [ "$status" -eq 1 ] + [[ "$output" == *"no DB_PASS in $INITHOOKS_CONF"* ]] + [[ "$output" == *"declare secrets.db_password in the instance description"* ]] + [ ! -f "$CALLS" ] +} + +@test "no conf at all is the same failure" { + rm -f "$INITHOOKS_CONF" + run "$HOOK" < /dev/null + [ "$status" -eq 1 ] + [[ "$output" == *"no DB_PASS"* ]] +} + +@test "PGSQL_PASS in the conf is not a password: it is a build time variable" { + printf 'export PGSQL_PASS=from-the-build\n' > "$INITHOOKS_CONF" + run "$HOOK" < /dev/null + [ "$status" -eq 1 ] + [[ "$output" == *"no DB_PASS"* ]] + [ ! -f "$CALLS" ] || ! grep -q "from-the-build" "$CALLS" +} + +@test "APP_DB_USER names the role the password belongs to" { + write_conf "export APP_DB_USER=lappuser" + run "$HOOK" + [ "$status" -eq 0 ] + grep -q -- "psql --username=lappuser" "$CALLS" +} + +@test "a role name this layer will not verify fails before anything runs" { + write_conf "export APP_DB_USER='rm -rf /'" + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"is not a role name this layer will verify"* ]] + [ ! -f "$CALLS" ] +} + +@test "the cluster is started and waited for on the address it checks" { + write_conf + run "$HOOK" + [ "$status" -eq 0 ] + [ "$(head -1 "$CALLS")" = "systemctl start postgresql.service" ] + [ "$(sed -n 2p "$CALLS")" = "pg_isready --quiet --host=::1 --port=5432" ] +} + +@test "a cluster that never answers fails before the client runs" { + write_conf + stub pg_isready 'exit 1' + export PGSQL_WAIT_TRIES=2 + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"postgresql did not answer on [::1]:5432 after 2 tries"* ]] + ! grep -q psql "$CALLS" +} diff --git a/tests/instance.yaml b/tests/instance.yaml new file mode 100644 index 0000000..54a3d79 --- /dev/null +++ b/tests/instance.yaml @@ -0,0 +1,53 @@ +# Instance description of the boot test (format: docs/spec.md of the keel +# repository). IPv6 first, address from the bridge, no certificate request. +# The two passwords are files the test writes into the rootfs before +# booting, which is the whole point of this test: secrets.db_password is +# declared here, nothing is typed at the console, and after the first boot +# a database client authenticates with exactly that value. +# +# This is the throwaway container the gate boots. The description an +# operator would start from is keel/instance.example.yaml. +version: 1 + +instance: + hostname: postgresql + fqdn: postgresql.example.org + +network: + # The container's address comes from the bridge, so the host owns the + # interface configuration and 01ipconfig writes nothing. + managed_by: host + interfaces: + eth0: + ipv6: + method: auto + +tls: + acme: + enabled: false + +secrets: + root_password: + file: /etc/keel/secrets/root_password + # What this layer exists for. It renders to DB_PASS, which + # firstboot.d/35mysqlpass reads and hands to bin/pgsqlconf.py. + db_password: + file: /etc/keel/secrets/db_password + +app: + options: + # Renders to APP_DB_USER: the role the declared password belongs to. + # It is the role common's conf/pgsql configures and 35pgsqlpass sets, + # named here so an appliance built on this layer sees where it is. + db_user: postgres + +hub: + api_key: skip + +security: + alerts: skip + # A first boot input, not machine state: 95secupdates installs the + # pending security updates once. diff does not compare it. + updates_at_first_boot: skip + +first_login_wizard: false diff --git a/tests/lib/boot-test-lib.sh b/tests/lib/boot-test-lib.sh new file mode 100644 index 0000000..8903fbe --- /dev/null +++ b/tests/lib/boot-test-lib.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# Pure helpers of tests/boot-test.sh (decision 0004: logic apart from +# effect). Same shape as the one in keel-core, keel-nodebb and keel-mariadb, with the +# checks this layer adds: a client connection to the database with the +# password the instance description declared, the Webmin module for that +# database, and Webmin answering over IPv6. Nothing here starts a +# container, writes outside a path it is given or opens a socket. The two +# functions that run a command, bt_now and bt_container_ipv6, take it from +# the environment or from PATH so a test can replace it. Sourced by +# boot-test.sh and by tests/boot-test.bats. +# shellcheck disable=SC2034 # the BT_* variables are read by the caller + +BT_DEFAULT_TIMEOUT=900 +BT_DEFAULT_INTERVAL=5 +BT_DEFAULT_BRIDGE=br0 +BT_DEFAULT_LAYERS_DIR=/mnt/builds/layers +BT_DEFAULT_CACHE_DIR=/var/cache/keel/layers +BT_DEFAULT_LXC_PATH=/var/lib/lxc +BT_SSH_PORT=22 +BT_PASSWORD_LENGTH=24 +BT_RANDOM_BYTES=1024 +# The secrets tests/instance.yaml references, one file each under +# etc/keel/secrets in the rootfs: the root account and the database +# account, which is the one this layer exists for. +BT_SECRETS="root_password db_password" +# The panel core carries and this layer adds a module to. +BT_WEBMIN_PORT=12321 +BT_WEBMIN_MODULE=webmin-postgresql +# The database connection the test proves. The client runs inside the +# container, because the cluster listens on the loopback of both families +# and nowhere else: an appliance above this layer talks to it from the same +# machine, and opening it to the network is that appliance's decision, not +# this layer's. +BT_DB_USER="postgres" +BT_DB_HOST=::1 +BT_DB_PORT=5432 +BT_DB_NAME=postgres +BT_DB_PROBE_QUERY="SELECT 1" +BT_DB_PROBE_ANSWER=1 +# Where the first boot reads the instance description. inithooks reads +# etc/inithooks.yaml (hook 00declarative), keel reads etc/keel/instance.yaml; +# the final name is a maintainer decision (brief section 11), so the test +# installs the same file at both until then. +BT_SPEC_PATHS="etc/keel/instance.yaml etc/inithooks.yaml" + +bt_usage() { + cat <&2 + return 1 + } + [ "$1" = --timeout ] && BT_TIMEOUT=$2 || BT_INTERVAL=$2 + shift + ;; + --bridge|--layers-dir|--cache-dir|--lxc-path|--name|--spec) + [ -n "${2-}" ] || { + echo "boot-test: $1 needs a value" >&2 + return 1 + } + case "$1" in + --bridge) BT_BRIDGE=$2 ;; + --layers-dir) BT_LAYERS_DIR=$2 ;; + --cache-dir) BT_CACHE_DIR=$2 ;; + --lxc-path) BT_LXC_PATH=$2 ;; + --name) BT_NAME=$2 ;; + --spec) BT_SPEC=$2 ;; + esac + shift + ;; + --keep) BT_KEEP=1 ;; + -h|--help) + bt_usage + return 2 + ;; + -*) + echo "boot-test: unknown option $1" >&2 + return 1 + ;; + *) + if [ -n "$BT_APPLIANCE" ]; then + echo "boot-test: one appliance at a time ($BT_APPLIANCE, $1)" >&2 + return 1 + fi + BT_APPLIANCE=$1 + ;; + esac + shift + done + if [ -z "$BT_APPLIANCE" ]; then + echo "boot-test: APPLIANCE is required (core, lamp, ...)" >&2 + return 1 + fi + if ! bt_is_appliance_name "$BT_APPLIANCE"; then + echo "boot-test: '$BT_APPLIANCE' is not an appliance name (lower case, no keel- prefix)" >&2 + return 1 + fi + BT_NAME=${BT_NAME:-$(bt_container_name "$BT_APPLIANCE")} + if ! bt_is_container_name "$BT_NAME"; then + echo "boot-test: '$BT_NAME' is not a container name (lower case, digits, dot, dash)" >&2 + return 1 + fi + BT_ROOTFS=$BT_LXC_PATH/$BT_NAME/rootfs + return 0 +} + +bt_is_global_ipv6() { + # Global unicast, which includes ULA (fc00::/7); not link local + # (fe80::/10), loopback or multicast. IPv4 has no colon. + local addr=${1,,} + [[ $addr == *:* ]] || return 1 + [[ $addr == fe[89ab]?:* ]] && return 1 + [[ $addr == ::1 ]] && return 1 + [[ $addr == ff* ]] && return 1 + return 0 +} + +bt_global_ipv6() { + # stdin: the output of lxc-info -i ("IP: ADDRESS" per line). Prints + # the first global IPv6 address; returns 1 when there is none yet. + local label addr _ + while read -r label addr _; do + [ "$label" = "IP:" ] || continue + if bt_is_global_ipv6 "$addr"; then + printf '%s\n' "$addr" + return 0 + fi + done + return 1 +} + +bt_container_ipv6() { + # bt_container_ipv6 NAME LXCPATH: the container's first global IPv6. + lxc-info -P "$2" -n "$1" -i 2>/dev/null | bt_global_ipv6 +} + +bt_now() { + ${BT_CLOCK:-date +%s} +} + +bt_deadline_passed() { + # bt_deadline_passed START TIMEOUT NOW + [ $(( $3 - $1 )) -ge "$2" ] +} + +bt_wait_for() { + # bt_wait_for TIMEOUT INTERVAL DESCRIPTION COMMAND [ARGS...] + # Runs COMMAND until it succeeds; returns 1 once TIMEOUT seconds passed. + local timeout=$1 interval=$2 what=$3 start now + shift 3 + start=$(bt_now) + until "$@"; do + now=$(bt_now) + if bt_deadline_passed "$start" "$timeout" "$now"; then + echo "boot-test: timeout after ${timeout}s waiting for $what" >&2 + return 1 + fi + ${BT_SLEEP:-sleep} "$interval" + done +} + +bt_is_ssh_banner() { + [[ ${1-} == SSH-2.0-* ]] +} + +bt_firstboot_done_in() { + # bt_firstboot_done_in FILE: FILE is the rootfs copy of + # /etc/default/inithooks; 98finalize sets RUN_FIRSTBOOT=false at the end. + [ -r "$1" ] && grep -q '^RUN_FIRSTBOOT=false' "$1" +} + +bt_lxc_config() { + # bt_lxc_config NAME ROOTFS BRIDGE: an LXC config for a plain rootfs + # directory on a bridge; the address comes from the bridge (SLAAC or + # DHCPv6), the spec declares managed_by: host. + cat <&2 + return 1 + fi + printf '%s\n' "${pool:0:BT_PASSWORD_LENGTH}" +} + +bt_secret_targets() { + # bt_secret_targets ROOTFS: the secret files the spec references. + local name + for name in $BT_SECRETS; do + printf '%s/etc/keel/secrets/%s\n' "$1" "$name" + done +} + +bt_webmin_verdict() { + # bt_webmin_verdict CODE: Webmin comes from core and is the panel this + # layer adds its database module to, so the boot test checks that it + # answers over IPv6 on 12321. It asks for credentials, so 200 (the + # login page) and 401 are both an answer; 000 is curl failing to + # connect at all. + case "${1-}" in + 200|401) + echo "boot-test: webmin answered $1 on port $BT_WEBMIN_PORT" + ;; + *) + echo "boot-test: webmin answered '${1-}' on port $BT_WEBMIN_PORT, not 200 or 401" >&2 + return 1 + ;; + esac +} + +bt_module_verdict() { + # bt_module_verdict PACKAGE STATUS: the Webmin module for this + # database must be installed on the booted machine, not only in the + # plan. Batteries included is a property of the distribution, so a + # panel without its database module is a failed boot test. + if [ "${2-}" = "install ok installed" ]; then + echo "boot-test: $1 is installed" + return 0 + fi + echo "boot-test: $1 is '${2-}', not 'install ok installed'" >&2 + return 1 +} + +bt_db_verdict() { + # bt_db_verdict OUTPUT: what the database client printed when it + # connected with the declared password and ran the probe query. The + # whole point of this appliance is that a declared password reaches + # the database, so the answer is the verdict: anything other than the + # single row the query asks for means the connection did not happen or + # did not authenticate. + local output + output=$(printf '%s' "${1-}" | tr -d '[:space:]') + if [ "$output" = "$BT_DB_PROBE_ANSWER" ]; then + echo "boot-test: $BT_DB_USER authenticated on [$BT_DB_HOST]:$BT_DB_PORT with the declared password" + return 0 + fi + echo "boot-test: the database client answered '${1-}', not '$BT_DB_PROBE_ANSWER': the declared password did not reach the database" >&2 + return 1 +} + +bt_db_client_argv() { + # bt_db_client_argv USER HOST PORT [DATABASE]: the client command the + # boot test runs inside the container, one argument per line. The + # password is not here: it goes through PGPASSWORD in the environment, + # so it never appears in the container's process list. --no-password + # makes psql fail instead of prompting, so a password that did not + # arrive is an error and not a hung test. + local user=$1 host=$2 port=$3 database=${4-$BT_DB_NAME} + [ -n "$user" ] && [ -n "$host" ] && [ -n "$database" ] || return 1 + case "$port" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' psql "--username=$user" "--host=$host" "--port=$port" \ + "--dbname=$database" --no-password --tuples-only --no-align --quiet \ + "--command=$BT_DB_PROBE_QUERY" +} +bt_diff_verdict() { + # bt_diff_verdict CODE: interprets the exit code of keel diff + # (docs/diff.md of the keel repository). 0 and 13 mean no drift. + case "$1" in + 0) echo "keel diff: no drift"; return 0 ;; + 13) echo "keel diff: no drift, but a declared field could not be observed offline (see the report above)"; return 0 ;; + 14) echo "keel diff: drift found" >&2; return 1 ;; + 2|3) echo "keel diff: the spec is unreadable or invalid (exit $1)" >&2; return 1 ;; + *) echo "keel diff: failed with exit $1" >&2; return 1 ;; + esac +} diff --git a/tests/postgresql.bats b/tests/postgresql.bats new file mode 100644 index 0000000..1f6f164 --- /dev/null +++ b/tests/postgresql.bats @@ -0,0 +1,99 @@ +#!/usr/bin/env bats +# Unit tests of overlay/usr/lib/inithooks/lib/postgresql.sh, the logic +# behind the first boot hook 36pgsqlverify (decision 0004). Every function +# is pure: nothing here needs a database, root or a network. + +bats_require_minimum_version 1.5.0 + +setup() { + load ../overlay/usr/lib/inithooks/lib/postgresql.sh +} + +@test "first_value: the first value that is set and is not DEFAULT" { + [ "$(pgsql_first_value "" DEFAULT keeper other)" = keeper ] + [ "$(pgsql_first_value default keeper)" = keeper ] + [ "$(pgsql_first_value Default keeper)" = keeper ] + [ "$(pgsql_first_value first second)" = first ] + run ! pgsql_first_value "" DEFAULT "" + run ! pgsql_first_value +} + +@test "role: the cluster superuser, or the one APP_DB_USER names" { + [ "$(pgsql_role)" = postgres ] + [ "$(pgsql_role "")" = postgres ] + [ "$(pgsql_role DEFAULT)" = postgres ] + [ "$(pgsql_role lappuser)" = lappuser ] + PGSQL_ROLE=dba + [ "$(pgsql_role)" = dba ] +} + +@test "role: a name that would need quoting is refused" { + run ! pgsql_role "rm -rf /" + run ! pgsql_role '"postgres"' + run ! pgsql_role "9lives" + run ! pgsql_role "a b" + run ! pgsql_role "pg;sql" +} + +@test "is_role_name: what this layer will verify" { + pgsql_is_role_name postgres + pgsql_is_role_name _pg + pgsql_is_role_name lapp-user + pgsql_is_role_name a1 + run ! pgsql_is_role_name "" + run ! pgsql_is_role_name "1a" + run ! pgsql_is_role_name "a%" +} + +@test "missing_values: DB_PASS only when the description declared none" { + [ -z "$(pgsql_missing_values s3cret)" ] + [ "$(pgsql_missing_values "")" = DB_PASS ] + [ "$(pgsql_missing_values)" = DB_PASS ] +} + +@test "verify_argv: the psql call that proves the password, without it" { + output=$(pgsql_verify_argv postgres ::1 5432 postgres) + [ "$output" = $'--username=postgres\n--host=::1\n--port=5432\n--dbname=postgres\n--no-password\n--tuples-only\n--no-align\n--quiet\n--command=SELECT 1' ] + [[ $output != *s3cret* ]] + [[ $output == *"--no-password"* ]] +} + +@test "verify_argv: a role, a host, a positive port and a database are needed" { + run ! pgsql_verify_argv "" ::1 5432 postgres + run ! pgsql_verify_argv "a b" ::1 5432 postgres + run ! pgsql_verify_argv postgres "" 5432 postgres + run ! pgsql_verify_argv postgres ::1 "" postgres + run ! pgsql_verify_argv postgres ::1 0 postgres + run ! pgsql_verify_argv postgres ::1 54o2 postgres + run ! pgsql_verify_argv postgres ::1 5432 "" +} + +@test "probe_verdict: the answer the query asks for, whitespace aside" { + pgsql_probe_verdict 1 + pgsql_probe_verdict $'\n 1 \n' + run ! pgsql_probe_verdict 0 + run ! pgsql_probe_verdict "" + run ! pgsql_probe_verdict + run ! pgsql_probe_verdict "psql: error: connection to server failed" +} + +@test "wait_ready: returns as soon as the command succeeds" { + PGSQL_SLEEP=: + attempts=0 + probe() { attempts=$((attempts + 1)); [ "$attempts" -ge 3 ]; } + pgsql_wait_ready probe 10 + [ "$attempts" -eq 3 ] +} + +@test "wait_ready: gives up after the last try" { + PGSQL_SLEEP=: + never() { return 1; } + run ! pgsql_wait_ready never 4 +} + +@test "masked: a length, never the password" { + [ "$(pgsql_masked s3cret)" = "(6 characters)" ] + [ "$(pgsql_masked "")" = "(none)" ] + [ "$(pgsql_masked)" = "(none)" ] + [[ "$(pgsql_masked s3cret)" != *s3cret* ]] +}