From 2d096e2e194af9c0fa769bfb38eaf090ccb56368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20M=C3=A9ndez?= Date: Sun, 27 Sep 2026 01:37:18 +0000 Subject: [PATCH] A boot test that proves the declared password, and the coverage gate The gate this repository was bootstrapped with measured nothing: threshold 0 is test-shell.yml's bootstrap case, which exists so main can be protected before the tests do. This raises it to the measured number and adds the acceptance test. What a boot test has to prove here. An appliance whose whole purpose is a database is not proved by a listening port: the database listens whatever password it ended up with, and keel diff never compares secrets, on either side, so nothing else would notice a declared password that did not arrive. So tests/boot-test.sh runs the declarative path end to end and the verdict is a client connection: 1. keel pull and keel assemble the published chain (core, postgresql) into an LXC rootfs; 2. a random password is written to etc/keel/secrets/db_password (0600) and tests/instance.yaml, which declares secrets.db_password from that file, is installed at both paths the first boot reads; 3. keel spec apply renders it into etc/inithooks.conf, and nothing else is configured by hand; 4. the container boots and the first boot runs through the inithooks runner alone, which is where common's firstboot.d/35pgsqlpass reads DB_PASS and this layer's 36pgsqlverify checks it; 5. psql --username=postgres --host=::1 --port=5432 --no-password --command='SELECT 1' inside the container, with that password in PGPASSWORD. The password comes from the file written in step 2 and from nowhere else, so a row back is the proof that the description carried it to the database. --no-password makes psql fail rather than prompt, so a password that did not arrive is an error and not a hung test; 6. webmin-postgresql is installed and Webmin answers over IPv6 on 12321, because batteries included is a property of this distribution and a panel without its database module is a failed boot; 7. keel diff --root --spec reports no drift. Its logic is in tests/lib/boot-test-lib.sh, the same shape as keel-core, keel-nodebb and keel-mariadb, with the verdicts this layer adds; boot-test.sh is the thin main that touches the system. Measured, 58 bats tests: overlay/usr/lib/inithooks/lib/postgresql.sh 100.00 (41/41) overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify 100.00 (15/15) tests/lib/boot-test-lib.sh 100.00 (141/141) 100 percent over the three (197/197), which is the threshold. It is only ever raised (decision 0006). The hook tests run 36pgsqlverify for real against scratch directories with every system command stubbed, and the psql stub answers the probe only for the password the description declared, the way a server does. They are about the half of the password defect that is this layer's: the declared password is proved against the database, a password the database refuses fails the hook, an answer that is not the probe's fails it, nothing declared fails naming the field and never prompts, a PGSQL_PASS in the conf is ignored because it is a build time variable, and APP_DB_USER renames the role while a name that would need quoting is refused before anything runs. 35pgsqlpass and bin/pgsqlconf.py belong to the common fork and are measured there. --- .github/workflows/tests.yml | 16 +- COVERAGE.md | 67 +++++++ tests/README.md | 97 ++++++++++ tests/boot-test.bats | 359 ++++++++++++++++++++++++++++++++++++ tests/boot-test.sh | 157 ++++++++++++++++ tests/coverage.sh | 47 +++++ tests/hook.bats | 143 ++++++++++++++ tests/instance.yaml | 53 ++++++ tests/lib/boot-test-lib.sh | 358 +++++++++++++++++++++++++++++++++++ tests/postgresql.bats | 99 ++++++++++ 10 files changed, 1388 insertions(+), 8 deletions(-) create mode 100644 COVERAGE.md create mode 100644 tests/README.md create mode 100644 tests/boot-test.bats create mode 100755 tests/boot-test.sh create mode 100755 tests/coverage.sh create mode 100644 tests/hook.bats create mode 100644 tests/instance.yaml create mode 100644 tests/lib/boot-test-lib.sh create mode 100644 tests/postgresql.bats diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6a2a054..2b36724 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,16 +19,16 @@ permissions: jobs: tests: - # Threshold 0 is the bootstrap case of test-shell.yml: there is no - # tests/coverage.sh on this branch yet, so the job passes with a notice - # and the check exists, which is what lets main be protected from the - # first day. It is raised to the measured number when the tests land, - # and only ever raised (decision 0006). This job produces the check - # "tests / coverage", the required status on main. The job id is part - # of that name, so renaming it silently detaches the protection rule. + # tests/coverage.sh measures three shell files (COVERAGE.md): the + # first boot library, the verification hook this layer adds and the + # boot test's library, all three at 100 percent. The threshold is the + # measured number and is only ever raised (decision 0006). This job + # produces the check "tests / coverage", the required status on main. + # The job id is part of that name, so renaming it silently detaches + # the protection rule. uses: keel-linux/.github/.github/workflows/test-shell.yml@main with: - threshold: 0 + threshold: 100 appliance: # The boot test (org-plan section 1): the postgresql layer is fetched from # https://mirror.keellinux.org/layers, verified, assembled into a diff --git a/COVERAGE.md b/COVERAGE.md new file mode 100644 index 0000000..ce6a18c --- /dev/null +++ b/COVERAGE.md @@ -0,0 +1,67 @@ +# Coverage + +Standard: decisions 0003 (90 percent per repository, 95 for code the +project writes) and 0004 (bats plus kcov for shell; a build and a boot on +LXC as the acceptance test of a recipe, docs/org-plan.md section 1). + +## Measured 2026-09-27 + +| File | Test | Lines | Note | +| --- | --- | --- | --- | +| overlay/usr/lib/inithooks/lib/postgresql.sh | tests/postgresql.bats (11 tests) | 100 percent (41/41) under kcov | every function and every branch | +| overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify | tests/hook.bats (10 tests) | 100 percent (15/15) under kcov | every path, including the two failures that matter | +| tests/lib/boot-test-lib.sh | tests/boot-test.bats (37 tests) | 100 percent (141/141) under kcov | argument parsing, address discovery, deadlines, the database, module, Webmin and diff verdicts | +| conf.d/main | the build | integration only | build time script, 0004 pragmatic limits | +| tests/boot-test.sh | itself | integration only | the thin main of the acceptance test: keel and LXC as root | +| overlay ... firstboot.d/35pgsqlpass | common | not this repository | the hook that sets the password belongs to the `common` fork and is used, not rewritten | + +Total over the three measured shell files: **100 percent (197/197)**, +58 bats tests. `tests/coverage.sh` fails below `COVERAGE_THRESHOLD`, which +the workflow sets to 100, the measured number. It is only ever raised +(decision 0006). + + $ COVERAGE_THRESHOLD=100 tests/coverage.sh + kcov line coverage (threshold 100 percent): + 100.00 41/41 postgresql.sh + 100.00 15/15 36pgsqlverify + 100.00 141/141 boot-test-lib.sh + +## What the hook tests cover + +The hook is executed for real against scratch directories, with PATH stubs +for `systemctl`, `pg_isready` and `psql` under a scratch `INITHOOKS_PATH` +whose `lib` is a symlink to the real library, so kcov measures the file the +layer ships. The `psql` stub answers the probe only for the password the +description declared, the way a server does. No test needs root, a +database or a network. + +What they are about is the half of the password defect that is this +layer's: the declared password is proved against the database, a password +the database refuses fails the hook, an answer that is not the probe's +fails it, nothing declared fails naming the field and never prompts, a +`PGSQL_PASS` in the conf is not a password and is ignored, `APP_DB_USER` +renames the role and a name that would need quoting is refused before +anything runs, and the cluster is started and waited for on the address the +client will use. + +## The appliance gate + +`appliance / build-and-boot` runs through the organization's +`test-appliance.yml` on the self-hosted `keel-lxc` runner, which fetches +the published layer from `https://mirror.keellinux.org/layers`, verifies +it, assembles it, boots it in LXC and runs `tests/boot-test.sh`. Nothing is +built there. + +State on 2026-09-27: **the layer is not published yet, so the job skips +with a notice and passes.** It becomes a required status on `main` the day +the first `postgresql` layer reaches the mirror. + +## Plan + +- Publish the layer, then require `appliance / build-and-boot` on `main`. +- Measure `conf.d/main`. A build time script that runs inside a chroot as + root is the case decision 0003 splits, and what is left here after the + logic moved to `lib/postgresql.sh` is SQL, three assertions about the + cluster configuration and the apt calls. +- `35pgsqlpass` and `bin/pgsqlconf.py` belong to the `common` fork and are + measured there; `common`'s own COVERAGE.md carries that plan. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..ea76424 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,97 @@ +# Tests + +What a test means for a layer recipe is written in `COVERAGE.md`: the +recipe builds, the result boots in an LXC container, its first boot +completes headless from an instance description, the database accepts the +password that description declared, the panel has the module for it, and +the machine matches the description. + +## Layout + +- `boot-test.sh`: the boot test. `test-appliance.yml` (reusable workflow of + `keel-linux/.github`) runs it on the self-hosted LXC runner after pulling + the layers from `https://mirror.keellinux.org/layers` and checking them + with `keel verify`. It is the thin main: assemble, mark the tree as a + container, install the description, the secrets and the conf, start the + container, wait, connect to the database, check Webmin, `keel diff`. It + builds nothing, so it needs no fab, deck or buildtasks. +- `lib/boot-test-lib.sh`: the logic (argument parsing, address discovery + from `lxc-info`, waiting with a deadline, the secret files, the client + call, the database, module and Webmin verdicts, the diff verdict), as + functions with no side effects, per decision 0004. Same shape as the one + in keel-core, keel-nodebb and keel-mariadb. +- `boot-test.bats`: unit tests of that library. `lxc-info` is a stub first + in `PATH`; the clock and `sleep` are functions. No root, no network, no + LXC, no database. +- `postgresql.bats`: unit tests of + `overlay/usr/lib/inithooks/lib/postgresql.sh`, the logic behind the + first boot hook `36pgsqlverify`. +- `hook.bats`: `36pgsqlverify` itself, run for real against scratch + directories with every system command stubbed. The hook that sets the + password, `35pgsqlpass`, belongs to the `common` fork and is tested + there; this layer uses it rather than rewriting it. +- `coverage.sh`: runs the bats suite under kcov and fails when any measured + file is below `COVERAGE_THRESHOLD` (default 100). +- `instance.yaml`: the description the test container boots from. It + declares `secrets.db_password` from a file, which is the point of the + test. + +## Unit tests and coverage + +Debian packages `bats` (1.11) and `kcov` (43); no root: + + bats tests/postgresql.bats + bats tests/hook.bats + bats tests/boot-test.bats + COVERAGE_THRESHOLD=100 tests/coverage.sh + +`COVERAGE_DIR=coverage tests/coverage.sh` keeps the kcov reports. + +## The boot test by hand + +Needs root, `keel` on `PATH`, LXC (`lxc-start`, `lxc-info`, `lxc-attach`, +`lxc-stop`), `curl`, and a bridge with IPv6 router advertisements or +DHCPv6. + + tests/boot-test.sh postgresql --layers-dir https://mirror.keellinux.org/layers \ + --bridge lxcbr0 + +`--layers-dir` is a directory or an http(s) URL, so on the build host it is +`/mnt/builds/layers` and on a runner it is the mirror. The other useful +options are `--bridge`, `--cache-dir`, `--lxc-path`, `--name`, `--timeout` +and `--keep` (leaves the container running; then `lxc-attach -n `). +`tests/boot-test.sh --help` lists them all. + +What it does, in order: + +1. `keel pull` and `keel assemble` the chain (core, postgresql) into + `//rootfs`. +2. Creates `var/lib/turnkey-info/inithooks.service/lxc` in the rootfs, the + marker `bt-container` writes and the one `keel inspect` reads to call + the machine a container (`network.managed_by: host`). +3. Writes a random `root_password` and `db_password` under + `etc/keel/secrets` (mode 0600) and installs `tests/instance.yaml` at + `etc/keel/instance.yaml` and `etc/inithooks.yaml`. +4. Renders the description into the rootfs `etc/inithooks.conf` with + `keel spec apply`, from a copy whose secret references point inside the + rootfs. Without the conf the first boot is not headless: `30rootpass` + would have nothing declared and no terminal, and `36pgsqlverify` + would fail naming the field the description has to declare. +5. Writes an LXC config for that rootfs on the bridge and starts the + container. +6. Waits for a global IPv6 address (`lxc-info -i`), then for the first boot + to finish: `RUN_FIRSTBOOT=false` in the rootfs copy of + `/etc/default/inithooks`, and then confconsole or an SSH banner. +7. **Connects to the database.** `psql --username=postgres --host=::1 + --port=5432 --no-password --command='SELECT 1'` inside the container, + with the declared password in `PGPASSWORD`. `--no-password` makes psql + fail rather than prompt, so a password that did not arrive is an error + and not a hung test. The password comes from the secret + file the test wrote in step 3 and from nowhere else, so a row back is + the proof that the declarative path carried it end to end. A listening + port would prove nothing: the database listens whatever password it + ended up with. +8. Checks `webmin-postgresql` is installed and that Webmin answers over IPv6 on + 12321. +9. Runs `keel diff --root --spec tests/instance.yaml`; exit 0 or + 13 (no drift) passes. diff --git a/tests/boot-test.bats b/tests/boot-test.bats new file mode 100644 index 0000000..152b59e --- /dev/null +++ b/tests/boot-test.bats @@ -0,0 +1,359 @@ +#!/usr/bin/env bats +# Unit tests of tests/lib/boot-test-lib.sh: argument parsing, address +# discovery from lxc-info, waiting with a deadline, the secret files, the +# database, Webmin and module verdicts, and the diff verdict. Nothing here +# needs root, a network, a database or LXC: lxc-info is a stub first in +# PATH, the clock and sleep are functions. + +bats_require_minimum_version 1.5.0 + +setup() { + load lib/boot-test-lib.sh + STUBS=$(mktemp -d) + PATH=$STUBS:$PATH +} + +teardown() { + rm -rf "$STUBS" +} + +stub_lxc_info() { + # stub_lxc_info OUTPUT: lxc-info prints OUTPUT and records its arguments + printf '#!/bin/bash\necho "$*" >> "%s/lxc-info.calls"\ncat <<"OUT"\n%s\nOUT\n' "$STUBS" "$1" > "$STUBS/lxc-info" + chmod +x "$STUBS/lxc-info" +} + +# argument parsing + +@test "parse_args: the appliance alone takes every default" { + bt_parse_args postgresql + [ "$BT_APPLIANCE" = postgresql ] + [ "$BT_TIMEOUT" = 900 ] + [ "$BT_INTERVAL" = 5 ] + [ "$BT_BRIDGE" = br0 ] + [ "$BT_LAYERS_DIR" = /mnt/builds/layers ] + [ "$BT_CACHE_DIR" = /var/cache/keel/layers ] + [ "$BT_LXC_PATH" = /var/lib/lxc ] + [ -z "$BT_SPEC" ] + [ "$BT_KEEP" = 0 ] + [ "$BT_NAME" = keel-postgresql-boot-test ] + [ "$BT_ROOTFS" = /var/lib/lxc/keel-postgresql-boot-test/rootfs ] +} + +@test "parse_args: every option is read" { + bt_parse_args --timeout 60 --interval 2 --bridge lxcbr0 --layers-dir /l \ + --cache-dir /c --lxc-path /x --name lamp-run-7 --spec /s.yaml --keep lamp + [ "$BT_APPLIANCE" = lamp ] + [ "$BT_TIMEOUT" = 60 ] + [ "$BT_INTERVAL" = 2 ] + [ "$BT_BRIDGE" = lxcbr0 ] + [ "$BT_LAYERS_DIR" = /l ] + [ "$BT_CACHE_DIR" = /c ] + [ "$BT_LXC_PATH" = /x ] + [ "$BT_SPEC" = /s.yaml ] + [ "$BT_KEEP" = 1 ] + [ "$BT_NAME" = lamp-run-7 ] + [ "$BT_ROOTFS" = /x/lamp-run-7/rootfs ] +} + +@test "parse_args: --name is checked as a container name" { + run bt_parse_args postgresql --name "Run 7" + [ "$status" -eq 1 ] + [[ $output == *"is not a container name"* ]] + run bt_parse_args postgresql --name -lead + [ "$status" -eq 1 ] +} + +@test "is_container_name" { + bt_is_container_name keel-postgresql-ci-36255612491-1 + bt_is_container_name 7 + run ! bt_is_container_name "keel core" + run ! bt_is_container_name -x + run ! bt_is_container_name "" +} + +@test "parse_args: the appliance is required" { + run bt_parse_args --keep + [ "$status" -eq 1 ] + [[ $output == *"APPLIANCE is required"* ]] +} + +@test "parse_args: one appliance at a time" { + run bt_parse_args postgresql lamp + [ "$status" -eq 1 ] + [[ $output == *"one appliance at a time"* ]] +} + +@test "parse_args: the keel- prefix and upper case are rejected" { + run bt_parse_args keel-postgresql + [ "$status" -eq 1 ] + [[ $output == *"not an appliance name"* ]] + run bt_parse_args NodeBB + [ "$status" -eq 1 ] +} + +@test "parse_args: an unknown option fails" { + run bt_parse_args postgresql --verbose + [ "$status" -eq 1 ] + [[ $output == *"unknown option --verbose"* ]] +} + +@test "parse_args: timeout and interval must be positive integers" { + run bt_parse_args postgresql --timeout 0 + [ "$status" -eq 1 ] + [[ $output == *"--timeout needs a positive number"* ]] + run bt_parse_args postgresql --interval abc + [ "$status" -eq 1 ] + run bt_parse_args postgresql --timeout + [ "$status" -eq 1 ] +} + +@test "parse_args: an option with a value refuses an empty one" { + run bt_parse_args postgresql --bridge + [ "$status" -eq 1 ] + [[ $output == *"--bridge needs a value"* ]] + run bt_parse_args postgresql --spec "" + [ "$status" -eq 1 ] +} + +@test "parse_args: --help prints the usage and returns 2" { + run bt_parse_args --help + [ "$status" -eq 2 ] + [[ ${lines[0]} == "usage: tests/boot-test.sh APPLIANCE"* ]] + [[ $output == *"--keep"* ]] + run bt_parse_args -h + [ "$status" -eq 2 ] +} + +@test "is_positive_int and is_appliance_name" { + bt_is_positive_int 1 + bt_is_positive_int 900 + run ! bt_is_positive_int 0 + run ! bt_is_positive_int 07 + run ! bt_is_positive_int -5 + run ! bt_is_positive_int "" + bt_is_appliance_name nginx-php-fastcgi + run ! bt_is_appliance_name keel-core + run ! bt_is_appliance_name 9core + run ! bt_is_appliance_name "" +} + +# address discovery + +@test "is_global_ipv6: global and ULA yes, link local, loopback, multicast, IPv4 no" { + bt_is_global_ipv6 2001:db8:1::10 + bt_is_global_ipv6 fd00:1::10 + bt_is_global_ipv6 2001:DB8::1 + run ! bt_is_global_ipv6 fe80::216:3eff:fe00:1 + run ! bt_is_global_ipv6 FEBF::1 + run ! bt_is_global_ipv6 ::1 + run ! bt_is_global_ipv6 ff02::1 + run ! bt_is_global_ipv6 192.0.2.10 + run ! bt_is_global_ipv6 "" +} + +@test "global_ipv6: picks the first global address out of lxc-info output" { + output=$(printf 'IP: fe80::216:3eff:fe00:1\nIP: 192.0.2.10\nIP: 2001:db8:1::10\nIP: 2001:db8:1::11\n' | bt_global_ipv6) + [ "$output" = 2001:db8:1::10 ] +} + +@test "global_ipv6: ignores lines that are not addresses" { + output=$(printf 'Name: keel-postgresql-boot-test\nState: RUNNING\nPID: 4242\nIP: fd00::10\nLink: veth0\n' | bt_global_ipv6) + [ "$output" = fd00::10 ] +} + +@test "global_ipv6: returns 1 while only link local or IPv4 addresses exist" { + run bt_global_ipv6 <<< $'IP: fe80::1\nIP: 192.0.2.10' + [ "$status" -eq 1 ] + [ -z "$output" ] + run bt_global_ipv6 < /dev/null + [ "$status" -eq 1 ] +} + +@test "container_ipv6: calls lxc-info with the lxcpath and the name" { + stub_lxc_info $'Name: keel-postgresql-boot-test\nIP: fe80::1\nIP: 2001:db8:1::10' + output=$(bt_container_ipv6 keel-postgresql-boot-test /var/lib/lxc) + [ "$output" = 2001:db8:1::10 ] + [ "$(cat "$STUBS/lxc-info.calls")" = "-P /var/lib/lxc -n keel-postgresql-boot-test -i" ] +} + +@test "container_ipv6: fails when lxc-info has no global address yet" { + stub_lxc_info $'Name: keel-postgresql-boot-test\nState: RUNNING' + run bt_container_ipv6 keel-postgresql-boot-test /var/lib/lxc + [ "$status" -eq 1 ] +} + +# timeouts + +fake_clock() { echo "$FAKE_NOW"; } +fake_sleep() { FAKE_NOW=$(( FAKE_NOW + $1 )); echo "sleep $1" >> "$STUBS/sleeps"; } +succeed_on_third() { CALLS=$(( CALLS + 1 )); [ "$CALLS" -ge 3 ]; } +never() { return 1; } + +@test "deadline_passed" { + run ! bt_deadline_passed 100 30 129 + bt_deadline_passed 100 30 130 + bt_deadline_passed 100 30 500 +} + +@test "now: the default clock is epoch seconds and BT_CLOCK replaces it" { + [[ $(bt_now) =~ ^[0-9]{10}$ ]] + BT_CLOCK=fake_clock FAKE_NOW=42 + [ "$(bt_now)" = 42 ] +} + +@test "wait_for: polls at the interval until the command succeeds" { + BT_CLOCK=fake_clock BT_SLEEP=fake_sleep FAKE_NOW=1000 CALLS=0 + bt_wait_for 60 5 "three calls" succeed_on_third + [ "$CALLS" -eq 3 ] + [ "$(cat "$STUBS/sleeps")" = $'sleep 5\nsleep 5' ] +} + +@test "wait_for: gives up with a message once the timeout has passed" { + # shellcheck disable=SC2034 # read by bt_now and bt_wait_for + BT_CLOCK=fake_clock BT_SLEEP=fake_sleep FAKE_NOW=1000 + run bt_wait_for 12 5 "something that never happens" never + [ "$status" -eq 1 ] + [[ $output == *"timeout after 12s waiting for something that never happens"* ]] + [ "$(wc -l < "$STUBS/sleeps")" -eq 3 ] +} + +# readiness and verdicts + +@test "is_ssh_banner" { + bt_is_ssh_banner "SSH-2.0-OpenSSH_10.0p2 Debian-7" + run ! bt_is_ssh_banner "HTTP/1.1 400 Bad Request" + run ! bt_is_ssh_banner "" +} + +@test "firstboot_done_in: RUN_FIRSTBOOT=false in the rootfs copy of /etc/default/inithooks" { + printf 'INITHOOKS_CONF=/etc/inithooks.conf\nRUN_FIRSTBOOT=false\n' > "$STUBS/done" + printf 'RUN_FIRSTBOOT=true\n' > "$STUBS/pending" + bt_firstboot_done_in "$STUBS/done" + run ! bt_firstboot_done_in "$STUBS/pending" + run ! bt_firstboot_done_in "$STUBS/missing" +} + +@test "lxc_config: names the container, the rootfs and the bridge" { + output=$(bt_lxc_config keel-postgresql-boot-test /var/lib/lxc/keel-postgresql-boot-test/rootfs br0) + [[ $output == *"lxc.uts.name = keel-postgresql-boot-test"* ]] + [[ $output == *"lxc.rootfs.path = dir:/var/lib/lxc/keel-postgresql-boot-test/rootfs"* ]] + [[ $output == *"lxc.net.0.link = br0"* ]] + [[ $output == *"lxc.net.0.type = veth"* ]] +} + +@test "spec_targets: both paths the first boot reads, under the rootfs" { + output=$(bt_spec_targets /r) + [ "$output" = $'/r/etc/keel/instance.yaml\n/r/etc/inithooks.yaml' ] +} + +@test "secret_targets: the two secret files the spec references" { + output=$(bt_secret_targets /r) + [ "$output" = $'/r/etc/keel/secrets/root_password\n/r/etc/keel/secrets/db_password' ] +} + +@test "webmin_verdict: the login page or a challenge is an answer" { + run bt_webmin_verdict 200 + [ "$status" -eq 0 ] + [[ $output == *"webmin answered 200 on port 12321"* ]] + run bt_webmin_verdict 401 + [ "$status" -eq 0 ] + run bt_webmin_verdict 000 + [ "$status" -eq 1 ] + [[ $output == *"not 200 or 401"* ]] + run bt_webmin_verdict 502 + [ "$status" -eq 1 ] + run bt_webmin_verdict + [ "$status" -eq 1 ] + [[ $output == *"answered ''"* ]] +} + +@test "module_verdict: the database module must be installed on the machine" { + run bt_module_verdict webmin-postgresql "install ok installed" + [ "$status" -eq 0 ] + [[ $output == *"webmin-postgresql is installed"* ]] + run bt_module_verdict webmin-postgresql "install ok unpacked" + [ "$status" -eq 1 ] + [[ $output == *"is 'install ok unpacked'"* ]] + run bt_module_verdict webmin-postgresql + [ "$status" -eq 1 ] + [[ $output == *"is ''"* ]] +} + +@test "db_verdict: the probe answer passes, anything else is a refusal" { + run bt_db_verdict 1 + [ "$status" -eq 0 ] + [[ $output == *"postgres authenticated on [::1]:5432 with the declared password"* ]] + run bt_db_verdict $'\n1\n' + [ "$status" -eq 0 ] + run bt_db_verdict "psql: error: connection failed: password authentication failed" + [ "$status" -eq 1 ] + [[ $output == *"did not reach the database"* ]] + run bt_db_verdict + [ "$status" -eq 1 ] + [[ $output == *"answered ''"* ]] +} + +@test "db_client_argv: the client call, with no password on the line" { + output=$(bt_db_client_argv postgres ::1 5432) + [ "$output" = $'psql\n--username=postgres\n--host=::1\n--port=5432\n--dbname=postgres\n--no-password\n--tuples-only\n--no-align\n--quiet\n--command=SELECT 1' ] + [[ $output != *"PGPASSWORD"* ]] + [[ $output == *"--no-password"* ]] +} + +@test "db_client_argv: another database can be named" { + output=$(bt_db_client_argv postgres ::1 5432 template1) + [[ $output == *"--dbname=template1"* ]] +} + +@test "db_client_argv: an empty user, host, database or a bad port fails" { + run ! bt_db_client_argv "" ::1 5432 + run ! bt_db_client_argv postgres "" 5432 + run ! bt_db_client_argv postgres ::1 "" + run ! bt_db_client_argv postgres ::1 fivefourthreetwo + run ! bt_db_client_argv postgres ::1 5432 "" +} + +@test "spec_in_rootfs: secret references are pointed inside the rootfs" { + printf 'secrets:\n root_password:\n file: /etc/keel/secrets/root_password\ntls:\n acme:\n enabled: false\n' > "$STUBS/spec" + output=$(bt_spec_in_rootfs "$STUBS/spec" /r/rootfs) + [[ $output == *"file: /r/rootfs/etc/keel/secrets/root_password"* ]] + [[ $output == *"enabled: false"* ]] + [[ $output != *"file: /etc/keel"* ]] +} + +@test "random_password: 24 alphanumeric characters from the random source" { + output=$(bt_random_password) + [[ $output =~ ^[A-Za-z0-9]{24}$ ]] + printf 'ab!!cd%%%%efghijklmnopqrstuvwxyz0123456789' > "$STUBS/random" + output=$(BT_RANDOM_SOURCE=$STUBS/random bt_random_password) + [ "$output" = abcdefghijklmnopqrstuvwx ] +} + +@test "random_password: a source too poor to fill the password fails loudly" { + printf '!!!!short!!!!' > "$STUBS/poor" + BT_RANDOM_SOURCE="$STUBS/poor" + run bt_random_password + [ "$status" -eq 1 ] + [[ $output == *"gave only 5 usable characters"* ]] +} + +@test "diff_verdict: 0 and 13 pass, everything else fails with a message" { + run bt_diff_verdict 0 + [ "$status" -eq 0 ] + [ "$output" = "keel diff: no drift" ] + run bt_diff_verdict 13 + [ "$status" -eq 0 ] + [[ $output == *"could not be observed offline"* ]] + run bt_diff_verdict 14 + [ "$status" -eq 1 ] + [[ $output == *"drift found"* ]] + run bt_diff_verdict 2 + [ "$status" -eq 1 ] + [[ $output == *"unreadable or invalid (exit 2)"* ]] + run bt_diff_verdict 3 + [ "$status" -eq 1 ] + run bt_diff_verdict 127 + [ "$status" -eq 1 ] + [[ $output == *"failed with exit 127"* ]] +} diff --git a/tests/boot-test.sh b/tests/boot-test.sh new file mode 100755 index 0000000..47d54cf --- /dev/null +++ b/tests/boot-test.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Boot test of this layer (org-plan section 1), modelled on the one in +# keel-nodebb and keel-mariadb: assemble the published layer chain into an LXC rootfs, boot +# it headless from tests/instance.yaml, wait for the first boot to finish, +# then prove the declarative path end to end. The instance description +# declares secrets.db_password from a file; nothing is configured by hand; +# and the test connects to the database as a client, with that password, to +# say whether it arrived. A listening port would prove nothing here: the +# database listens whatever password it ended up with. +# +# It also checks what the panel offers, because batteries included is a +# property of this distribution: the Webmin module for this database is +# installed and Webmin answers over IPv6 on 12321. +# +# Called by the reusable workflow test-appliance.yml after keel pull and +# keel verify; runnable by hand as root on any host with LXC, see +# tests/README.md. It builds nothing: the layers come from the mirror or +# from a directory bt-layer wrote, so the test needs no fab, deck or +# buildtasks. The logic lives in tests/lib/boot-test-lib.sh and is unit +# tested; this file is the thin main that touches the system. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=lib/boot-test-lib.sh +source "$here/lib/boot-test-lib.sh" + +bt_parse_args "$@" || { rc=$?; [ "$rc" -eq 2 ] && exit 0; exit 1; } +BT_SPEC=${BT_SPEC:-$here/instance.yaml} +if [ "$(id -u)" -ne 0 ]; then + echo "boot-test: must run as root (keel assemble, lxc-start)" >&2 + exit 1 +fi +for tool in keel lxc-start lxc-info lxc-attach lxc-stop curl; do + command -v "$tool" >/dev/null || { echo "boot-test: $tool not found" >&2; exit 1; } +done + +container_dir=$BT_LXC_PATH/$BT_NAME +log() { printf '%s boot-test: %s\n' "$(date -u +%H:%M:%S)" "$*"; } +lxc() { "lxc-$1" -P "$BT_LXC_PATH" -n "$BT_NAME" "${@:2}"; } + +cleanup() { + local rc=$? + if [ "$rc" -ne 0 ] && [ -r "$BT_ROOTFS/var/log/inithooks.log" ]; then + log "last lines of the container's inithooks log:" + tail -n 40 "$BT_ROOTFS/var/log/inithooks.log" + fi + if [ "$BT_KEEP" -eq 1 ]; then + log "keeping $BT_NAME under $BT_LXC_PATH (--keep); lxc-attach -P $BT_LXC_PATH -n $BT_NAME" + return + fi + lxc stop -k >/dev/null 2>&1 || true + rm -rf "$container_dir" +} +trap cleanup EXIT + +# 1. Assemble the chain from the layers the build host published. +log "assembling $BT_APPLIANCE from $BT_LAYERS_DIR into $BT_ROOTFS" +lxc stop -k >/dev/null 2>&1 || true +rm -rf "$container_dir" +mkdir -p "$BT_ROOTFS" +keel pull "$BT_APPLIANCE" --source "$BT_LAYERS_DIR" --cache-dir "$BT_CACHE_DIR" --non-interactive +keel assemble "$BT_APPLIANCE" --rootfs "$BT_ROOTFS" --cache-dir "$BT_CACHE_DIR" --non-interactive + +# 2. The container marker, the instance description, the secrets it +# references and the conf the first boot hooks read. The marker under +# /var/lib/turnkey-info is what bt-container writes and what inspect +# reads to call the machine a container (managed_by: host); the conf is +# what makes the first boot headless, and without it 30rootpass and +# 35pgsqlpass wait on a dialog forever. +log "installing the description, the secrets and the conf into $BT_ROOTFS" +install -D -m 0644 /dev/null "$BT_ROOTFS/var/lib/turnkey-info/inithooks.service/lxc" +install -d -m 0700 "$BT_ROOTFS/etc/keel/secrets" +for target in $(bt_secret_targets "$BT_ROOTFS"); do + bt_random_password > "$target" + chmod 0600 "$target" +done +for target in $(bt_spec_targets "$BT_ROOTFS"); do + install -D -m 0600 "$BT_SPEC" "$target" +done +bt_spec_in_rootfs "$BT_SPEC" "$BT_ROOTFS" > "$container_dir/instance-host.yaml" +keel spec apply --spec "$container_dir/instance-host.yaml" \ + --conf "$BT_ROOTFS/etc/inithooks.conf" --non-interactive + +# The declared password, as the description names it. Everything below +# uses this and nothing else: no value is read out of the container. +declared_password=$(cat "$BT_ROOTFS/etc/keel/secrets/db_password") + +# 3. Boot. +bt_lxc_config "$BT_NAME" "$BT_ROOTFS" "$BT_BRIDGE" > "$container_dir/config" +log "starting $BT_NAME on bridge $BT_BRIDGE" +lxc start -d + +# 4. A global IPv6 address from the bridge. +bt_wait_for "$BT_TIMEOUT" "$BT_INTERVAL" "a global IPv6 address on $BT_NAME" \ + bt_container_ipv6 "$BT_NAME" "$BT_LXC_PATH" > /dev/null +addr=$(bt_container_ipv6 "$BT_NAME" "$BT_LXC_PATH") +log "container address $addr" + +# 5. First boot finished: 98finalize has cleared RUN_FIRSTBOOT and the +# machine answers, on the console (confconsole's usage screen) or on +# SSH. The answer alone is not enough: sshd is up long before the hooks +# are done, so the flag is what says the first boot ended. +usage_screen() { + lxc attach -- pgrep -f confconsole > /dev/null 2>&1 +} +ssh_answers() { + local banner + banner=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && read -r -t 5 line <&3 && printf "%s" "$line"' \ + "$addr" "$BT_SSH_PORT" 2>/dev/null) || return 1 + bt_is_ssh_banner "$banner" +} +first_boot_done() { + bt_firstboot_done_in "$BT_ROOTFS/etc/default/inithooks" || return 1 + usage_screen || ssh_answers +} +bt_wait_for "$BT_TIMEOUT" "$BT_INTERVAL" "the first boot of $BT_NAME to finish" \ + first_boot_done +log "first boot finished; ssh root@$addr" + +# 6. What the first boot hook reported, quoted here so a failure below is +# read next to it. +log "the database lines of the container's inithooks log:" +grep -E '35pgsqlpass|36pgsqlverify|PostgreSQL' "$BT_ROOTFS/var/log/inithooks.log" || true + +# 7. The declarative path, end to end: a client connection to the database +# with the password the description declared. The client runs inside the +# container over TCP on [::1], because the cluster listens on the +# loopback of both families and nowhere else, and the password reaches +# it in the environment so it never appears in the container's process +# list. --no-password makes psql fail rather than prompt, so a password +# that did not arrive is an error and not a hung test. +mapfile -t client < <(bt_db_client_argv "$BT_DB_USER" "$BT_DB_HOST" "$BT_DB_PORT") +log "connecting as $BT_DB_USER on [$BT_DB_HOST]:$BT_DB_PORT with the declared password" +answer=$(lxc attach --set-var "PGPASSWORD=$declared_password" -- "${client[@]}") \ + || { echo "boot-test: the database refused the declared password" >&2; exit 1; } +bt_db_verdict "$answer" + +# 8. The panel core carries, with the module this layer adds to it. +status=$(lxc attach -- dpkg-query -W -f '${Status}' "$BT_WEBMIN_MODULE" 2>/dev/null || true) +bt_module_verdict "$BT_WEBMIN_MODULE" "$status" +code="" +webmin_answers() { + code=$(curl -6 -k -s -o /dev/null -w '%{http_code}' \ + "https://[$addr]:$BT_WEBMIN_PORT/" || true) + [ "$code" = 200 ] || [ "$code" = 401 ] +} +bt_wait_for "$BT_TIMEOUT" "$BT_INTERVAL" "webmin on https://[$addr]:$BT_WEBMIN_PORT/" \ + webmin_answers +bt_webmin_verdict "$code" + +# 9. No drift between the declared description and the booted root. +set +e +keel diff --root "$BT_ROOTFS" --spec "$BT_SPEC" +code=$? +set -e +bt_diff_verdict "$code" +log "$BT_APPLIANCE boot test passed" diff --git a/tests/coverage.sh b/tests/coverage.sh new file mode 100755 index 0000000..3beedf1 --- /dev/null +++ b/tests/coverage.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Line coverage of the shell this layer writes, measured with kcov over the +# bats suite (decision 0004). The measured files are the first boot library +# (lib/postgresql.sh), the first boot hook this layer adds +# (firstboot.d/36pgsqlverify) and the logic of the boot test +# (tests/lib/boot-test-lib.sh); the 95 percent bar of decision 0003 applies +# to all three and all three are at 100. Exits 1 below the threshold, 2 when +# a tool is missing. tests/boot-test.sh is the thin main that runs keel and +# LXC as root and is exercised by the container run in test-appliance.yml, +# not measured here; conf.d/main is a build time script and is exercised by +# the build. firstboot.d/35pgsqlpass is not measured here either: it belongs +# to common, which this layer uses rather than rewrites. +# +# tests/coverage.sh [THRESHOLD] +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +threshold="${1:-${COVERAGE_THRESHOLD:-100}}" + +for tool in kcov bats python3; do + if ! command -v "$tool" >/dev/null; then + echo "$tool not found (apt-get install $tool)" >&2 + exit 2 + fi +done + +report="${COVERAGE_DIR:-$(mktemp -d)}" +# The include pattern is the whitelist, so no exclude pattern is needed; an +# exclude of /tests/ would drop tests/lib/boot-test-lib.sh with it. +kcov --include-pattern=/lib/postgresql.sh,/firstboot.d/36pgsqlverify,/tests/lib/boot-test-lib.sh \ + "$report" bats "$here" + +json="$(find "$report" -mindepth 2 -maxdepth 2 -name coverage.json -not -path "*/kcov-merged/*" | head -1)" +echo +echo "kcov line coverage (threshold $threshold percent):" +awk -F'"' -v threshold="$threshold" ' + /^ *\{"file":/ { + n = split($4, parts, "/") + printf "%7.2f %s/%s %s", $8, $12, $16, parts[n] + if ($8 + 0 < threshold) { printf " BELOW THRESHOLD"; below = 1 } + printf "\n" + seen = 1 + } + END { + if (!seen) { print "no file measured"; exit 1 } + exit below + }' "$json" diff --git a/tests/hook.bats b/tests/hook.bats new file mode 100644 index 0000000..7f664a1 --- /dev/null +++ b/tests/hook.bats @@ -0,0 +1,143 @@ +#!/usr/bin/env bats +# The first boot hook firstboot.d/36pgsqlverify (decision 0004): every path +# it takes runs for real against scratch directories, and every command +# that would touch the system (systemctl, pg_isready, psql) is a PATH stub +# that records its arguments. Nothing here needs root, a database or a +# network. +# +# What these tests are about is the half of the password defect that is +# this layer's: common's 35pgsqlpass already reads DB_PASS and sets the +# role, and nothing said whether the database would accept it. keel diff +# never compares secrets, on either side, so on an appliance whose whole +# purpose is the database a password that did not arrive would have been +# silent. + +bats_require_minimum_version 1.5.0 + +setup() { + ROOT="$BATS_TEST_DIRNAME/.." + HOOK="$ROOT/overlay/usr/lib/inithooks/firstboot.d/36pgsqlverify" + scratch="$BATS_TEST_TMPDIR/hook" + mkdir -p "$scratch/bin" "$scratch/inithooks/bin" + # the library is the real file, not a copy: kcov measures the one the + # layer ships, and a copy per test would be measured as its own + # uncovered file + ln -s "$(cd "$ROOT/overlay/usr/lib/inithooks/lib" && pwd)" "$scratch/inithooks/lib" + + export INITHOOKS_DEFAULT="$scratch/default-inithooks" + export INITHOOKS_CONF="$scratch/inithooks.conf" + export CALLS="$scratch/calls" + export PGSQL_SLEEP=: + + cat > "$INITHOOKS_DEFAULT" <> "$CALLS"' + stub pg_isready 'echo "pg_isready $*" >> "$CALLS"' + # the client the hook proves the password with: it answers the probe + # only for the password the description declared, the way a server does + stub psql 'echo "psql $* PGPASSWORD=$PGPASSWORD" >> "$CALLS" +if [ "$PGPASSWORD" = "s3cret-from-the-description" ]; then echo 1; else +echo "psql: error: connection failed: password authentication failed" >&2; exit 2; fi' + PATH="$scratch/bin:$PATH" +} + +stub() { + printf '#!/bin/sh\n%s\n' "$2" > "$scratch/bin/$1" + chmod +x "$scratch/bin/$1" +} + +DECLARED_PASS=s3cret-from-the-description + +# write_conf [EXTRA_LINE...]: the conf a declared description renders to +write_conf() { + printf 'export DB_PASS=%s\n' "$DECLARED_PASS" > "$INITHOOKS_CONF" + printf '%s\n' "$@" >> "$INITHOOKS_CONF" +} + +@test "the declared password is proved against the database" { + write_conf + run "$HOOK" + [ "$status" -eq 0 ] + grep -q -- "psql --username=postgres --host=::1 --port=5432 --dbname=postgres" "$CALLS" + grep -q -- "PGPASSWORD=$DECLARED_PASS" "$CALLS" + [[ "$output" == *"set from DB_PASS (${#DECLARED_PASS} characters)"* ]] + [[ "$output" == *"verified on [::1]:5432"* ]] + [[ "$output" != *"$DECLARED_PASS"* ]] +} + +@test "a password the database refuses fails the hook" { + write_conf + DECLARED_PASS=not-the-one-the-server-has + write_conf + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"cannot authenticate on [::1]:5432 with the declared password"* ]] +} + +@test "a server that answers something else fails the hook" { + write_conf + stub psql 'echo "surprise"' + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"answered 'surprise', not '1'"* ]] +} + +@test "nothing declared: the hook names the field, and never prompts" { + printf 'export HOSTNAME=db\n' > "$INITHOOKS_CONF" + run "$HOOK" < /dev/null + [ "$status" -eq 1 ] + [[ "$output" == *"no DB_PASS in $INITHOOKS_CONF"* ]] + [[ "$output" == *"declare secrets.db_password in the instance description"* ]] + [ ! -f "$CALLS" ] +} + +@test "no conf at all is the same failure" { + rm -f "$INITHOOKS_CONF" + run "$HOOK" < /dev/null + [ "$status" -eq 1 ] + [[ "$output" == *"no DB_PASS"* ]] +} + +@test "PGSQL_PASS in the conf is not a password: it is a build time variable" { + printf 'export PGSQL_PASS=from-the-build\n' > "$INITHOOKS_CONF" + run "$HOOK" < /dev/null + [ "$status" -eq 1 ] + [[ "$output" == *"no DB_PASS"* ]] + [ ! -f "$CALLS" ] || ! grep -q "from-the-build" "$CALLS" +} + +@test "APP_DB_USER names the role the password belongs to" { + write_conf "export APP_DB_USER=lappuser" + run "$HOOK" + [ "$status" -eq 0 ] + grep -q -- "psql --username=lappuser" "$CALLS" +} + +@test "a role name this layer will not verify fails before anything runs" { + write_conf "export APP_DB_USER='rm -rf /'" + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"is not a role name this layer will verify"* ]] + [ ! -f "$CALLS" ] +} + +@test "the cluster is started and waited for on the address it checks" { + write_conf + run "$HOOK" + [ "$status" -eq 0 ] + [ "$(head -1 "$CALLS")" = "systemctl start postgresql.service" ] + [ "$(sed -n 2p "$CALLS")" = "pg_isready --quiet --host=::1 --port=5432" ] +} + +@test "a cluster that never answers fails before the client runs" { + write_conf + stub pg_isready 'exit 1' + export PGSQL_WAIT_TRIES=2 + run "$HOOK" + [ "$status" -eq 1 ] + [[ "$output" == *"postgresql did not answer on [::1]:5432 after 2 tries"* ]] + ! grep -q psql "$CALLS" +} diff --git a/tests/instance.yaml b/tests/instance.yaml new file mode 100644 index 0000000..54a3d79 --- /dev/null +++ b/tests/instance.yaml @@ -0,0 +1,53 @@ +# Instance description of the boot test (format: docs/spec.md of the keel +# repository). IPv6 first, address from the bridge, no certificate request. +# The two passwords are files the test writes into the rootfs before +# booting, which is the whole point of this test: secrets.db_password is +# declared here, nothing is typed at the console, and after the first boot +# a database client authenticates with exactly that value. +# +# This is the throwaway container the gate boots. The description an +# operator would start from is keel/instance.example.yaml. +version: 1 + +instance: + hostname: postgresql + fqdn: postgresql.example.org + +network: + # The container's address comes from the bridge, so the host owns the + # interface configuration and 01ipconfig writes nothing. + managed_by: host + interfaces: + eth0: + ipv6: + method: auto + +tls: + acme: + enabled: false + +secrets: + root_password: + file: /etc/keel/secrets/root_password + # What this layer exists for. It renders to DB_PASS, which + # firstboot.d/35mysqlpass reads and hands to bin/pgsqlconf.py. + db_password: + file: /etc/keel/secrets/db_password + +app: + options: + # Renders to APP_DB_USER: the role the declared password belongs to. + # It is the role common's conf/pgsql configures and 35pgsqlpass sets, + # named here so an appliance built on this layer sees where it is. + db_user: postgres + +hub: + api_key: skip + +security: + alerts: skip + # A first boot input, not machine state: 95secupdates installs the + # pending security updates once. diff does not compare it. + updates_at_first_boot: skip + +first_login_wizard: false diff --git a/tests/lib/boot-test-lib.sh b/tests/lib/boot-test-lib.sh new file mode 100644 index 0000000..8903fbe --- /dev/null +++ b/tests/lib/boot-test-lib.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# Pure helpers of tests/boot-test.sh (decision 0004: logic apart from +# effect). Same shape as the one in keel-core, keel-nodebb and keel-mariadb, with the +# checks this layer adds: a client connection to the database with the +# password the instance description declared, the Webmin module for that +# database, and Webmin answering over IPv6. Nothing here starts a +# container, writes outside a path it is given or opens a socket. The two +# functions that run a command, bt_now and bt_container_ipv6, take it from +# the environment or from PATH so a test can replace it. Sourced by +# boot-test.sh and by tests/boot-test.bats. +# shellcheck disable=SC2034 # the BT_* variables are read by the caller + +BT_DEFAULT_TIMEOUT=900 +BT_DEFAULT_INTERVAL=5 +BT_DEFAULT_BRIDGE=br0 +BT_DEFAULT_LAYERS_DIR=/mnt/builds/layers +BT_DEFAULT_CACHE_DIR=/var/cache/keel/layers +BT_DEFAULT_LXC_PATH=/var/lib/lxc +BT_SSH_PORT=22 +BT_PASSWORD_LENGTH=24 +BT_RANDOM_BYTES=1024 +# The secrets tests/instance.yaml references, one file each under +# etc/keel/secrets in the rootfs: the root account and the database +# account, which is the one this layer exists for. +BT_SECRETS="root_password db_password" +# The panel core carries and this layer adds a module to. +BT_WEBMIN_PORT=12321 +BT_WEBMIN_MODULE=webmin-postgresql +# The database connection the test proves. The client runs inside the +# container, because the cluster listens on the loopback of both families +# and nowhere else: an appliance above this layer talks to it from the same +# machine, and opening it to the network is that appliance's decision, not +# this layer's. +BT_DB_USER="postgres" +BT_DB_HOST=::1 +BT_DB_PORT=5432 +BT_DB_NAME=postgres +BT_DB_PROBE_QUERY="SELECT 1" +BT_DB_PROBE_ANSWER=1 +# Where the first boot reads the instance description. inithooks reads +# etc/inithooks.yaml (hook 00declarative), keel reads etc/keel/instance.yaml; +# the final name is a maintainer decision (brief section 11), so the test +# installs the same file at both until then. +BT_SPEC_PATHS="etc/keel/instance.yaml etc/inithooks.yaml" + +bt_usage() { + cat <&2 + return 1 + } + [ "$1" = --timeout ] && BT_TIMEOUT=$2 || BT_INTERVAL=$2 + shift + ;; + --bridge|--layers-dir|--cache-dir|--lxc-path|--name|--spec) + [ -n "${2-}" ] || { + echo "boot-test: $1 needs a value" >&2 + return 1 + } + case "$1" in + --bridge) BT_BRIDGE=$2 ;; + --layers-dir) BT_LAYERS_DIR=$2 ;; + --cache-dir) BT_CACHE_DIR=$2 ;; + --lxc-path) BT_LXC_PATH=$2 ;; + --name) BT_NAME=$2 ;; + --spec) BT_SPEC=$2 ;; + esac + shift + ;; + --keep) BT_KEEP=1 ;; + -h|--help) + bt_usage + return 2 + ;; + -*) + echo "boot-test: unknown option $1" >&2 + return 1 + ;; + *) + if [ -n "$BT_APPLIANCE" ]; then + echo "boot-test: one appliance at a time ($BT_APPLIANCE, $1)" >&2 + return 1 + fi + BT_APPLIANCE=$1 + ;; + esac + shift + done + if [ -z "$BT_APPLIANCE" ]; then + echo "boot-test: APPLIANCE is required (core, lamp, ...)" >&2 + return 1 + fi + if ! bt_is_appliance_name "$BT_APPLIANCE"; then + echo "boot-test: '$BT_APPLIANCE' is not an appliance name (lower case, no keel- prefix)" >&2 + return 1 + fi + BT_NAME=${BT_NAME:-$(bt_container_name "$BT_APPLIANCE")} + if ! bt_is_container_name "$BT_NAME"; then + echo "boot-test: '$BT_NAME' is not a container name (lower case, digits, dot, dash)" >&2 + return 1 + fi + BT_ROOTFS=$BT_LXC_PATH/$BT_NAME/rootfs + return 0 +} + +bt_is_global_ipv6() { + # Global unicast, which includes ULA (fc00::/7); not link local + # (fe80::/10), loopback or multicast. IPv4 has no colon. + local addr=${1,,} + [[ $addr == *:* ]] || return 1 + [[ $addr == fe[89ab]?:* ]] && return 1 + [[ $addr == ::1 ]] && return 1 + [[ $addr == ff* ]] && return 1 + return 0 +} + +bt_global_ipv6() { + # stdin: the output of lxc-info -i ("IP: ADDRESS" per line). Prints + # the first global IPv6 address; returns 1 when there is none yet. + local label addr _ + while read -r label addr _; do + [ "$label" = "IP:" ] || continue + if bt_is_global_ipv6 "$addr"; then + printf '%s\n' "$addr" + return 0 + fi + done + return 1 +} + +bt_container_ipv6() { + # bt_container_ipv6 NAME LXCPATH: the container's first global IPv6. + lxc-info -P "$2" -n "$1" -i 2>/dev/null | bt_global_ipv6 +} + +bt_now() { + ${BT_CLOCK:-date +%s} +} + +bt_deadline_passed() { + # bt_deadline_passed START TIMEOUT NOW + [ $(( $3 - $1 )) -ge "$2" ] +} + +bt_wait_for() { + # bt_wait_for TIMEOUT INTERVAL DESCRIPTION COMMAND [ARGS...] + # Runs COMMAND until it succeeds; returns 1 once TIMEOUT seconds passed. + local timeout=$1 interval=$2 what=$3 start now + shift 3 + start=$(bt_now) + until "$@"; do + now=$(bt_now) + if bt_deadline_passed "$start" "$timeout" "$now"; then + echo "boot-test: timeout after ${timeout}s waiting for $what" >&2 + return 1 + fi + ${BT_SLEEP:-sleep} "$interval" + done +} + +bt_is_ssh_banner() { + [[ ${1-} == SSH-2.0-* ]] +} + +bt_firstboot_done_in() { + # bt_firstboot_done_in FILE: FILE is the rootfs copy of + # /etc/default/inithooks; 98finalize sets RUN_FIRSTBOOT=false at the end. + [ -r "$1" ] && grep -q '^RUN_FIRSTBOOT=false' "$1" +} + +bt_lxc_config() { + # bt_lxc_config NAME ROOTFS BRIDGE: an LXC config for a plain rootfs + # directory on a bridge; the address comes from the bridge (SLAAC or + # DHCPv6), the spec declares managed_by: host. + cat <&2 + return 1 + fi + printf '%s\n' "${pool:0:BT_PASSWORD_LENGTH}" +} + +bt_secret_targets() { + # bt_secret_targets ROOTFS: the secret files the spec references. + local name + for name in $BT_SECRETS; do + printf '%s/etc/keel/secrets/%s\n' "$1" "$name" + done +} + +bt_webmin_verdict() { + # bt_webmin_verdict CODE: Webmin comes from core and is the panel this + # layer adds its database module to, so the boot test checks that it + # answers over IPv6 on 12321. It asks for credentials, so 200 (the + # login page) and 401 are both an answer; 000 is curl failing to + # connect at all. + case "${1-}" in + 200|401) + echo "boot-test: webmin answered $1 on port $BT_WEBMIN_PORT" + ;; + *) + echo "boot-test: webmin answered '${1-}' on port $BT_WEBMIN_PORT, not 200 or 401" >&2 + return 1 + ;; + esac +} + +bt_module_verdict() { + # bt_module_verdict PACKAGE STATUS: the Webmin module for this + # database must be installed on the booted machine, not only in the + # plan. Batteries included is a property of the distribution, so a + # panel without its database module is a failed boot test. + if [ "${2-}" = "install ok installed" ]; then + echo "boot-test: $1 is installed" + return 0 + fi + echo "boot-test: $1 is '${2-}', not 'install ok installed'" >&2 + return 1 +} + +bt_db_verdict() { + # bt_db_verdict OUTPUT: what the database client printed when it + # connected with the declared password and ran the probe query. The + # whole point of this appliance is that a declared password reaches + # the database, so the answer is the verdict: anything other than the + # single row the query asks for means the connection did not happen or + # did not authenticate. + local output + output=$(printf '%s' "${1-}" | tr -d '[:space:]') + if [ "$output" = "$BT_DB_PROBE_ANSWER" ]; then + echo "boot-test: $BT_DB_USER authenticated on [$BT_DB_HOST]:$BT_DB_PORT with the declared password" + return 0 + fi + echo "boot-test: the database client answered '${1-}', not '$BT_DB_PROBE_ANSWER': the declared password did not reach the database" >&2 + return 1 +} + +bt_db_client_argv() { + # bt_db_client_argv USER HOST PORT [DATABASE]: the client command the + # boot test runs inside the container, one argument per line. The + # password is not here: it goes through PGPASSWORD in the environment, + # so it never appears in the container's process list. --no-password + # makes psql fail instead of prompting, so a password that did not + # arrive is an error and not a hung test. + local user=$1 host=$2 port=$3 database=${4-$BT_DB_NAME} + [ -n "$user" ] && [ -n "$host" ] && [ -n "$database" ] || return 1 + case "$port" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' psql "--username=$user" "--host=$host" "--port=$port" \ + "--dbname=$database" --no-password --tuples-only --no-align --quiet \ + "--command=$BT_DB_PROBE_QUERY" +} +bt_diff_verdict() { + # bt_diff_verdict CODE: interprets the exit code of keel diff + # (docs/diff.md of the keel repository). 0 and 13 mean no drift. + case "$1" in + 0) echo "keel diff: no drift"; return 0 ;; + 13) echo "keel diff: no drift, but a declared field could not be observed offline (see the report above)"; return 0 ;; + 14) echo "keel diff: drift found" >&2; return 1 ;; + 2|3) echo "keel diff: the spec is unreadable or invalid (exit $1)" >&2; return 1 ;; + *) echo "keel diff: failed with exit $1" >&2; return 1 ;; + esac +} diff --git a/tests/postgresql.bats b/tests/postgresql.bats new file mode 100644 index 0000000..1f6f164 --- /dev/null +++ b/tests/postgresql.bats @@ -0,0 +1,99 @@ +#!/usr/bin/env bats +# Unit tests of overlay/usr/lib/inithooks/lib/postgresql.sh, the logic +# behind the first boot hook 36pgsqlverify (decision 0004). Every function +# is pure: nothing here needs a database, root or a network. + +bats_require_minimum_version 1.5.0 + +setup() { + load ../overlay/usr/lib/inithooks/lib/postgresql.sh +} + +@test "first_value: the first value that is set and is not DEFAULT" { + [ "$(pgsql_first_value "" DEFAULT keeper other)" = keeper ] + [ "$(pgsql_first_value default keeper)" = keeper ] + [ "$(pgsql_first_value Default keeper)" = keeper ] + [ "$(pgsql_first_value first second)" = first ] + run ! pgsql_first_value "" DEFAULT "" + run ! pgsql_first_value +} + +@test "role: the cluster superuser, or the one APP_DB_USER names" { + [ "$(pgsql_role)" = postgres ] + [ "$(pgsql_role "")" = postgres ] + [ "$(pgsql_role DEFAULT)" = postgres ] + [ "$(pgsql_role lappuser)" = lappuser ] + PGSQL_ROLE=dba + [ "$(pgsql_role)" = dba ] +} + +@test "role: a name that would need quoting is refused" { + run ! pgsql_role "rm -rf /" + run ! pgsql_role '"postgres"' + run ! pgsql_role "9lives" + run ! pgsql_role "a b" + run ! pgsql_role "pg;sql" +} + +@test "is_role_name: what this layer will verify" { + pgsql_is_role_name postgres + pgsql_is_role_name _pg + pgsql_is_role_name lapp-user + pgsql_is_role_name a1 + run ! pgsql_is_role_name "" + run ! pgsql_is_role_name "1a" + run ! pgsql_is_role_name "a%" +} + +@test "missing_values: DB_PASS only when the description declared none" { + [ -z "$(pgsql_missing_values s3cret)" ] + [ "$(pgsql_missing_values "")" = DB_PASS ] + [ "$(pgsql_missing_values)" = DB_PASS ] +} + +@test "verify_argv: the psql call that proves the password, without it" { + output=$(pgsql_verify_argv postgres ::1 5432 postgres) + [ "$output" = $'--username=postgres\n--host=::1\n--port=5432\n--dbname=postgres\n--no-password\n--tuples-only\n--no-align\n--quiet\n--command=SELECT 1' ] + [[ $output != *s3cret* ]] + [[ $output == *"--no-password"* ]] +} + +@test "verify_argv: a role, a host, a positive port and a database are needed" { + run ! pgsql_verify_argv "" ::1 5432 postgres + run ! pgsql_verify_argv "a b" ::1 5432 postgres + run ! pgsql_verify_argv postgres "" 5432 postgres + run ! pgsql_verify_argv postgres ::1 "" postgres + run ! pgsql_verify_argv postgres ::1 0 postgres + run ! pgsql_verify_argv postgres ::1 54o2 postgres + run ! pgsql_verify_argv postgres ::1 5432 "" +} + +@test "probe_verdict: the answer the query asks for, whitespace aside" { + pgsql_probe_verdict 1 + pgsql_probe_verdict $'\n 1 \n' + run ! pgsql_probe_verdict 0 + run ! pgsql_probe_verdict "" + run ! pgsql_probe_verdict + run ! pgsql_probe_verdict "psql: error: connection to server failed" +} + +@test "wait_ready: returns as soon as the command succeeds" { + PGSQL_SLEEP=: + attempts=0 + probe() { attempts=$((attempts + 1)); [ "$attempts" -ge 3 ]; } + pgsql_wait_ready probe 10 + [ "$attempts" -eq 3 ] +} + +@test "wait_ready: gives up after the last try" { + PGSQL_SLEEP=: + never() { return 1; } + run ! pgsql_wait_ready never 4 +} + +@test "masked: a length, never the password" { + [ "$(pgsql_masked s3cret)" = "(6 characters)" ] + [ "$(pgsql_masked "")" = "(none)" ] + [ "$(pgsql_masked)" = "(none)" ] + [[ "$(pgsql_masked s3cret)" != *s3cret* ]] +}