From 5da406d3c30b04a213ac5eabc45d5436b58ce3dd Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Wed, 12 Aug 2026 17:08:21 +0100 Subject: [PATCH 1/3] Standardize lock generation on Python 3.12 Dependency markers are evaluated by the Python interpreter running the resolver. Allowing different Python minor versions and unpinned generator tools therefore produces host-dependent lock files and noisy generator metadata. Require Python 3.12 for source and lock generation, pin the resolver tools, and normalize annotations, headers, and index configuration from generated locks. Add a pinned-source reproducibility workflow and architecture checks, then regenerate the tracked lock files with the canonical environment. Assisted-By: Pi gpt-5.6-sol Signed-off-by: Sean Mooney (cherry picked from commit d28af263061f18db6fda7cc9b19dfc1835f77528) --- .github/actions/setup-tox/action.yml | 6 +- .github/workflows/test-update-sources.yml | 17 +- build.sh | 27 +- containers/base/buildrequirements.lock.master | 4 - containers/base/requirements.lock.master | 14 - .../cyborg/buildrequirements.lock.master | 4 - containers/cyborg/requirements.lock.master | 483 -------------- .../glance/buildrequirements.lock.master | 6 +- containers/glance/requirements.lock.master | 610 ------------------ .../manila/buildrequirements.lock.master | 6 +- containers/manila/requirements.lock.master | 593 ----------------- .../watcher/buildrequirements.lock.master | 4 - containers/watcher/requirements.lock.master | 609 ----------------- docs/TESTING.md | 10 + docs/developer-guide.md | 12 +- tox.ini | 19 +- 16 files changed, 75 insertions(+), 2349 deletions(-) diff --git a/.github/actions/setup-tox/action.yml b/.github/actions/setup-tox/action.yml index bef55cb0..007939b0 100644 --- a/.github/actions/setup-tox/action.yml +++ b/.github/actions/setup-tox/action.yml @@ -1,13 +1,13 @@ name: Setup Python and tox -description: Install Python and tox (repository must already be checked out) +description: Install tox with the repository's canonical Python 3.12 runs: using: composite steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: - python-version: "3.x" + python-version: "3.12" - name: Install tox shell: bash - run: pip install tox + run: python3 -m pip install tox diff --git a/.github/workflows/test-update-sources.yml b/.github/workflows/test-update-sources.yml index 1e5c6275..e93daf23 100644 --- a/.github/workflows/test-update-sources.yml +++ b/.github/workflows/test-update-sources.yml @@ -1,26 +1,35 @@ -name: Tests (update-sources) +name: Tests (source reproducibility) on: pull_request: paths: + - .github/actions/setup-tox/action.yml + - .github/workflows/test-update-sources.yml - build.sh - containers/** + - tox.ini - '!**/OWNERS' - '!**/OWNERS_ALIASES' push: paths: + - .github/actions/setup-tox/action.yml + - .github/workflows/test-update-sources.yml - build.sh - containers/** + - tox.ini - '!**/OWNERS' - '!**/OWNERS_ALIASES' workflow_dispatch: jobs: - update-sources: + update-lockfiles: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-tox - - name: Verify update-sources runs successfully - run: tox -e update-sources + - name: Regenerate from committed source pins + run: tox -e update-lockfiles + + - name: Require reproducible generated files + run: git diff --exit-code -- containers/ diff --git a/build.sh b/build.sh index 9b6f4bab..a3f934a0 100755 --- a/build.sh +++ b/build.sh @@ -618,6 +618,24 @@ filter_lockfile_rpm_packages() { rm "${tmp}" } +# Remove generator-runtime and package-index details from a lock file. +normalize_generated_lock() { + local lockfile="$1" + local tmp + tmp=$(mktemp) + + awk ' + !seen_package && /^--(index-url|extra-index-url|trusted-host)[[:space:]]/ { + next + } + !seen_package && (/^#/ || /^$/) { next } + !/^#/ { seen_package = 1 } + { print } + ' "${lockfile}" > "${tmp}" + install -m 644 "${tmp}" "${lockfile}" + rm "${tmp}" +} + # Generate a single requirements.lock for a project by running pip-compile # against requirements.txt from all source packages (project + all images) # plus pythondeps.txt and pythonbuilddeps.txt from every image, @@ -676,11 +694,11 @@ generate_requirements_lock() { echo "--- Generating ${project_dir}/${lock_file} ---" (cd "${project_dir}" && \ - pip-compile --allow-unsafe --strip-extras \ + pip-compile --allow-unsafe --no-annotate --strip-extras \ -c "${UPSTREAM_CONSTRAINTS}.${stream}" \ -o "${lock_file}" \ "${input_files[@]}" && \ - sed -i "/# This file is autogenerated/{N;d;}" "${lock_file}") + normalize_generated_lock "${lock_file}") local rpm_pkgs rpm_pkgs=$(collect_rpm_python_packages "${project_dir}") @@ -738,10 +756,11 @@ generate_buildrequirements_lock() { echo "--- Generating ${project_dir}/${build_lock_file} ---" (cd "${project_dir}" && \ - pybuild-deps compile --no-annotate \ + pybuild-deps compile \ + --no-annotate \ -o "${build_lock_file}" \ "${CONSTRAINTS_FILE}.${stream}" && \ - sed -i "/# This file is autogenerated/{N;d;}" "${build_lock_file}") + normalize_generated_lock "${build_lock_file}") } # Generate buildrequirements.lock for each project in the target scope. diff --git a/containers/base/buildrequirements.lock.master b/containers/base/buildrequirements.lock.master index bde466c0..9cd81cad 100644 --- a/containers/base/buildrequirements.lock.master +++ b/containers/base/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# flit-core==4.0.2 packaging==26.3 setuptools-scm==10.2.1 diff --git a/containers/base/requirements.lock.master b/containers/base/requirements.lock.master index ea0d8ab6..f94e40f5 100644 --- a/containers/base/requirements.lock.master +++ b/containers/base/requirements.lock.master @@ -1,22 +1,8 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras pythondeps.txt -# crudini==0.9.6 - # via -r pythondeps.txt dumb-init==1.2.5.post1 - # via -r pythondeps.txt iniparse==0.5 - # via crudini pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r pythondeps.txt six==1.17.0 - # via - # -c upper-constraints.txt.master - # iniparse # The following packages are considered to be unsafe in a requirements file: setuptools==82.0.1 - # via pbr diff --git a/containers/cyborg/buildrequirements.lock.master b/containers/cyborg/buildrequirements.lock.master index eee1c155..23f26bc2 100644 --- a/containers/cyborg/buildrequirements.lock.master +++ b/containers/cyborg/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# calver==2025.10.20 coherent-licensed==0.5.2 cython==3.2.9 diff --git a/containers/cyborg/requirements.lock.master b/containers/cyborg/requirements.lock.master index 87747144..1bc56468 100644 --- a/containers/cyborg/requirements.lock.master +++ b/containers/cyborg/requirements.lock.master @@ -1,590 +1,107 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras cyborg-agent/pythonbuilddeps.txt cyborg-agent/pythondeps.txt cyborg/pythonbuilddeps.txt cyborg/pythondeps.txt src/cyborg/requirements.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # oslo-messaging certifi==2026.6.17 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography - # oslo-privsep charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cotyledon==2.2.0 - # via - # -c upper-constraints.txt.master - # oslo-service debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # futurist - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-privsep - # oslo-service - # python-keystoneclient decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # oslo-service greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # eventlet - # oslo-service - # sqlalchemy idna==3.18 - # via - # -c upper-constraints.txt.master - # requests importlib-metadata==9.0.0 - # via - # -c upper-constraints.txt.master - # wsme iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk - # oslo-utils jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # openstacksdk - # warlock jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # warlock jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # openstacksdk - # python-glanceclient - # python-keystoneclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic - # pecan markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako microversion-parse==2.1.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-privsep - # oslo-serialization netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # oslo-config - # oslo-utils - # oslo-versionedobjects - # wsme openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt os-resource-classes==1.1.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r cyborg-agent/pythondeps.txt - # -r cyborg/pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-service oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # pycadf - # python-keystoneclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy - # oslo-versionedobjects oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r cyborg-agent/pythondeps.txt - # -r cyborg/pythondeps.txt - # -r src/cyborg/requirements.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-service - # oslo-upgradecheck - # oslo-utils - # oslo-versionedobjects - # python-glanceclient - # python-keystoneclient oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-messaging - # oslo-metrics - # oslo-privsep - # oslo-service - # oslo-versionedobjects oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-versionedobjects oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-upgradecheck oslo-privsep==3.12.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-policy - # pycadf - # python-keystoneclient oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-messaging - # oslo-service oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # python-glanceclient - # python-keystoneclient oslo-versionedobjects==3.11.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-utils - # python-keystoneclient - # wheel paste==3.10.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r cyborg-agent/pythonbuilddeps.txt - # -r cyborg/pythonbuilddeps.txt - # -r src/cyborg/requirements.txt - # keystonemiddleware - # openstacksdk - # os-resource-classes - # os-service-types - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-utils - # python-glanceclient - # python-keystoneclient pecan==1.8.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # oslo-upgradecheck - # python-glanceclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # openstacksdk - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pyopenssl==24.2.1 - # via - # -c upper-constraints.txt.master - # python-glanceclient pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # oslo-utils python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # oslo-log python-glanceclient==4.12.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache pytz==2026.2 - # via - # -c upper-constraints.txt.master - # wsme pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # keystonemiddleware - # oslo-config - # oslo-policy - # python-glanceclient - # python-keystoneclient rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config routes==2.5.1 - # via - # -c upper-constraints.txt.master - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing setproctitle==1.3.7 - # via - # -c upper-constraints.txt.master - # cotyledon simplegeneric==0.8.1 - # via - # -c upper-constraints.txt.master - # wsme six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # alembic - # oslo-db statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # dogpile-cache - # keystoneauth1 - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-keystoneclient testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # cotyledon - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu warlock==2.1.0 - # via - # -c upper-constraints.txt.master - # python-glanceclient wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable webob==1.8.10 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # microversion-parse - # oslo-messaging - # oslo-middleware - # oslo-service - # pecan - # wsme wheel==0.47.0 - # via - # -r cyborg-agent/pythonbuilddeps.txt - # -r cyborg/pythonbuilddeps.txt wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector - # python-glanceclient wsme==0.12.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service zipp==4.1.0 - # via - # -c upper-constraints.txt.master - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/glance/buildrequirements.lock.master b/containers/glance/buildrequirements.lock.master index a4625a84..87851e32 100644 --- a/containers/glance/buildrequirements.lock.master +++ b/containers/glance/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# build==1.5.0 calver==2025.10.20 coherent-licensed==0.5.2 @@ -26,7 +22,7 @@ tomlkit==0.15.1 trove-classifiers==2026.6.1.19 typing-extensions==4.15.0 vcs-versioning==2.2.4 -wheel==0.47.0 +wheel==0.48.0 # The following packages are considered to be unsafe in a requirements file: setuptools==82.0.1 diff --git a/containers/glance/requirements.lock.master b/containers/glance/requirements.lock.master index 2fa70252..7f0f33c4 100644 --- a/containers/glance/requirements.lock.master +++ b/containers/glance/requirements.lock.master @@ -1,736 +1,126 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras glance-api/pythonbuilddeps.txt glance-api/pythondeps.txt src/glance/requirements.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing automaton==3.4.0 - # via - # -c upper-constraints.txt.master - # taskflow autopage==0.6.0 - # via - # -c upper-constraints.txt.master - # cliff bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # taskflow castellan==5.8.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # cursive certifi==2026.7.22 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography - # oslo-privsep charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cliff==4.15.0 - # via - # -c upper-constraints.txt.master - # python-barbicanclient cmd2==4.0.0 - # via - # -c upper-constraints.txt.master - # cliff cursive==0.3.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # futurist - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-privsep - # oslo-rootwrap - # oslo-service - # python-keystoneclient - # taskflow decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk defusedxml==0.7.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency - # taskflow futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # glance-store - # oslo-messaging - # taskflow glance-store==5.6.0 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythondeps.txt - # -r src/glance/requirements.txt greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # eventlet - # oslo-service - # sqlalchemy httplib2==0.32.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt idna==3.18 - # via - # -c upper-constraints.txt.master - # requests importlib-metadata==9.0.0 - # via - # -c upper-constraints.txt.master - # wsme iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk - # oslo-utils jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # oslo-reports jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # glance-store - # taskflow jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # glance-store - # keystonemiddleware - # openstacksdk - # oslo-limit - # python-barbicanclient - # python-cinderclient - # python-keystoneclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic markdown-it-py==4.2.0 - # via - # -c upper-constraints.txt.master - # rich markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako mdurl==0.1.2 - # via - # -c upper-constraints.txt.master - # markdown-it-py msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-privsep - # oslo-serialization netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # oslo-config - # oslo-utils - # osprofiler - # wsme networkx==3.6.1 - # via - # -c upper-constraints.txt.master - # taskflow openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # oslo-limit os-brick==7.1.0 - # via - # -c upper-constraints.txt.master - # glance-store os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # glance-store - # os-brick - # oslo-service - # osprofiler oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # glance-store - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-limit - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # osprofiler - # pycadf - # python-keystoneclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # keystonemiddleware - # os-brick - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythondeps.txt - # -r src/glance/requirements.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cursive - # glance-store - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-limit - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient oslo-limit==2.12.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cursive - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-limit - # oslo-messaging - # oslo-metrics - # oslo-privsep - # oslo-service oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-upgradecheck oslo-privsep==3.12.0 - # via - # -c upper-constraints.txt.master - # glance-store - # os-brick oslo-reports==3.9.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-rootwrap==7.10.0 - # via - # -c upper-constraints.txt.master - # glance-store oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # cursive - # glance-store - # keystonemiddleware - # os-brick - # oslo-log - # oslo-messaging - # oslo-policy - # oslo-reports - # osprofiler - # pycadf - # python-barbicanclient - # python-keystoneclient - # taskflow oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # os-brick - # oslo-messaging oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cursive - # glance-store - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # taskflow osprofiler==4.4.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-utils - # python-keystoneclient paste==3.10.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythonbuilddeps.txt - # -r src/glance/requirements.txt - # castellan - # cursive - # keystonemiddleware - # openstacksdk - # os-brick - # os-service-types - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-reports - # oslo-rootwrap - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # taskflow platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # automaton - # cliff - # oslo-upgradecheck - # osprofiler - # python-cinderclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics prompt-toolkit==3.0.52 - # via - # -c upper-constraints.txt.master - # cmd2 psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # openstacksdk - # os-brick - # oslo-reports - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pydot==4.0.1 - # via - # -c upper-constraints.txt.master - # taskflow pygments==2.20.0 - # via - # -c upper-constraints.txt.master - # rich pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # httplib2 - # oslo-utils - # pydot pyperclip==1.11.0 - # via - # -c upper-constraints.txt.master - # cmd2 python-barbicanclient==7.5.0 - # via - # -c upper-constraints.txt.master - # castellan python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-cinderclient==9.9.0 - # via - # -c upper-constraints.txt.master - # glance-store python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # oslo-log python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # glance-store - # keystonemiddleware python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache python-swiftclient==4.10.0 - # via - # -c upper-constraints.txt.master - # glance-store pytz==2026.2 - # via - # -c upper-constraints.txt.master - # wsme pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # cliff - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # castellan - # glance-store - # keystoneauth1 - # keystonemiddleware - # os-brick - # oslo-config - # oslo-policy - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-swiftclient retrying==1.4.2 - # via - # -c upper-constraints.txt.master - # glance-store rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config rich==15.0.0 - # via - # -c upper-constraints.txt.master - # cmd2 - # rich-argparse rich-argparse==1.8.0 - # via - # -c upper-constraints.txt.master - # cmd2 routes==2.5.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing simplegeneric==0.8.1 - # via - # -c upper-constraints.txt.master - # wsme six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # alembic - # oslo-db statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cliff - # dogpile-cache - # glance-store - # keystoneauth1 - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-cinderclient - # python-keystoneclient - # taskflow taskflow==6.3.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt tenacity==9.1.4 - # via - # -c upper-constraints.txt.master - # os-brick - # taskflow testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # automaton - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable - # prompt-toolkit webob==1.8.10 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # keystonemiddleware - # oslo-messaging - # oslo-middleware - # oslo-service - # osprofiler - # wsme wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector wsme==0.12.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service zipp==4.1.0 - # via - # -c upper-constraints.txt.master - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/manila/buildrequirements.lock.master b/containers/manila/buildrequirements.lock.master index c9246d98..db159a4b 100644 --- a/containers/manila/buildrequirements.lock.master +++ b/containers/manila/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# build==1.5.0 calver==2025.10.20 cffi==2.0.0 @@ -27,7 +23,7 @@ tomlkit==0.15.1 trove-classifiers==2026.6.1.19 typing-extensions==4.15.0 vcs-versioning==2.2.4 -wheel==0.47.0 +wheel==0.48.0 # The following packages are considered to be unsafe in a requirements file: setuptools==82.0.1 diff --git a/containers/manila/requirements.lock.master b/containers/manila/requirements.lock.master index c6a6a0d7..57fe0dfa 100644 --- a/containers/manila/requirements.lock.master +++ b/containers/manila/requirements.lock.master @@ -1,715 +1,122 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras pythonbuilddeps.txt pythondeps.txt src/manila/requirements.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing autopage==0.6.0 - # via - # -c upper-constraints.txt.master - # cliff bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # paramiko cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-messaging castellan==5.8.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt certifi==2026.7.22 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography - # oslo-privsep - # pynacl charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cliff==4.15.0 - # via - # -c upper-constraints.txt.master - # osc-lib - # python-barbicanclient - # python-neutronclient cmd2==4.0.0 - # via - # -c upper-constraints.txt.master - # cliff cotyledon==2.2.0 - # via - # -c upper-constraints.txt.master - # oslo-service debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # futurist - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-privsep - # oslo-rootwrap - # oslo-service - # python-keystoneclient - # python-neutronclient decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk defusedxml==0.7.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency - # tooz futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # oslo-service - # tooz greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # eventlet - # oslo-service - # sqlalchemy idna==3.18 - # via - # -c upper-constraints.txt.master - # requests invoke==3.0.3 - # via - # -c upper-constraints.txt.master - # paramiko iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk - # oslo-utils - # python-novaclient jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # oslo-reports jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # openstacksdk - # osc-lib - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging lxml==6.1.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic markdown-it-py==4.2.0 - # via - # -c upper-constraints.txt.master - # rich markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako mdurl==0.1.2 - # via - # -c upper-constraints.txt.master - # markdown-it-py msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-privsep - # oslo-serialization - # tooz netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-config - # oslo-utils - # osprofiler - # python-neutronclient openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # osc-lib - # python-neutronclient os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk osc-lib==4.7.0 - # via - # -c upper-constraints.txt.master - # python-neutronclient oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service - # osprofiler oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # osprofiler - # pycadf - # python-keystoneclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r pythondeps.txt - # -r src/manila/requirements.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # oslo-cache - # oslo-messaging - # oslo-metrics - # oslo-privsep - # oslo-service - # python-neutronclient oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-upgradecheck oslo-privsep==3.12.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-reports==3.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-rootwrap==7.10.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-policy - # oslo-reports - # osprofiler - # pycadf - # python-barbicanclient - # python-keystoneclient - # python-neutronclient - # python-novaclient - # tooz oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-messaging oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient - # tooz osprofiler==4.4.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # kombu - # oslo-utils - # python-keystoneclient paramiko==4.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt paste==3.10.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r pythonbuilddeps.txt - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # openstacksdk - # os-service-types - # osc-lib - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-reports - # oslo-rootwrap - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # cliff - # oslo-upgradecheck - # osprofiler - # python-cinderclient - # python-novaclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics prompt-toolkit==3.0.52 - # via - # -c upper-constraints.txt.master - # cmd2 psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-reports - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pygments==2.20.0 - # via - # -c upper-constraints.txt.master - # rich pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pynacl==1.6.2 - # via - # -c upper-constraints.txt.master - # paramiko pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-utils pyperclip==1.11.0 - # via - # -c upper-constraints.txt.master - # cmd2 python-barbicanclient==7.5.0 - # via - # -c upper-constraints.txt.master - # castellan python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-cinderclient==9.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # oslo-log python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # python-neutronclient python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache python-neutronclient==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt python-novaclient==18.13.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # cliff - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystoneauth1 - # keystonemiddleware - # osc-lib - # oslo-config - # oslo-policy - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config rich==15.0.0 - # via - # -c upper-constraints.txt.master - # cmd2 - # rich-argparse rich-argparse==1.8.0 - # via - # -c upper-constraints.txt.master - # cmd2 routes==2.5.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing setproctitle==1.3.7 - # via - # -c upper-constraints.txt.master - # cotyledon six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # alembic - # oslo-db - # sqlalchemy-utils sqlalchemy-utils==0.42.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # cliff - # dogpile-cache - # keystoneauth1 - # osc-lib - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-cinderclient - # python-keystoneclient - # python-novaclient - # tooz tenacity==9.1.4 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # tooz testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db tooz==9.0.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # cotyledon - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu voluptuous==0.16.0 - # via - # -c upper-constraints.txt.master - # tooz wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable - # prompt-toolkit webob==1.8.10 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # keystonemiddleware - # oslo-messaging - # oslo-middleware - # oslo-service - # osprofiler wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/watcher/buildrequirements.lock.master b/containers/watcher/buildrequirements.lock.master index f9077f6e..dafe28fa 100644 --- a/containers/watcher/buildrequirements.lock.master +++ b/containers/watcher/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# build==1.5.0 calver==2025.10.20 coherent-licensed==0.5.2 diff --git a/containers/watcher/requirements.lock.master b/containers/watcher/requirements.lock.master index 7cfa9716..3c7e735e 100644 --- a/containers/watcher/requirements.lock.master +++ b/containers/watcher/requirements.lock.master @@ -1,739 +1,130 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras src/watcher/requirements.txt watcher-base/pythonbuilddeps.txt watcher-base/pythondeps.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging apscheduler==3.11.2 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing automaton==3.4.0 - # via - # -c upper-constraints.txt.master - # taskflow autopage==0.6.0 - # via - # -c upper-constraints.txt.master - # cliff bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # taskflow certifi==2026.6.17 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cliff==4.15.0 - # via - # -c upper-constraints.txt.master - # gnocchiclient - # osc-lib - # python-ironicclient - # python-observabilityclient - # python-openstackclient cmd2==4.0.0 - # via - # -c upper-constraints.txt.master - # cliff cotyledon==2.2.0 - # via - # -c upper-constraints.txt.master - # oslo-service croniter==6.2.2 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # futurist - # gnocchiclient - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-service - # python-keystoneclient - # python-manilaclient - # taskflow decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache - # python-ironicclient eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency - # taskflow futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # gnocchiclient - # oslo-messaging - # oslo-service - # taskflow gnocchiclient==7.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # eventlet - # oslo-service - # sqlalchemy idna==3.18 - # via - # -c upper-constraints.txt.master - # requests importlib-metadata==9.0.0 - # via - # -c upper-constraints.txt.master - # wsme iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # gnocchiclient - # keystoneauth1 - # openstacksdk - # oslo-utils - # python-openstackclient jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # oslo-reports jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # openstacksdk jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # python-ironicclient - # taskflow jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # gnocchiclient - # keystonemiddleware - # openstacksdk - # osc-lib - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # python-observabilityclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging lxml==6.1.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic - # pecan markdown-it-py==4.2.0 - # via - # -c upper-constraints.txt.master - # rich markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako mdurl==0.1.2 - # via - # -c upper-constraints.txt.master - # markdown-it-py microversion-parse==2.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-serialization netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # oslo-config - # oslo-utils - # oslo-versionedobjects - # wsme networkx==3.6.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # taskflow openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # osc-lib - # python-ironicclient - # python-openstackclient os-resource-classes==1.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk osc-lib==4.7.0 - # via - # -c upper-constraints.txt.master - # python-ironicclient - # python-manilaclient - # python-observabilityclient - # python-openstackclient oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # -r watcher-base/pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-service oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # pycadf - # python-keystoneclient - # python-manilaclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy - # oslo-versionedobjects oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # -r watcher-base/pythondeps.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-utils - # oslo-versionedobjects - # python-keystoneclient - # python-openstackclient oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-messaging - # oslo-metrics - # oslo-service - # oslo-versionedobjects - # python-manilaclient oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-versionedobjects oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-upgradecheck oslo-reports==3.9.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-policy - # oslo-reports - # pycadf - # python-keystoneclient - # python-manilaclient - # taskflow oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-messaging - # oslo-service oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-reports - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # taskflow oslo-versionedobjects==3.11.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-utils - # python-keystoneclient - # wheel paste==3.10.1 - # via - # -c upper-constraints.txt.master - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # -r watcher-base/pythonbuilddeps.txt - # keystonemiddleware - # openstacksdk - # os-resource-classes - # os-service-types - # osc-lib - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-reports - # oslo-utils - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # taskflow pecan==1.8.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # python-ironicclient prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # automaton - # cliff - # oslo-upgradecheck - # python-manilaclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics prompt-toolkit==3.0.52 - # via - # -c upper-constraints.txt.master - # cmd2 psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-reports - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pydot==4.0.1 - # via - # -c upper-constraints.txt.master - # taskflow pygments==2.20.0 - # via - # -c upper-constraints.txt.master - # rich pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # oslo-utils - # pydot pyperclip==1.11.0 - # via - # -c upper-constraints.txt.master - # cmd2 python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # croniter - # gnocchiclient - # oslo-log python-ironicclient==6.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # python-openstackclient python-manilaclient==6.2.0 - # via - # -c upper-constraints.txt.master - # python-openstackclient python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache python-observabilityclient==1.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt python-openstackclient==10.2.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt pytz==2026.2 - # via - # -c upper-constraints.txt.master - # wsme pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # cliff - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils - # python-ironicclient - # python-observabilityclient redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # keystonemiddleware - # osc-lib - # oslo-config - # oslo-policy - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # python-openstackclient rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config rich==15.0.0 - # via - # -c upper-constraints.txt.master - # cmd2 - # rich-argparse rich-argparse==1.8.0 - # via - # -c upper-constraints.txt.master - # cmd2 routes==2.5.1 - # via - # -c upper-constraints.txt.master - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing setproctitle==1.3.7 - # via - # -c upper-constraints.txt.master - # cotyledon simplegeneric==0.8.1 - # via - # -c upper-constraints.txt.master - # wsme six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # alembic - # oslo-db statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # cliff - # dogpile-cache - # keystoneauth1 - # osc-lib - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-ironicclient - # python-keystoneclient - # python-openstackclient - # taskflow taskflow==6.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt tenacity==9.1.4 - # via - # -c upper-constraints.txt.master - # taskflow testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # automaton - # cotyledon - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu tzlocal==5.4.3 - # via - # -c upper-constraints.txt.master - # apscheduler uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached ujson==5.13.0 - # via - # -c upper-constraints.txt.master - # gnocchiclient urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable - # prompt-toolkit webob==1.8.10 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # microversion-parse - # oslo-messaging - # oslo-middleware - # oslo-service - # pecan - # wsme wheel==0.47.0 - # via -r watcher-base/pythonbuilddeps.txt wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector wsme==0.12.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service zipp==4.1.0 - # via - # -c upper-constraints.txt.master - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/docs/TESTING.md b/docs/TESTING.md index d76bb1a3..4458ba4c 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -19,5 +19,15 @@ The linter environment checks tracked Containerfiles and shell content through pre-commit. Use the narrowest applicable command first, then run both environments before proposing changes to build or source-maintenance behavior. +Regenerate dependency locks from committed source pins with Python 3.12: + +```console +uvx --python 3.12 tox -e update-lockfiles +``` + +The environment rejects other Python minor versions because environment +markers can resolve a different package set. A clean regeneration must leave no +tracked diff under `containers/`. + Additional change-specific validation is described in the [developer guide](developer-guide.md#ci-path-filtering). diff --git a/docs/developer-guide.md b/docs/developer-guide.md index df913c69..eecd7cd6 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -253,12 +253,20 @@ Auto-cloned repos in `src/` are cleaned up automatically on exit. Pre-existing checkouts in `src/` are used as-is and not removed. To regenerate lockfiles without updating pinned hashes (steps 1--3 are -skipped; repos are cloned at the existing pinned hashes instead): +skipped; repos are cloned at the existing pinned hashes instead), use the +canonical Python 3.12 environment: ```bash -STREAM=master SKIP_HASH_UPDATE=1 ./build.sh update-sources +STREAM=master uvx --python 3.12 tox -e update-lockfiles -- ``` +Python environment markers are evaluated by the generator interpreter, so +other Python minor versions can produce a different dependency set. The tox +environment rejects non-3.12 interpreters and pins the generator tool versions. +Generated lock headers, resolver annotations, and package-index directives are +removed because they describe the generation environment rather than the +resolved dependency set. + ### Building images ```bash diff --git a/tox.ini b/tox.ini index 2a9ebb16..988d08a6 100644 --- a/tox.ini +++ b/tox.ini @@ -24,11 +24,11 @@ passenv = BUILD_LOGS_DIR UPDATE_LOCKFILES_TARGETS # pin to pip==26.1.2 as 26.2 has broken pip-tools -# pin pip-tools<7.6.1: pybuild-deps 0.5.0 passes generate_hashes to +# pin pip-tools==7.6.0: pybuild-deps 0.5.0 passes generate_hashes to # OutputWriter which was removed in pip-tools 7.6.1 deps = - pip-tools<7.6.1 - pybuild-deps + pip-tools==7.6.0 + pybuild-deps==0.5.0 pip==26.1.2 allowlist_externals = bash @@ -48,15 +48,24 @@ commands = pre-commit run -a --show-diff-on-failure [testenv:update-sources] +description = Refresh source pins and generated lockfiles +setenv = + XDG_CACHE_HOME = {envtmpdir}/cache +commands_pre = + python -c "import sys; assert sys.version_info[:2] == (3, 12), 'dependency generation requires Python 3.12'" commands = - bash -c "rm -f ~/.cache/pybuild-deps/find-build-deps" + bash -c "rm -f {env:XDG_CACHE_HOME}/pybuild-deps/find-build-deps" bash {toxinidir}/build.sh update-sources {posargs:all} [testenv:update-lockfiles] +description = Refresh generated lockfiles without advancing source pins setenv = + XDG_CACHE_HOME = {envtmpdir}/cache SKIP_HASH_UPDATE = 1 +commands_pre = + {[testenv:update-sources]commands_pre} commands = - bash -c "rm -f ~/.cache/pybuild-deps/find-build-deps" + bash -c "rm -f {env:XDG_CACHE_HOME}/pybuild-deps/find-build-deps" bash {toxinidir}/build.sh update-sources {posargs:all} [testenv:build] From 40b6f2bb645113517fd47f5b153f5ce8dbdeb185 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Wed, 12 Aug 2026 17:22:48 +0100 Subject: [PATCH 2/3] Make source updates atomic Source maintenance currently resolves and applies repositories one at a time. A late failure can therefore leave tracked source pins partially updated, and a moving branch can provide different content during one invocation. Preflight every selected source record and freeze each effective commit before tracked mutation. Retain the fetched objects for the complete run, record their authority in a deterministic manifest, and install source updates atomically. Replace the shell test harness with the existing stdlib unittest coverage for failure isolation, stream safety, checkout ownership, and multi-target scope. Assisted-By: Pi gpt-5.6-sol Signed-off-by: Sean Mooney (cherry picked from commit 0a14e6ae897ac39a0e065ad27e0ebc263231d0ce) --- .github/workflows/test-update-sources.yml | 8 + build.sh | 389 +++++++++++----- docs/TESTING.md | 12 +- docs/developer-guide.md | 21 +- tests/test_update_sources.py | 533 ++++++++++++++++++++++ tests/test_update_sources.sh | 529 --------------------- tox.ini | 9 +- 7 files changed, 836 insertions(+), 665 deletions(-) create mode 100644 tests/test_update_sources.py delete mode 100755 tests/test_update_sources.sh diff --git a/.github/workflows/test-update-sources.yml b/.github/workflows/test-update-sources.yml index e93daf23..75532ff9 100644 --- a/.github/workflows/test-update-sources.yml +++ b/.github/workflows/test-update-sources.yml @@ -33,3 +33,11 @@ jobs: - name: Require reproducible generated files run: git diff --exit-code -- containers/ + + - name: Preserve frozen source references + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: frozen-source-refs + path: .tmp/source-maintenance/frozen-source-refs.master.tsv + if-no-files-found: ignore diff --git a/build.sh b/build.sh index a3f934a0..9938efd9 100755 --- a/build.sh +++ b/build.sh @@ -103,6 +103,8 @@ DEFAULT_STREAM="${DEFAULT_STREAM:-master}" SKIP_HASH_UPDATE="${SKIP_HASH_UPDATE:-}" PIP_NO_BINARY="${PIP_NO_BINARY:-}" PARALLEL="${PARALLEL:-$(nproc)}" +SOURCE_REFS_STREAM="${STREAM//\//%2F}" +SOURCE_REFS_MANIFEST="${REPO_ROOT}/.tmp/source-maintenance/frozen-source-refs.${SOURCE_REFS_STREAM}.tsv" # Discover all buildable images from the directory structure. discover_images() { @@ -174,14 +176,18 @@ image_tag_args() { echo "${args}" } -# Track which repos were auto-cloned so we can clean up +# Track temporary source and frozen-ref repositories so they can be cleaned. declare -A _AUTO_CLONED=() +declare -A _FROZEN_REPOSITORIES=() -# Remove auto-cloned sources on exit cleanup_auto() { - for src_dir in "${!_AUTO_CLONED[@]}"; do - echo "--- Removing auto-cloned source: ${src_dir} ---" - rm -rf "${src_dir}" + local path + for path in "${!_AUTO_CLONED[@]}"; do + echo "--- Removing auto-cloned source: ${path} ---" + rm -rf "${path}" + done + for path in "${_FROZEN_REPOSITORIES[@]}"; do + rm -rf "${path}" done } trap cleanup_auto EXIT @@ -441,101 +447,280 @@ resolve_targets() { echo "${resolved[@]}" } -# Clone a repo at a branch tip (or tag) and store the resolved commit hash -# in _CLONE_RESULT. Must NOT be called via command substitution ($(...)) -# because _AUTO_CLONED assignments would be lost in the subshell. -# If the destination already exists, use it as-is (same policy as clone_at_hash). -# Args: -clone_at_branch() { - local dest="$1" - local url="$2" - local branch="$3" +# Resolve an expression without losing failures in command substitutions. +declare -a _RESOLVED_TARGETS=() +resolve_targets_array() { + local output + if ! output=$(resolve_targets "$@"); then + return 1 + fi + _RESOLVED_TARGETS=(${output}) +} - if [[ -d "${dest}" ]]; then - echo "--- Using existing source: ${dest} ---" - else - mkdir -p "$(dirname "${dest}")" - echo "--- Cloning ${url} (${branch}) into ${dest} ---" - if ! git clone --branch "${branch}" "${url}" "${dest}" 2>/dev/null; then - git clone "${url}" "${dest}" 2>/dev/null - git -C "${dest}" checkout "${branch}" +# Collect selected source manifests in deterministic target order. The parallel +# arrays describe the manifest, source checkout directory, and project context. +declare -a _SOURCE_FILES=() +declare -a _SOURCE_DIRS=() +declare -a _SOURCE_PROJECT_DIRS=() +collect_source_scopes() { + local -a targets + local img project sources_file + declare -A seen=() + + resolve_targets_array "$@" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") + _SOURCE_FILES=() + _SOURCE_DIRS=() + _SOURCE_PROJECT_DIRS=() + + for img in "${targets[@]}"; do + project="$(project_name "${img}")" + if [[ -z "${project}" ]]; then + [[ "${img}" == "base" ]] || continue + sources_file="${CONTAINERS_DIR}/base/sources.txt" + if [[ -f "${sources_file}" && -z "${seen[${sources_file}]:-}" ]]; then + seen["${sources_file}"]=1 + _SOURCE_FILES+=("${sources_file}") + _SOURCE_DIRS+=("${CONTAINERS_DIR}/base/src") + _SOURCE_PROJECT_DIRS+=("${CONTAINERS_DIR}/base") + fi + continue + fi + + sources_file="${CONTAINERS_DIR}/${project}/sources.txt" + if [[ -f "${sources_file}" && -z "${seen[${sources_file}]:-}" ]]; then + seen["${sources_file}"]=1 + _SOURCE_FILES+=("${sources_file}") + _SOURCE_DIRS+=("${CONTAINERS_DIR}/${project}/src") + _SOURCE_PROJECT_DIRS+=("${CONTAINERS_DIR}/${project}") fi - _AUTO_CLONED["${dest}"]=1 + + sources_file="${CONTAINERS_DIR}/${img}/sources.txt" + if [[ -f "${sources_file}" && -z "${seen[${sources_file}]:-}" ]]; then + seen["${sources_file}"]=1 + _SOURCE_FILES+=("${sources_file}") + _SOURCE_DIRS+=("${CONTAINERS_DIR}/${img}/src") + _SOURCE_PROJECT_DIRS+=("${CONTAINERS_DIR}/${project}") + fi + done +} + +source_record_key() { + printf '%s\x1f%s\x1f%s\x1f%s' "$1" "$2" "$3" "$4" +} + +source_ref_key() { + printf '%s\x1f%s' "$1" "$2" +} + +declare -A _FROZEN_COMMITS=() +declare -A _FROZEN_AUTHORITIES=() +declare -A _FROZEN_REF_COMMITS=() + +# Fetch one declared ref into a temporary bare repository and retain its exact +# commit for the rest of this process. This prevents a moving branch from +# changing inputs after preflight. +freeze_remote_ref() { + local url="$1" + local ref="$2" + local ref_key repository + ref_key="$(source_ref_key "${url}" "${ref}")" + if [[ -n "${_FROZEN_REF_COMMITS[${ref_key}]:-}" ]]; then + _FREEZE_RESULT="${_FROZEN_REF_COMMITS[${ref_key}]}" + return fi - _CLONE_RESULT=$(git -C "${dest}" rev-parse HEAD) + repository=$(mktemp -d) + git -C "${repository}" init --quiet --bare + if ! git -C "${repository}" fetch --quiet "${url}" "${ref}"; then + echo "ERROR: Could not freeze ref '${ref}' for ${url}" >&2 + rm -rf "${repository}" + return 1 + fi + if ! _FREEZE_RESULT=$(git -C "${repository}" rev-parse --verify 'FETCH_HEAD^{commit}'); then + echo "ERROR: Ref '${ref}' for ${url} does not resolve to a commit" >&2 + rm -rf "${repository}" + return 1 + fi + _FROZEN_REF_COMMITS["${ref_key}"]="${_FREEZE_RESULT}" + _FROZEN_REPOSITORIES["${ref_key}"]="${repository}" } -# Update pinned hashes in a single sources.txt file for the given stream. -# Clones source repos at the branch tip to resolve hashes, and extracts -# upper-constraints.txt when encountered. -# Args: +validate_stream_name() { + local stream="$1" + local component + local -a components + if [[ ! "${stream}" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] || \ + [[ "${stream}" == */ || "${stream}" == *//* ]]; then + echo "ERROR: Unsafe stream name '${stream}'" >&2 + return 1 + fi + IFS='/' read -ra components <<< "${stream}" + for component in "${components[@]}"; do + if [[ "${component}" == "." || "${component}" == ".." ]]; then + echo "ERROR: Unsafe stream name '${stream}'" >&2 + return 1 + fi + done +} + +# Resolve and record every selected source before the first tracked mutation. +freeze_source_refs() { + local stream="${!#}" + local targets_args=("${@:1:$#-1}") + local manifest_dir manifest_tmp + local index line entry_stream name url branch pinned_hash extra + local sources_file src_dir relative_file record_key authority frozen + + validate_stream_name "${stream}" || return 1 + collect_source_scopes "${targets_args[@]}" || return 1 + _FROZEN_COMMITS=() + _FROZEN_AUTHORITIES=() + _FROZEN_REF_COMMITS=() + _FROZEN_REPOSITORIES=() + + manifest_dir="$(dirname "${SOURCE_REFS_MANIFEST}")" + if [[ "${manifest_dir}" != "${REPO_ROOT}/.tmp/source-maintenance" ]]; then + echo "ERROR: Frozen source manifest escaped repository temporary state" >&2 + return 1 + fi + mkdir -p "${manifest_dir}" + rm -f "${SOURCE_REFS_MANIFEST}" + manifest_tmp=$(mktemp "${manifest_dir}/.frozen-source-refs.XXXXXX") + printf 'source_file\tstream\tname\turl\tdeclared_ref\tcommitted_pin\tfrozen_commit\tauthority\n' > "${manifest_tmp}" + + for index in "${!_SOURCE_FILES[@]}"; do + sources_file="${_SOURCE_FILES[${index}]}" + src_dir="${_SOURCE_DIRS[${index}]}" + relative_file="${sources_file#"${REPO_ROOT}/"}" + while IFS= read -r line; do + [[ -z "${line}" || "${line}" == \#* ]] && continue + read -r entry_stream name url branch pinned_hash extra <<< "${line}" + [[ "${entry_stream}" == "${stream}" ]] || continue + if [[ -z "${name}" || -z "${url}" || -z "${branch}" || -z "${pinned_hash}" || -n "${extra:-}" ]]; then + echo "ERROR: Malformed source record in ${relative_file}: ${line}" >&2 + rm -f "${manifest_tmp}" + return 1 + fi + + record_key="$(source_record_key "${sources_file}" "${name}" "${url}" "${branch}")" + if [[ "${name}" != "upper-constraints" && -d "${src_dir}/${name}" ]]; then + local checkout_root + checkout_root=$(git -C "${src_dir}/${name}" rev-parse --show-toplevel 2>/dev/null || true) + if [[ -z "${checkout_root}" || "$(realpath -e "${checkout_root}")" != "$(realpath -e "${src_dir}/${name}")" ]] || \ + ! frozen=$(git -C "${src_dir}/${name}" rev-parse --verify 'HEAD^{commit}' 2>/dev/null); then + echo "ERROR: Pre-existing source is not a Git checkout: ${src_dir}/${name}" >&2 + rm -f "${manifest_tmp}" + return 1 + fi + authority="pre-existing-checkout" + else + if [[ -n "${SKIP_HASH_UPDATE}" ]]; then + freeze_remote_ref "${url}" "${pinned_hash}" || { + rm -f "${manifest_tmp}" + return 1 + } + authority="committed-pin" + else + freeze_remote_ref "${url}" "${branch}" || { + rm -f "${manifest_tmp}" + return 1 + } + authority="declared-ref" + fi + frozen="${_FREEZE_RESULT}" + fi + + _FROZEN_COMMITS["${record_key}"]="${frozen}" + _FROZEN_AUTHORITIES["${record_key}"]="${authority}" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${relative_file}" "${entry_stream}" "${name}" "${url}" \ + "${branch}" "${pinned_hash}" "${frozen}" "${authority}" \ + >> "${manifest_tmp}" + done < "${sources_file}" + done + + mv "${manifest_tmp}" "${SOURCE_REFS_MANIFEST}" + echo "--- Frozen source references: ${SOURCE_REFS_MANIFEST} ---" +} + +# Materialize one source declaration from its frozen preflight repository and +# update its maintained pin only in advancement mode. update_sources_file() { local sources_file="$1" local stream="$2" local src_dir="$3" local project_dir="$4" - - if [[ ! -f "${sources_file}" ]]; then - return - fi - - local tmp_file - tmp_file=$(mktemp) + local tmp_file line entry_stream name url branch pinned_hash extra + local record_key ref_key frozen authority repository output_tmp local updated=0 + tmp_file=$(mktemp) while IFS= read -r line; do - # Preserve comments and blank lines if [[ -z "${line}" || "${line}" == \#* ]]; then echo "${line}" >> "${tmp_file}" continue fi - read -r entry_stream name url branch pinned_hash <<< "${line}" - - # Only update entries for the requested stream + read -r entry_stream name url branch pinned_hash extra <<< "${line}" if [[ "${entry_stream}" != "${stream}" ]]; then echo "${line}" >> "${tmp_file}" continue fi - local new_hash - if [[ "${name}" == "upper-constraints" ]]; then - # Clone without checkout, resolve hash from branch, extract file - local uc_tmp - uc_tmp=$(mktemp -d) - git clone --no-checkout "${url}" "${uc_tmp}" 2>/dev/null - new_hash=$(git -C "${uc_tmp}" rev-parse --verify "origin/${branch}" 2>/dev/null \ - || git -C "${uc_tmp}" rev-parse --verify "${branch}" 2>/dev/null) - git -C "${uc_tmp}" checkout "${new_hash}" -- upper-constraints.txt - cp "${uc_tmp}/upper-constraints.txt" "${project_dir}/${UPSTREAM_CONSTRAINTS}.${stream}" - rm -rf "${uc_tmp}" - elif [[ -d "${src_dir}/${name}" ]]; then - # Pre-existing checkout — use it for pip-compile but don't update the hash + record_key="$(source_record_key "${sources_file}" "${name}" "${url}" "${branch}")" + frozen="${_FROZEN_COMMITS[${record_key}]:-}" + authority="${_FROZEN_AUTHORITIES[${record_key}]:-}" + if [[ -z "${frozen}" || -z "${authority}" ]]; then + echo "ERROR: Missing frozen source record for ${name} in ${sources_file}" >&2 + rm -f "${tmp_file}" + return 1 + fi + + if [[ "${authority}" == "pre-existing-checkout" ]]; then echo " ${name}: skipped (pre-existing checkout at ${src_dir}/${name})" - new_hash="${pinned_hash}" else - clone_at_branch "${src_dir}/${name}" "${url}" "${branch}" - new_hash="${_CLONE_RESULT}" - fi + if [[ "${authority}" == "committed-pin" ]]; then + ref_key="$(source_ref_key "${url}" "${pinned_hash}")" + else + ref_key="$(source_ref_key "${url}" "${branch}")" + fi + repository="${_FROZEN_REPOSITORIES[${ref_key}]:-}" + if [[ -z "${repository}" || ! -d "${repository}" ]]; then + echo "ERROR: Missing frozen repository for ${name}" >&2 + rm -f "${tmp_file}" + return 1 + fi - if [[ -z "${new_hash}" ]]; then - echo "ERROR: Could not resolve ref '${branch}' for ${url}" >&2 - rm "${tmp_file}" - return 1 + if [[ "${name}" == "upper-constraints" ]]; then + output_tmp=$(mktemp "${project_dir}/.${UPSTREAM_CONSTRAINTS}.${stream}.XXXXXX") + if ! git -C "${repository}" show "${frozen}:upper-constraints.txt" > "${output_tmp}"; then + rm -f "${output_tmp}" "${tmp_file}" + return 1 + fi + mv "${output_tmp}" "${project_dir}/${UPSTREAM_CONSTRAINTS}.${stream}" + else + mkdir -p "${src_dir}" + echo "--- Materializing ${url} at ${frozen} into ${src_dir}/${name} ---" + git -C "${src_dir}" init --quiet "${name}" + _AUTO_CLONED["${src_dir}/${name}"]=1 + git -C "${src_dir}/${name}" fetch --quiet "${repository}" "${frozen}" + git -C "${src_dir}/${name}" checkout --quiet --detach FETCH_HEAD + fi fi - if [[ "${new_hash}" != "${pinned_hash}" ]]; then - echo " ${name}: ${pinned_hash:-} → ${new_hash} (${branch})" + if [[ -z "${SKIP_HASH_UPDATE}" && "${authority}" != "pre-existing-checkout" && "${frozen}" != "${pinned_hash}" ]]; then + echo " ${name}: ${pinned_hash} → ${frozen} (${branch})" + pinned_hash="${frozen}" updated=1 fi - echo "${entry_stream} ${name} ${url} ${branch} ${new_hash}" >> "${tmp_file}" + echo "${entry_stream} ${name} ${url} ${branch} ${pinned_hash}" >> "${tmp_file}" done < "${sources_file}" if [[ ${updated} -eq 1 ]]; then install -m 644 "${tmp_file}" "${sources_file}" else - echo " (no changes)" + echo " (no pin changes)" fi rm "${tmp_file}" } @@ -920,71 +1105,30 @@ ensure_sources_for_targets() { done } -# Update sources.txt files for targets in scope. -# Clones source repos at branch tips to resolve hashes, fetches -# upper-constraints.txt, and updates pinned hashes in sources.txt. +# Materialize all preflight-frozen sources and update maintained pins only when +# SKIP_HASH_UPDATE is unset. update_sources() { local stream="${!#}" - local targets_args=("${@:1:$#-1}") + local index sources_file if [[ -z "${stream}" ]]; then echo "ERROR: STREAM is required for update-sources." >&2 echo " Example: STREAM=master ./build.sh update-sources watcher" >&2 return 1 fi + if [[ ${#_SOURCE_FILES[@]} -eq 0 ]]; then + echo "ERROR: Source preflight did not collect any manifests" >&2 + return 1 + fi - local targets - targets=($(resolve_targets "${targets_args[@]}")) - - declare -A projects_seen - - for img in "${targets[@]}"; do - local project - project="$(project_name "${img}")" - - # Base container: flat layout, sources.txt directly in containers/base/ - if [[ -z "${project}" ]]; then - if [[ "${img}" == "base" ]] && [[ -z "${projects_seen[base]:-}" ]]; then - projects_seen["base"]=1 - local base_sources="${CONTAINERS_DIR}/base/sources.txt" - if [[ -f "${base_sources}" ]]; then - echo "--- Updating ${base_sources} (stream: ${stream}) ---" - if ! update_sources_file "${base_sources}" "${stream}" \ - "${CONTAINERS_DIR}/base/src" \ - "${CONTAINERS_DIR}/base"; then - echo "ERROR: Failed to update ${base_sources}" >&2 - return 1 - fi - fi - fi - continue - fi - - # Project-level sources.txt (only process once per project) - if [[ -z "${projects_seen[$project]:-}" ]]; then - projects_seen["${project}"]=1 - local project_sources="${CONTAINERS_DIR}/${project}/sources.txt" - if [[ -f "${project_sources}" ]]; then - echo "--- Updating ${project_sources} (stream: ${stream}) ---" - if ! update_sources_file "${project_sources}" "${stream}" \ - "${CONTAINERS_DIR}/${project}/src" \ - "${CONTAINERS_DIR}/${project}"; then - echo "ERROR: Failed to update ${project_sources}" >&2 - return 1 - fi - fi - fi - - # Image-level sources.txt - local image_sources="${CONTAINERS_DIR}/${img}/sources.txt" - if [[ -f "${image_sources}" ]]; then - echo "--- Updating ${image_sources} (stream: ${stream}) ---" - if ! update_sources_file "${image_sources}" "${stream}" \ - "${CONTAINERS_DIR}/${img}/src" \ - "${CONTAINERS_DIR}/${project}"; then - echo "ERROR: Failed to update ${image_sources}" >&2 - return 1 - fi + for index in "${!_SOURCE_FILES[@]}"; do + sources_file="${_SOURCE_FILES[${index}]}" + echo "--- Updating ${sources_file} (stream: ${stream}) ---" + if ! update_sources_file "${sources_file}" "${stream}" \ + "${_SOURCE_DIRS[${index}]}" \ + "${_SOURCE_PROJECT_DIRS[${index}]}"; then + echo "ERROR: Failed to update ${sources_file}" >&2 + return 1 fi done } @@ -1104,10 +1248,9 @@ case "${ACTION}" in update-sources) if [[ -n "${SKIP_HASH_UPDATE}" ]]; then echo "=== Skipping hash update (SKIP_HASH_UPDATE is set) ===" - ensure_sources_for_targets "${TARGETS[@]}" "${STREAM}" - else - update_sources "${TARGETS[@]}" "${STREAM}" fi + freeze_source_refs "${TARGETS[@]}" "${STREAM}" + update_sources "${TARGETS[@]}" "${STREAM}" # Generate lockfiles and metadata echo "" diff --git a/docs/TESTING.md b/docs/TESTING.md index 4458ba4c..9c50e7e2 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,13 +1,15 @@ # Testing -Run the repository's current test environment with: +Run the stdlib unittest suite through any supported alias: ```console +tox -e unit tox -e test +tox -e py3 ``` -This executes `tests/test_update_sources.sh`, which exercises source update and -lock-generation behavior with local temporary Git repositories. +The suite exercises source update and lock-generation behavior with local +temporary Git repositories. Run repository lint checks with: @@ -27,7 +29,9 @@ uvx --python 3.12 tox -e update-lockfiles The environment rejects other Python minor versions because environment markers can resolve a different package set. A clean regeneration must leave no -tracked diff under `containers/`. +tracked diff under `containers/`. Inspect the corresponding +`.tmp/source-maintenance/frozen-source-refs..tsv` manifest to confirm +that pinned regeneration used `committed-pin` authority throughout. Additional change-specific validation is described in the [developer guide](developer-guide.md#ci-path-filtering). diff --git a/docs/developer-guide.md b/docs/developer-guide.md index eecd7cd6..7926331e 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -241,16 +241,27 @@ STREAM=master ./build.sh update-sources ``` This will: -1. Clone each source repo at the branch tip to resolve the latest commit hash. -2. Update `sources.txt` with the new pinned hashes. -3. Fetch `upper-constraints.txt` from the requirements repo. +1. Resolve every selected branch or tag to one exact commit before mutation. +2. Update `sources.txt` with the frozen commits. +3. Fetch `upper-constraints.txt` from the same frozen input set. 4. Generate `rpms.in.yaml` from all `bindeps.txt` + `builddeps.txt` files. 5. Run `pip-compile` to generate `requirements.lock.`. 6. Run `pybuild-deps compile` to generate `buildrequirements.lock.`. 7. Create default-stream symlinks if `STREAM == DEFAULT_STREAM`. -Auto-cloned repos in `src/` are cleaned up automatically on exit. -Pre-existing checkouts in `src/` are used as-is and not removed. +The preflight record is written to +`.tmp/source-maintenance/frozen-source-refs..tsv`. It records each +source manifest, declared ref, committed pin, frozen commit, and authority. +Advancing runs use `declared-ref`; pinned runs use `committed-pin`; intentional +Git checkouts already under `src/` use `pre-existing-checkout`. A slash in a +stream name is encoded as `%2F`, and unsafe stream names are rejected before +filesystem mutation. + +All selected records must pass preflight before a tracked source manifest is +changed. The fetched repositories are retained for the complete run so a moving +branch cannot provide different content after preflight. Auto-created source +checkouts are cleaned up on exit. Pre-existing checkouts are used as-is and are +not removed. To regenerate lockfiles without updating pinned hashes (steps 1--3 are skipped; repos are cloned at the existing pinned hashes instead), use the diff --git a/tests/test_update_sources.py b/tests/test_update_sources.py new file mode 100644 index 00000000..2d57b45e --- /dev/null +++ b/tests/test_update_sources.py @@ -0,0 +1,533 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for build.sh update-sources behavior.""" + +import csv +import os +import pathlib +import subprocess +import tempfile +import unittest + + +class UpdateSourcesTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + tmp_root = self.repo_root / ".tmp" + tmp_root.mkdir(exist_ok=True) + self.temporary_directory = tempfile.TemporaryDirectory( + dir=tmp_root, prefix="update-sources-test." + ) + self.addCleanup(self.temporary_directory.cleanup) + self.test_root = pathlib.Path(self.temporary_directory.name) + upstream_root = self.test_root / "upstream" + upstream_root.mkdir() + + self.upstream_requirements = upstream_root / "requirements.git" + self.requirements_old, self.requirements_new = self.create_remote( + self.upstream_requirements, + [ + {"upper-constraints.txt": "six==1.17.0\n"}, + {"upper-constraints.txt": ("six==1.17.0\npbr==7.0.3\n")}, + ], + ) + self.upstream_service = upstream_root / "test-svc.git" + self.service_old, self.service_new = self.create_remote( + self.upstream_service, + [ + {"requirements.txt": "six\npbr\n"}, + {"requirements.txt": "six\npbr\n# v2\n"}, + ], + ) + (self.test_root / "build.sh").symlink_to(self.repo_root / "build.sh") + self.project_root = self.test_root / "containers" / "test-svc" + image_root = self.project_root / "test-svc" + (self.project_root / "src").mkdir(parents=True) + (image_root / "src").mkdir(parents=True) + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} master {self.service_old}\n" + ) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + (image_root / "bindeps.txt").write_text("python3\n", encoding="utf-8") + (image_root / "builddeps.txt").write_text("gcc\n", encoding="utf-8") + (image_root / "pythondeps.txt").touch() + (image_root / "pythonbuilddeps.txt").touch() + + self.additional_services = {} + for name in ("test-svc2", "test-svc3"): + remote = upstream_root / f"{name}.git" + old, new = self.create_remote( + remote, + [ + {"requirements.txt": "six\npbr\n"}, + {"requirements.txt": "six\npbr\n# v2\n"}, + ], + ) + self.additional_services[name] = (remote, old, new) + self.create_project_fixture(name, remote, old) + + self.manifest_path = ( + self.test_root + / ".tmp/source-maintenance/frozen-source-refs.master.tsv" + ) + + def run_command(self, command, cwd=None): + result = subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"command failed with {result.returncode}: {command}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result.stdout.strip() + + def create_remote(self, destination, commits): + work = self.test_root / f"work-{destination.stem}" + self.run_command(["git", "init", "-b", "master", str(work)]) + self.run_command( + ["git", "config", "user.email", "test@example.com"], work + ) + self.run_command(["git", "config", "user.name", "Test"], work) + self.run_command(["git", "config", "commit.gpgsign", "false"], work) + hashes = [] + for index, files in enumerate(commits, start=1): + for name, content in files.items(): + (work / name).write_text(content, encoding="utf-8") + self.run_command(["git", "add", "-A"], work) + self.run_command(["git", "commit", "-m", f"v{index}"], work) + hashes.append(self.run_command(["git", "rev-parse", "HEAD"], work)) + self.run_command( + ["git", "clone", "--bare", str(work), str(destination)] + ) + return hashes[0], hashes[-1] + + def write_sources(self, content): + (self.project_root / "sources.txt").write_text( + content, encoding="utf-8" + ) + + def create_project_fixture(self, name, remote, pinned_hash): + project_root = self.test_root / "containers" / name + image_root = project_root / name + (project_root / "src").mkdir(parents=True) + (image_root / "src").mkdir(parents=True) + (project_root / "sources.txt").write_text( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + f"master {name} {remote} master {pinned_hash}\n", + encoding="utf-8", + ) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + (image_root / "bindeps.txt").write_text("python3\n", encoding="utf-8") + (image_root / "builddeps.txt").write_text("gcc\n", encoding="utf-8") + (image_root / "pythondeps.txt").touch() + (image_root / "pythonbuilddeps.txt").touch() + + def run_update_result(self, *targets, **environment): + command_environment = os.environ.copy() + command_environment.update({"STREAM": "master"}) + command_environment.update(environment) + if not targets: + targets = ("test-svc",) + result = subprocess.run( + ["bash", "./build.sh", "update-sources", *targets], + cwd=self.test_root, + check=False, + capture_output=True, + text=True, + env=command_environment, + ) + (self.test_root / "build.log").write_text( + result.stdout + result.stderr, encoding="utf-8" + ) + return result + + def run_update(self, *targets, **environment): + result = self.run_update_result(*targets, **environment) + if result.returncode != 0: + self.fail( + f"update-sources failed with {result.returncode}:\n" + f"{result.stdout}{result.stderr}" + ) + return result.stdout + result.stderr + + def manifest_for_stream(self, stream): + filename = f"frozen-source-refs.{stream.replace('/', '%2F')}.tsv" + return self.test_root / ".tmp/source-maintenance" / filename + + def manifest_rows(self): + with self.manifest_path.open(encoding="utf-8", newline="") as stream: + return list(csv.DictReader(stream, delimiter="\t")) + + def source_field(self, name, field, project_root=None): + if project_root is None: + project_root = self.project_root + for line in ( + (project_root / "sources.txt") + .read_text(encoding="utf-8") + .splitlines() + ): + columns = line.split() + if columns[:2] == ["master", name]: + return columns[field - 1] + self.fail(f"source entry was not found: {name}") + + def test_updates_hashes_to_frozen_branch_tips(self): + self.run_update() + + self.assertEqual( + self.requirements_new, + self.source_field("upper-constraints", 5), + ) + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + rows = self.manifest_rows() + self.assertEqual( + ["upper-constraints", "test-svc"], [row["name"] for row in rows] + ) + self.assertEqual( + [self.requirements_new, self.service_new], + [row["frozen_commit"] for row in rows], + ) + self.assertEqual({"declared-ref"}, {row["authority"] for row in rows}) + + def test_fetches_upper_constraints(self): + self.run_update() + + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertIn("pbr==7.0.3", constraints) + + def test_generates_rpms_in_yaml(self): + self.run_update() + + rpms = (self.project_root / "rpms.in.yaml").read_text(encoding="utf-8") + self.assertIn("python3", rpms) + self.assertIn("gcc", rpms) + + def test_generates_requirements_lock(self): + self.run_update() + + lock = (self.project_root / "requirements.lock.master").read_text( + encoding="utf-8" + ) + self.assertIn("six", lock) + + def test_generates_buildrequirements_lock(self): + self.run_update() + + build_lock = self.project_root / "buildrequirements.lock.master" + self.assertTrue(build_lock.is_file()) + self.assertNotIn("# via", build_lock.read_text(encoding="utf-8")) + + def test_creates_default_stream_symlinks(self): + self.run_update(DEFAULT_STREAM="master") + + expected = { + "upper-constraints.txt": "upper-constraints.txt.master", + "requirements.lock": "requirements.lock.master", + "buildrequirements.lock": "buildrequirements.lock.master", + } + for name, target in expected.items(): + link = self.project_root / name + self.assertTrue(link.is_symlink()) + self.assertEqual(target, os.readlink(link)) + + def test_skips_symlinks_for_non_default_stream(self): + self.run_update(DEFAULT_STREAM="other") + + for name in ( + "upper-constraints.txt", + "requirements.lock", + "buildrequirements.lock", + ): + self.assertFalse((self.project_root / name).is_symlink()) + + def test_skip_hash_update_preserves_and_records_committed_hashes(self): + self.run_update(SKIP_HASH_UPDATE="1") + + self.assertEqual( + self.requirements_old, + self.source_field("upper-constraints", 5), + ) + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + self.assertTrue( + (self.project_root / "requirements.lock.master").is_file() + ) + rows = self.manifest_rows() + self.assertEqual( + [self.requirements_old, self.service_old], + [row["frozen_commit"] for row in rows], + ) + self.assertEqual({"committed-pin"}, {row["authority"] for row in rows}) + + def test_skip_hash_update_uses_pinned_constraints(self): + self.run_update(SKIP_HASH_UPDATE="1") + + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertNotIn("pbr", constraints) + + def test_hash_in_branch_field_selects_constraints_commit(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} {self.requirements_old} " + f"{self.requirements_new}\n" + "master test-svc " + f"{self.upstream_service} master {self.service_old}\n" + ) + + self.run_update() + + self.assertEqual( + self.requirements_old, + self.source_field("upper-constraints", 5), + ) + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertNotIn("pbr", constraints) + + def test_hash_in_branch_field_selects_regular_repo_commit(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} {self.service_old} " + f"{self.service_new}\n" + ) + + self.run_update() + + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + self.assertTrue( + (self.project_root / "requirements.lock.master").is_file() + ) + + def test_lockfile_excludes_rpm_python_packages(self): + image_root = self.project_root / "test-svc" + (image_root / "bindeps.txt").write_text( + "python3\npython3-six\n", encoding="utf-8" + ) + + output = self.run_update() + + lock = (self.project_root / "requirements.lock.master").read_text( + encoding="utf-8" + ) + self.assertNotIn("six==", lock) + self.assertIn("Filtering RPM-provided packages", output) + + def test_preexisting_checkout_is_preserved_and_recorded(self): + source = self.project_root / "src" / "test-svc" + self.run_command( + ["git", "clone", str(self.upstream_service), str(source)] + ) + (source / "MARKER").write_text("local-dev\n", encoding="utf-8") + + self.run_update() + + self.assertEqual( + "local-dev\n", + (source / "MARKER").read_text(encoding="utf-8"), + ) + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + service_row = self.manifest_rows()[1] + self.assertEqual("pre-existing-checkout", service_row["authority"]) + self.assertEqual(self.service_new, service_row["frozen_commit"]) + + def test_unversioned_preexisting_source_fails_before_mutation(self): + source = self.project_root / "src" / "test-svc" + source.mkdir() + (source / "requirements.txt").write_text("six\n", encoding="utf-8") + original_sources = (self.project_root / "sources.txt").read_bytes() + + result = self.run_update_result() + + self.assertNotEqual(0, result.returncode) + self.assertIn("is not a Git checkout", result.stderr) + self.assertEqual( + original_sources, (self.project_root / "sources.txt").read_bytes() + ) + self.assertFalse( + (self.project_root / "upper-constraints.txt.master").exists() + ) + self.assertFalse(self.manifest_path.exists()) + + def test_slash_stream_uses_safe_manifest_filename(self): + result = self.run_update_result(STREAM="stable/test") + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + manifest = self.manifest_for_stream("stable/test") + self.assertTrue(manifest.is_file()) + self.assertEqual( + self.test_root / ".tmp/source-maintenance", manifest.parent + ) + self.assertFalse( + (self.test_root / ".tmp/source-maintenance/stable").exists() + ) + + def test_unsafe_stream_fails_before_filesystem_mutation(self): + original_sources = (self.project_root / "sources.txt").read_bytes() + + result = self.run_update_result(STREAM="../../escape") + + self.assertNotEqual(0, result.returncode) + self.assertIn("Unsafe stream name", result.stderr) + self.assertEqual( + original_sources, (self.project_root / "sources.txt").read_bytes() + ) + self.assertFalse((self.test_root / ".tmp").exists()) + + def test_unresolvable_ref_fails_before_mutation(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} missing-branch {self.service_old}\n" + ) + original_sources = (self.project_root / "sources.txt").read_bytes() + + result = self.run_update_result() + + self.assertNotEqual(0, result.returncode) + self.assertIn("Could not freeze ref 'missing-branch'", result.stderr) + self.assertEqual( + original_sources, (self.project_root / "sources.txt").read_bytes() + ) + self.assertFalse( + (self.project_root / "upper-constraints.txt.master").exists() + ) + self.assertFalse(self.manifest_path.exists()) + + def test_failed_preflight_removes_stale_manifest(self): + self.run_update() + self.assertTrue(self.manifest_path.is_file()) + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_new}\n" + "master test-svc " + f"{self.upstream_service} missing-branch {self.service_new}\n" + ) + + result = self.run_update_result() + + self.assertNotEqual(0, result.returncode) + self.assertFalse(self.manifest_path.exists()) + + def test_list_discovers_all_projects(self): + result = subprocess.run( + ["bash", "./build.sh", "list"], + cwd=self.test_root, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("test-svc/test-svc", result.stdout) + self.assertIn("test-svc2/test-svc2", result.stdout) + self.assertIn("test-svc3/test-svc3", result.stdout) + + def test_multiple_targets_update_only_selected_projects(self): + self.run_update("test-svc", "test-svc2", DEFAULT_STREAM="master") + + service2_new = self.additional_services["test-svc2"][2] + service3_old = self.additional_services["test-svc3"][1] + project2 = self.test_root / "containers/test-svc2" + project3 = self.test_root / "containers/test-svc3" + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + self.assertEqual( + service2_new, + self.source_field("test-svc2", 5, project2), + ) + self.assertEqual( + service3_old, + self.source_field("test-svc3", 5, project3), + ) + for project in (self.project_root, project2): + self.assertTrue( + (project / "upper-constraints.txt.master").is_file() + ) + self.assertTrue((project / "requirements.lock.master").is_file()) + self.assertTrue((project / "rpms.in.yaml").is_file()) + self.assertTrue((project / "requirements.lock").is_symlink()) + self.assertFalse((project3 / "upper-constraints.txt.master").exists()) + self.assertFalse((project3 / "requirements.lock.master").exists()) + self.assertFalse((project3 / "rpms.in.yaml").exists()) + + def test_single_target_does_not_affect_other_projects(self): + self.run_update("test-svc") + + project2 = self.test_root / "containers/test-svc2" + self.assertEqual( + self.additional_services["test-svc2"][1], + self.source_field("test-svc2", 5, project2), + ) + self.assertFalse((project2 / "requirements.lock.master").exists()) + + def test_all_updates_every_project(self): + self.run_update("all") + + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + for name in ("test-svc2", "test-svc3"): + project = self.test_root / "containers" / name + self.assertEqual( + self.additional_services[name][2], + self.source_field(name, 5, project), + ) + self.assertTrue((project / "requirements.lock.master").is_file()) + + def test_unknown_target_fails_before_mutation(self): + result = self.run_update_result("nonexistent") + + self.assertNotEqual(0, result.returncode) + self.assertIn("ERROR: Unknown image or project", result.stderr) + self.assertFalse(self.manifest_path.exists()) + + def test_duplicate_ref_records_retain_deterministic_rows(self): + image_sources = self.project_root / "test-svc" / "sources.txt" + image_sources.write_text( + "master helper " + f"{self.upstream_service} master {self.service_old}\n", + encoding="utf-8", + ) + + self.run_update() + + rows = self.manifest_rows() + self.assertEqual( + ["upper-constraints", "test-svc", "helper"], + [row["name"] for row in rows], + ) + self.assertEqual(rows[1]["frozen_commit"], rows[2]["frozen_commit"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_update_sources.sh b/tests/test_update_sources.sh deleted file mode 100755 index 9a3ed9b6..00000000 --- a/tests/test_update_sources.sh +++ /dev/null @@ -1,529 +0,0 @@ -#!/usr/bin/env bash -# Tests for build.sh update-sources functionality. -# -# Uses local bare git repos as fake remotes so tests run offline and fast. -# pip-compile and pybuild-deps must be on PATH for lockfile tests. -# -# Usage: -# PATH=".tox/update-sources/bin:$PATH" bash tests/test_update_sources.sh -# tox -e test -# -set -uo pipefail - -# ── Test runner ────────────────────────────────────────────────────────── - -_PASS=0 -_FAIL=0 -_SKIP=0 - -assert() { - local desc="$1" - shift - if "$@"; then - return 0 - fi - echo " ASSERTION FAILED: ${desc}" - echo " command: $*" - return 1 -} - -assert_file_exists() { assert "file exists: $1" test -f "$1"; } -assert_symlink() { assert "symlink exists: $1" test -L "$1"; } -assert_no_symlink() { assert "no symlink: $1" test ! -L "$1"; } -assert_grep() { assert "grep '$1' in $2" grep -q "$1" "$2"; } -assert_no_grep() { - if grep -q "$1" "$2" 2>/dev/null; then - echo " ASSERTION FAILED: '$1' should not appear in $2" - return 1 - fi -} - -assert_link_target() { - local link="$1" expected="$2" - local actual - actual="$(readlink "$1")" - assert "symlink $1 -> $2 (actual: ${actual})" test "${actual}" = "${expected}" -} - -assert_field() { - local file="$1" stream="$2" name="$3" field="$4" expected="$5" - local actual - actual=$(awk -v s="${stream}" -v n="${name}" '$1==s && $2==n {print $'${field}'}' "${file}") - assert "sources.txt ${name} field ${field} == ${expected} (actual: ${actual})" \ - test "${actual}" = "${expected}" -} - -run_test() { - local name="$1" - - _setup_fixture - - local rc=0 - # Run in subshell with set -e so first failed assertion stops the test - ( set -e; "${name}" ) || rc=$? - - if [[ ${rc} -eq 0 ]]; then - echo " PASS ${name}" - ((_PASS++)) - elif [[ ${rc} -eq 99 ]]; then - echo " SKIP ${name}" - ((_SKIP++)) - else - echo " FAIL ${name}" - ((_FAIL++)) - if [[ -f "${TEST_DIR}/build.log" ]]; then - echo " --- build.log (last 20 lines) ---" - tail -20 "${TEST_DIR}/build.log" | sed 's/^/ /' - echo " ---" - fi - fi - - _teardown_fixture -} - -skip_test() { - echo " skipping: $1" - return 99 -} - -# ── Fixture ────────────────────────────────────────────────────────────── - -SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -TEST_DIR="" -UPSTREAM_REQ="" -UPSTREAM_SVC="" -UPSTREAM_SVC2="" -UPSTREAM_SVC3="" -REQ_HASH_OLD="" -REQ_HASH_NEW="" -SVC_HASH_OLD="" -SVC_HASH_NEW="" -SVC2_HASH_OLD="" -SVC2_HASH_NEW="" -SVC3_HASH_OLD="" -SVC3_HASH_NEW="" - -_init_work_repo() { - local dir="$1" - git init -b master "${dir}" >/dev/null 2>&1 - git -C "${dir}" config user.email "test@test.com" - git -C "${dir}" config user.name "Test" -} - -_setup_fixture() { - TEST_DIR="$(mktemp -d)" - local work - - # ── Upstream requirements repo (2 commits) ── - UPSTREAM_REQ="${TEST_DIR}/upstream/requirements.git" - mkdir -p "${TEST_DIR}/upstream" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six==1.17.0" > "${work}/upper-constraints.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six==1.17.0\npbr==7.0.3\n' > "${work}/upper-constraints.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_REQ}" >/dev/null 2>&1 - rm -rf "${work}" - - REQ_HASH_OLD="$(git -C "${UPSTREAM_REQ}" rev-parse master~1)" - REQ_HASH_NEW="$(git -C "${UPSTREAM_REQ}" rev-parse master)" - - # ── Upstream service repo (2 commits) ── - UPSTREAM_SVC="${TEST_DIR}/upstream/test-svc.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC_HASH_OLD="$(git -C "${UPSTREAM_SVC}" rev-parse master~1)" - SVC_HASH_NEW="$(git -C "${UPSTREAM_SVC}" rev-parse master)" - - # ── Upstream service 2 repo (2 commits) ── - UPSTREAM_SVC2="${TEST_DIR}/upstream/test-svc2.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC2}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC2_HASH_OLD="$(git -C "${UPSTREAM_SVC2}" rev-parse master~1)" - SVC2_HASH_NEW="$(git -C "${UPSTREAM_SVC2}" rev-parse master)" - - # ── Upstream service 3 repo (2 commits) ── - UPSTREAM_SVC3="${TEST_DIR}/upstream/test-svc3.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC3}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC3_HASH_OLD="$(git -C "${UPSTREAM_SVC3}" rev-parse master~1)" - SVC3_HASH_NEW="$(git -C "${UPSTREAM_SVC3}" rev-parse master)" - - # ── Symlink build.sh ── - ln -s "${SCRIPT_DIR}/build.sh" "${TEST_DIR}/build.sh" - - # ── Containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc/src" - mkdir -p "${TEST_DIR}/containers/test-svc/test-svc/src" - - cat > "${TEST_DIR}/containers/test-svc/sources.txt" < "${TEST_DIR}/containers/test-svc/test-svc/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc/test-svc/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc/test-svc/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc/test-svc/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc/test-svc/pythonbuilddeps.txt" - - # ── Second project containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc2/src" - mkdir -p "${TEST_DIR}/containers/test-svc2/test-svc2/src" - - cat > "${TEST_DIR}/containers/test-svc2/sources.txt" < "${TEST_DIR}/containers/test-svc2/test-svc2/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc2/test-svc2/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc2/test-svc2/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc2/test-svc2/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc2/test-svc2/pythonbuilddeps.txt" - - # ── Third project containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc3/src" - mkdir -p "${TEST_DIR}/containers/test-svc3/test-svc3/src" - - cat > "${TEST_DIR}/containers/test-svc3/sources.txt" < "${TEST_DIR}/containers/test-svc3/test-svc3/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc3/test-svc3/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc3/test-svc3/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc3/test-svc3/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc3/test-svc3/pythonbuilddeps.txt" -} - -_teardown_fixture() { - [[ -n "${TEST_DIR}" ]] && rm -rf "${TEST_DIR}" -} - -# Helper: run build.sh inside TEST_DIR with env vars passed as arguments. -# Usage: _run_build STREAM=master [SKIP_HASH_UPDATE=1 ...] -_run_build() { - (cd "${TEST_DIR}" && env "$@" ./build.sh update-sources test-svc) >"${TEST_DIR}/build.log" 2>&1 -} - -# Helper: run build.sh with arbitrary action and targets. -# Usage: _run_cmd -- [targets...] -_run_cmd() { - local env_args=() - while [[ $# -gt 0 && "$1" != "--" ]]; do - env_args+=("$1") - shift - done - [[ "$1" == "--" ]] && shift - (cd "${TEST_DIR}" && env "${env_args[@]}" ./build.sh "$@") >"${TEST_DIR}/build.log" 2>&1 -} - -# ── Tests ──────────────────────────────────────────────────────────────── - -test_updates_hashes_to_branch_tip() { - _run_build STREAM=master - - local src="${TEST_DIR}/containers/test-svc/sources.txt" - assert_field "${src}" master upper-constraints 5 "${REQ_HASH_NEW}" - assert_field "${src}" master test-svc 5 "${SVC_HASH_NEW}" -} - -test_fetches_upper_constraints() { - _run_build STREAM=master - - local uc="${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${uc}" - assert_grep "six==1.17.0" "${uc}" - assert_grep "pbr==7.0.3" "${uc}" -} - -test_generates_rpms_in_yaml() { - _run_build STREAM=master - - local rpms="${TEST_DIR}/containers/test-svc/rpms.in.yaml" - assert_file_exists "${rpms}" - assert_grep "python3" "${rpms}" - assert_grep "gcc" "${rpms}" -} - -test_generates_requirements_lock() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_build STREAM=master - - local lock="${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${lock}" - assert_grep "six" "${lock}" -} - -test_generates_buildrequirements_lock() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master - - assert_file_exists "${TEST_DIR}/containers/test-svc/buildrequirements.lock.master" -} - -test_creates_default_stream_symlinks() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master DEFAULT_STREAM=master - - local d="${TEST_DIR}/containers/test-svc" - assert_symlink "${d}/upper-constraints.txt" - assert_symlink "${d}/requirements.lock" - assert_symlink "${d}/buildrequirements.lock" - assert_link_target "${d}/requirements.lock" "requirements.lock.master" - assert_link_target "${d}/buildrequirements.lock" "buildrequirements.lock.master" - assert_link_target "${d}/upper-constraints.txt" "upper-constraints.txt.master" -} - -test_skips_symlinks_for_non_default_stream() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master DEFAULT_STREAM=other - - local d="${TEST_DIR}/containers/test-svc" - assert_no_symlink "${d}/requirements.lock" - assert_no_symlink "${d}/buildrequirements.lock" - assert_no_symlink "${d}/upper-constraints.txt" -} - -test_skip_hash_update_preserves_hashes() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_build STREAM=master SKIP_HASH_UPDATE=1 - - local src="${TEST_DIR}/containers/test-svc/sources.txt" - assert_field "${src}" master upper-constraints 5 "${REQ_HASH_OLD}" - assert_field "${src}" master test-svc 5 "${SVC_HASH_OLD}" - - assert_file_exists "${TEST_DIR}/containers/test-svc/requirements.lock.master" -} - -test_skip_hash_update_fetches_constraints_at_pinned_hash() { - _run_build STREAM=master SKIP_HASH_UPDATE=1 - - local uc="${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${uc}" - assert_grep "six==1.17.0" "${uc}" - assert_no_grep "pbr" "${uc}" -} - -test_hash_in_branch_field_upper_constraints() { - cat > "${TEST_DIR}/containers/test-svc/sources.txt" </dev/null 2>&1 || skip_test "pip-compile not on PATH" - - cat > "${TEST_DIR}/containers/test-svc/sources.txt" </dev/null 2>&1 || skip_test "pip-compile not on PATH" - - printf 'python3\npython3-six\n' > "${TEST_DIR}/containers/test-svc/test-svc/bindeps.txt" - - _run_build STREAM=master - - local lock="${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${lock}" - assert_no_grep "^six==" "${lock}" - assert_grep "Filtering RPM-provided packages" "${TEST_DIR}/build.log" -} - -test_preexisting_checkout_is_preserved() { - local src_dir="${TEST_DIR}/containers/test-svc/src/test-svc" - mkdir -p "${src_dir}" - echo "local-dev" > "${src_dir}/MARKER" - echo "six" > "${src_dir}/requirements.txt" - - _run_build STREAM=master - - assert_file_exists "${src_dir}/MARKER" - assert_grep "local-dev" "${src_dir}/MARKER" - assert_field "${TEST_DIR}/containers/test-svc/sources.txt" master test-svc 5 "${SVC_HASH_OLD}" -} - -# ── Multi-target tests ────────────────────────────────────────────────── - -test_list_discovers_all_projects() { - _run_cmd STREAM=master -- list - assert_grep "test-svc/test-svc" "${TEST_DIR}/build.log" - assert_grep "test-svc2/test-svc2" "${TEST_DIR}/build.log" -} - -test_update_sources_multiple_targets() { - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - local src1="${TEST_DIR}/containers/test-svc/sources.txt" - local src2="${TEST_DIR}/containers/test-svc2/sources.txt" - local src3="${TEST_DIR}/containers/test-svc3/sources.txt" - assert_field "${src1}" master test-svc 5 "${SVC_HASH_NEW}" - assert_field "${src2}" master test-svc2 5 "${SVC2_HASH_NEW}" - assert_field "${src3}" master test-svc3 5 "${SVC3_HASH_OLD}" -} - -test_update_sources_multiple_targets_fetches_constraints() { - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - assert_file_exists "${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${TEST_DIR}/containers/test-svc2/upper-constraints.txt.master" - assert "no constraints for test-svc3" test ! -f "${TEST_DIR}/containers/test-svc3/upper-constraints.txt.master" -} - -test_update_sources_multiple_targets_generates_lockfiles() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - assert_file_exists "${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${TEST_DIR}/containers/test-svc2/requirements.lock.master" - assert "no lockfile for test-svc3" test ! -f "${TEST_DIR}/containers/test-svc3/requirements.lock.master" -} - -test_update_sources_multiple_targets_generates_rpms_in() { - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - assert_file_exists "${TEST_DIR}/containers/test-svc/rpms.in.yaml" - assert_file_exists "${TEST_DIR}/containers/test-svc2/rpms.in.yaml" - assert "no rpms.in.yaml for test-svc3" test ! -f "${TEST_DIR}/containers/test-svc3/rpms.in.yaml" -} - -test_update_sources_single_target_does_not_affect_other() { - _run_cmd STREAM=master -- update-sources test-svc - - local src2="${TEST_DIR}/containers/test-svc2/sources.txt" - assert_field "${src2}" master test-svc2 5 "${SVC2_HASH_OLD}" -} - -test_update_sources_all_updates_everything() { - _run_cmd STREAM=master -- update-sources all - - local src1="${TEST_DIR}/containers/test-svc/sources.txt" - local src2="${TEST_DIR}/containers/test-svc2/sources.txt" - local src3="${TEST_DIR}/containers/test-svc3/sources.txt" - assert_field "${src1}" master test-svc 5 "${SVC_HASH_NEW}" - assert_field "${src2}" master test-svc2 5 "${SVC2_HASH_NEW}" - assert_field "${src3}" master test-svc3 5 "${SVC3_HASH_NEW}" -} - -test_update_sources_unknown_target_fails() { - if _run_cmd STREAM=master -- update-sources nonexistent 2>/dev/null; then - echo " ASSERTION FAILED: expected failure for unknown target" - return 1 - fi - assert_grep "ERROR: Unknown image or project" "${TEST_DIR}/build.log" -} - -test_update_sources_multiple_targets_symlinks() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_cmd STREAM=master DEFAULT_STREAM=master -- update-sources test-svc test-svc2 - - for proj in test-svc test-svc2; do - local d="${TEST_DIR}/containers/${proj}" - assert_symlink "${d}/requirements.lock" - assert_link_target "${d}/requirements.lock" "requirements.lock.master" - done -} - -# ── Run all tests ──────────────────────────────────────────────────────── - -echo "=== update-sources tests ===" -echo "" - -TESTS=( - test_updates_hashes_to_branch_tip - test_fetches_upper_constraints - test_generates_rpms_in_yaml - test_generates_requirements_lock - test_generates_buildrequirements_lock - test_creates_default_stream_symlinks - test_skips_symlinks_for_non_default_stream - test_skip_hash_update_preserves_hashes - test_skip_hash_update_fetches_constraints_at_pinned_hash - test_hash_in_branch_field_upper_constraints - test_hash_in_branch_field_regular_repo - test_lockfile_excludes_rpm_python_packages - test_preexisting_checkout_is_preserved - test_list_discovers_all_projects - test_update_sources_multiple_targets - test_update_sources_multiple_targets_fetches_constraints - test_update_sources_multiple_targets_generates_lockfiles - test_update_sources_multiple_targets_generates_rpms_in - test_update_sources_single_target_does_not_affect_other - test_update_sources_all_updates_everything - test_update_sources_unknown_target_fails - test_update_sources_multiple_targets_symlinks -) - -for t in "${TESTS[@]}"; do - run_test "${t}" -done - -echo "" -echo "=== ${_PASS} passed, ${_FAIL} failed, ${_SKIP} skipped ===" - -[[ ${_FAIL} -eq 0 ]] diff --git a/tox.ini b/tox.ini index 988d08a6..bec1fbe4 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,11 @@ deps = allowlist_externals = bash +[testenv:{unit,test,py3}] +description = Run all non-Ansible tests through stdlib unittest +commands = + python -m unittest discover -s {toxinidir}/tests -p 'test_*.py' -v {posargs} + [testenv:linters] description = Run formatting and style checks through pre-commit deps = @@ -72,10 +77,6 @@ commands = commands = bash {toxinidir}/build.sh build {posargs:all} -[testenv:test] -commands = - bash {toxinidir}/tests/test_update_sources.sh - [testenv:custom] commands = bash {toxinidir}/build.sh {posargs} From 822f61e99223012af3357601487d012d1414b580 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Wed, 12 Aug 2026 17:48:43 +0100 Subject: [PATCH 3/3] Add a Zuul image content provider Dependent jobs need exact references to service images built from the current container repository, but the repository has no reusable job that publishes a selected image set to the Zuul buildset registry. Add shared provider playbooks, a Zuul adapter, and contract validation for registry handling, deployment mappings, returned data, and exact cleanup. Extend the build interface with ordered target unions, machine-readable target and reference output, authenticated pushes, and live parallel logs. Attach the provider and Molecule contract jobs to the GitHub check pipeline. Assisted-By: Pi gpt-5.6-sol Signed-off-by: Sean Mooney (cherry picked from commit 028d93283544020362adb789aece89e4634a6f34) --- build.sh | 168 ++++++-- containers/watcher/rpms.in.yaml | 1 + containers/watcher/watcher-base/bindeps.txt | 1 + docs/developer-guide.md | 83 ++++ molecule/provider-contract/cleanup.yml | 13 + molecule/provider-contract/converge.yml | 126 ++++++ .../provider-contract/duplicate-metadata.yml | 16 + .../files/fake-container-client.py | 55 +++ .../provider-contract/malformed-mapping.yml | 18 + molecule/provider-contract/molecule.yml | 34 ++ molecule/provider-contract/prepare.yml | 57 +++ .../provider-contract/unknown-mapping.yml | 17 + molecule/provider-contract/verify.yml | 89 +++++ .../container-ci/shared/cleanup-images.yaml | 208 ++++++++++ .../shared/deployment-map-item.yaml | 7 + .../shared/image-metadata-item.yaml | 42 ++ .../shared/load-image-mappings.yaml | 67 ++++ .../container-ci/shared/prepare-host.yaml | 46 +++ .../shared/published-image-item.yaml | 10 + playbooks/container-ci/shared/run.yaml | 361 ++++++++++++++++++ .../shared/validate-deployment-keys.yaml | 12 + .../shared/validate-mapping-overrides.yaml | 8 + .../shared/validate-registry.yaml | 109 ++++++ .../shared/validate-tracked-mapping-item.yaml | 24 ++ .../zuul/content-provider-return.yaml | 32 ++ playbooks/container-ci/zuul/post.yaml | 129 +++++++ playbooks/container-ci/zuul/pre.yaml | 127 ++++++ playbooks/container-ci/zuul/run.yaml | 12 + tests/test_provider_architecture.py | 254 ++++++++++++ tests/test_provider_shell.py | 200 ++++++++++ tox.ini | 17 + zuul.d/jobs.yaml | 38 ++ zuul.d/projects.yaml | 10 +- 33 files changed, 2346 insertions(+), 45 deletions(-) create mode 100644 molecule/provider-contract/cleanup.yml create mode 100644 molecule/provider-contract/converge.yml create mode 100644 molecule/provider-contract/duplicate-metadata.yml create mode 100755 molecule/provider-contract/files/fake-container-client.py create mode 100644 molecule/provider-contract/malformed-mapping.yml create mode 100644 molecule/provider-contract/molecule.yml create mode 100644 molecule/provider-contract/prepare.yml create mode 100644 molecule/provider-contract/unknown-mapping.yml create mode 100644 molecule/provider-contract/verify.yml create mode 100644 playbooks/container-ci/shared/cleanup-images.yaml create mode 100644 playbooks/container-ci/shared/deployment-map-item.yaml create mode 100644 playbooks/container-ci/shared/image-metadata-item.yaml create mode 100644 playbooks/container-ci/shared/load-image-mappings.yaml create mode 100644 playbooks/container-ci/shared/prepare-host.yaml create mode 100644 playbooks/container-ci/shared/published-image-item.yaml create mode 100644 playbooks/container-ci/shared/run.yaml create mode 100644 playbooks/container-ci/shared/validate-deployment-keys.yaml create mode 100644 playbooks/container-ci/shared/validate-mapping-overrides.yaml create mode 100644 playbooks/container-ci/shared/validate-registry.yaml create mode 100644 playbooks/container-ci/shared/validate-tracked-mapping-item.yaml create mode 100644 playbooks/container-ci/zuul/content-provider-return.yaml create mode 100644 playbooks/container-ci/zuul/post.yaml create mode 100644 playbooks/container-ci/zuul/pre.yaml create mode 100644 playbooks/container-ci/zuul/run.yaml create mode 100644 tests/test_provider_architecture.py create mode 100644 tests/test_provider_shell.py create mode 100644 zuul.d/jobs.yaml diff --git a/build.sh b/build.sh index 9938efd9..f7461cc6 100755 --- a/build.sh +++ b/build.sh @@ -83,6 +83,8 @@ # PIP_NO_BINARY If set, passed as --build-arg to buildah so Containerfiles # can set ENV PIP_NO_BINARY. Use ":all:" to force pip to # build all packages from source instead of using wheels. +# REGISTRY_AUTH_FILE Authentication file passed explicitly to buildah push. +# REGISTRY_CERT_DIR TLS certificate directory passed explicitly to buildah push. set -euo pipefail @@ -102,6 +104,8 @@ UPSTREAM_CONSTRAINTS="upper-constraints.txt" DEFAULT_STREAM="${DEFAULT_STREAM:-master}" SKIP_HASH_UPDATE="${SKIP_HASH_UPDATE:-}" PIP_NO_BINARY="${PIP_NO_BINARY:-}" +REGISTRY_AUTH_FILE="${REGISTRY_AUTH_FILE:-}" +REGISTRY_CERT_DIR="${REGISTRY_CERT_DIR:-}" PARALLEL="${PARALLEL:-$(nproc)}" SOURCE_REFS_STREAM="${STREAM//\//%2F}" SOURCE_REFS_MANIFEST="${REPO_ROOT}/.tmp/source-maintenance/frozen-source-refs.${SOURCE_REFS_STREAM}.tsv" @@ -376,10 +380,32 @@ push_image() { name="$(image_name "${dir_name}")" IFS=',' read -ra tags <<< "${TAG}" + local registry_args=() + [[ -n "${REGISTRY_AUTH_FILE}" ]] && \ + registry_args+=(--authfile "${REGISTRY_AUTH_FILE}") + [[ -n "${REGISTRY_CERT_DIR}" ]] && \ + registry_args+=(--cert-dir "${REGISTRY_CERT_DIR}") + for t in "${tags[@]}"; do local full_tag="${REGISTRY}/${NAMESPACE}/${name}:${t}" echo "=== Pushing ${full_tag} ===" - buildah push "${full_tag}" + buildah push "${registry_args[@]}" "${full_tag}" + done +} + +# Print the exact image references produced for a target. +target_refs() { + local dir_name + local name + local tag + local -a tags + resolve_targets_array "$@" || return 1 + for dir_name in "${_RESOLVED_TARGETS[@]}"; do + name="$(image_name "${dir_name}")" + IFS=',' read -ra tags <<< "${TAG}" + for tag in "${tags[@]}"; do + echo "${REGISTRY}/${NAMESPACE}/${name}:${tag}" + done done } @@ -403,47 +429,86 @@ list_images() { fi } -# Resolve which images to process (accepts one or more targets) +# Resolve one or more image expressions in their requested order. A +# comma-separated expression is an explicit ordered union and includes base +# when it selects a service image. Separate positional targets preserve the +# upstream multi-target behavior. resolve_targets() { local all_images all_images=($(discover_images)) - local resolved=() + local -a resolved=() + local expression item image item_images + declare -A seen=() - for target in "$@"; do - if [[ "${target}" == "all" ]]; then + for expression in "$@"; do + if [[ "${expression}" == "all" ]]; then echo "${all_images[@]}" return fi - local found=0 + if [[ "${expression}" == *,* ]]; then + local -a requested + local selected_service=0 + IFS=',' read -ra requested <<< "${expression}" + for item in "${requested[@]}"; do + if [[ -z "${item}" || "${item}" != "${item//[[:space:]]/}" ]]; then + echo "ERROR: Invalid empty or whitespace-containing target '${item}'" >&2 + return 1 + fi + if ! item_images=$(resolve_targets "${item}"); then + return 1 + fi + for image in ${item_images}; do + [[ "${image}" != "base" ]] && selected_service=1 + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi + done + done + if [[ ${selected_service} -eq 1 && -z "${seen[base]:-}" ]]; then + resolved=("base" "${resolved[@]}") + seen[base]=1 + fi + continue + fi - # Exact match - for dir_name in "${all_images[@]}"; do - if [[ "${dir_name}" == "${target}" ]]; then - resolved+=("${target}") + local found=0 + for image in "${all_images[@]}"; do + if [[ "${image}" == "${expression}" ]]; then + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi found=1 break fi done [[ ${found} -eq 1 ]] && continue - # Project prefix match - for dir_name in "${all_images[@]}"; do - if [[ "${dir_name}" == "${target}/"* ]]; then - resolved+=("${dir_name}") + for image in "${all_images[@]}"; do + if [[ "${image}" == "${expression}/"* ]]; then + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi found=1 fi done [[ ${found} -eq 1 ]] && continue - echo "ERROR: Unknown image or project '${target}'" >&2 + echo "ERROR: Unknown image or project '${expression}'" >&2 echo "Available images:" >&2 - for dir_name in "${all_images[@]}"; do - echo " ${dir_name}" >&2 + for image in "${all_images[@]}"; do + echo " ${image}" >&2 done return 1 done + if [[ ${#resolved[@]} -eq 0 ]]; then + echo "ERROR: Explicit target selection is empty" >&2 + return 1 + fi echo "${resolved[@]}" } @@ -1150,11 +1215,24 @@ case "${ACTION}" in ;; build-parallel) _bp_targets=($(resolve_targets "${TARGETS[@]}")) + if [[ ! "${PARALLEL}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: PARALLEL must be a positive integer" >&2 + exit 1 + fi + if [[ -n "${BUILD_LOGS_DIR:-}" ]]; then + _bp_logdir="${BUILD_LOGS_DIR}" + mkdir -p "${_bp_logdir}" + else + _bp_logdir=$(mktemp -d) + fi # Build base first (all service images depend on it) for _bp_img in "${_bp_targets[@]}"; do [[ -n "$(project_name "${_bp_img}")" ]] && continue - build_image "${_bp_img}" + set -o pipefail + build_image "${_bp_img}" 2>&1 | + sed -u "s|^|[${_bp_img}] |" | + tee "${_bp_logdir}/${_bp_img//\//_}.log" done # Pre-clone sources so parallel builds don't race on the same directories @@ -1164,12 +1242,6 @@ case "${ACTION}" in done # Build service images in parallel (max PARALLEL at a time) - if [[ -n "${BUILD_LOGS_DIR:-}" ]]; then - _bp_logdir="${BUILD_LOGS_DIR}" - mkdir -p "${_bp_logdir}" - else - _bp_logdir=$(mktemp -d) - fi _bp_service_imgs=() for _bp_img in "${_bp_targets[@]}"; do [[ -z "$(project_name "${_bp_img}")" ]] && continue @@ -1183,47 +1255,50 @@ case "${ACTION}" in _bp_running=0 for _bp_img in "${_bp_service_imgs[@]}"; do - # Wait for a slot if at the limit while [[ ${_bp_running} -ge ${PARALLEL} ]]; do - if ! wait -n; then + _bp_finished="" + if wait -n -p _bp_finished "${!_bp_pids[@]}"; then + unset '_bp_pids['"${_bp_finished}"']' + ((_bp_running--)) || true + else _bp_fail=1 + [[ -n "${_bp_finished}" ]] && unset '_bp_pids['"${_bp_finished}"']' break 2 fi - ((_bp_running--)) || true done _bp_log="${_bp_logdir}/${_bp_img//\//_}.log" - build_image "${_bp_img}" > "${_bp_log}" 2>&1 & + ( + set -o pipefail + build_image "${_bp_img}" 2>&1 | + sed -u "s|^|[${_bp_img}] |" | + tee "${_bp_log}" + ) & _bp_pids[$!]="${_bp_img}" ((_bp_running++)) || true done - # Wait for remaining builds if [[ ${_bp_fail} -eq 0 ]]; then while [[ ${_bp_running} -gt 0 ]]; do - if ! wait -n; then + _bp_finished="" + if wait -n -p _bp_finished "${!_bp_pids[@]}"; then + unset '_bp_pids['"${_bp_finished}"']' + ((_bp_running--)) || true + else _bp_fail=1 + [[ -n "${_bp_finished}" ]] && unset '_bp_pids['"${_bp_finished}"']' break fi - ((_bp_running--)) || true done fi - # Show logs for all builds - for _bp_log in "${_bp_logdir}"/*.log; do - _bp_name=$(basename "${_bp_log}" .log) - echo "=== ${_bp_name} ===" - cat "${_bp_log}" - echo "" - done - if [[ ${_bp_fail} -eq 1 ]]; then - echo "ERROR: A build failed, killing remaining builds" >&2 + echo "ERROR: A build failed; stopping remaining builds" >&2 for _bp_pid in "${!_bp_pids[@]}"; do kill "${_bp_pid}" 2>/dev/null || true done wait 2>/dev/null || true - [[ -z "${BUILD_LOGS_DIR:-}" ]] && rm -rf "${_bp_logdir}" + echo "Build logs are available in ${_bp_logdir}" >&2 exit 1 fi @@ -1245,6 +1320,13 @@ case "${ACTION}" in push_image "${img}" done ;; + refs) + target_refs "${TARGETS[@]}" + ;; + resolve) + resolve_targets_array "${TARGETS[@]}" + printf '%s\n' "${_RESOLVED_TARGETS[@]}" + ;; update-sources) if [[ -n "${SKIP_HASH_UPDATE}" ]]; then echo "=== Skipping hash update (SKIP_HASH_UPDATE is set) ===" @@ -1313,7 +1395,7 @@ case "${ACTION}" in list_images ;; *) - echo "Usage: STREAM= $0 {build|build-parallel|push|update-sources|install-deps|list} [target ...]" + echo "Usage: STREAM= $0 {build|build-parallel|push|refs|resolve|update-sources|install-deps|list} [target ...]" echo "" echo "Images (discovered from containers/):" for dir_name in $(discover_images); do @@ -1337,6 +1419,8 @@ case "${ACTION}" in echo " BUILD_LOGS_DIR Persist build-parallel logs to this directory" echo " SKIP_HASH_UPDATE Skip updating pinned hashes; regenerate locks only" echo " PIP_NO_BINARY Pass PIP_NO_BINARY to container build (e.g., ':all:')" + echo " REGISTRY_AUTH_FILE Registry authentication file for pushes" + echo " REGISTRY_CERT_DIR Registry TLS certificate directory for pushes" echo "" echo "Source directories: containers//src//" echo "Overrides: containers//src/overrides//" diff --git a/containers/watcher/rpms.in.yaml b/containers/watcher/rpms.in.yaml index ada31b87..9a0608dc 100644 --- a/containers/watcher/rpms.in.yaml +++ b/containers/watcher/rpms.in.yaml @@ -27,6 +27,7 @@ packages: - libxslt-devel - mod_ssl - openssl-devel + - openssl-libs - python3 - python3-cryptography - python3-devel diff --git a/containers/watcher/watcher-base/bindeps.txt b/containers/watcher/watcher-base/bindeps.txt index ade6c10f..6b9ce44f 100644 --- a/containers/watcher/watcher-base/bindeps.txt +++ b/containers/watcher/watcher-base/bindeps.txt @@ -9,3 +9,4 @@ python3-cryptography libffi libxml2 libxslt +openssl-libs diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 7926331e..cae48b2e 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -308,6 +308,89 @@ All image tags are verified to exist locally before any push begins. ./build.sh list ``` +## Zuul content provider + +The `s2i-openstack-container-content-provider` job runs on the +CentOS Stream 10 nodeset host named `builder`. All host preparation, registry +validation, builds, publication, result generation, and cleanup target that +host explicitly. The Zuul executor controls Ansible but does not perform those +mutations. + +In this repository the provider defaults to `all`, so every maintained image +is built from its exact `sources.txt` pins. A child job can set `s2i_ci_images` +to an explicit list of image targets; the provider adds `base`, resolves the +selection through `build.sh`, and publishes only that set. + +To compose the provider in an operator repository, inherit the job, add the +container repository and any related repositories to `required-projects`, set +`s2i_ci_container_project`, and override `s2i_ci_images` with the required +subset. Zuul places those projects in the shared buildset workspace, but this +provider does not yet consume speculative service checkouts or build operators. +Those integrations remain follow-up scope; the current provider always uses +maintained source pins. + +### Image deployment metadata + +The repository-level `containers/image-mappings.yaml` associates exact build +targets with OpenStackVersion custom-image fields. Unlisted targets have no +deployment mapping but are still built and returned. The consolidated +`watcher/watcher-base` image declares: + +```yaml +openstack_version: + custom_container_images: + watcher/watcher-base: + - watcherAPIImage + - watcherApplierImage + - watcherDecisionEngineImage +``` + +All three keys resolve to the same exact `openstack-watcher-base` reference. +The image contains the API, applier, and decision-engine entry points and the +union of their runtime dependencies. Watcher is intentionally not split into +process-specific images. + +Both Cyborg images build and publish but are absent from the central mapping. +Their exact references therefore appear in provider diagnostics without adding +fields to the default deployment map. + +A child job may provide `s2i_ci_image_mappings` as a mapping from a selected +image target to a replacement list of keys. Replacement is per image rather +than additive. The provider records whether each effective list came from +tracked or inventory metadata and rejects malformed values, unknown or unbuilt +image targets, empty key strings, duplicate keys, and a key assigned to more +than one image. + +### Registry and returned data + +The provider starts or inherits a Zuul buildset registry, validates push and +pull with a dedicated UBI tag, builds and pushes the selected image set, and +pulls every exact result back. Credentials and certificate data remain in +Zuul secret data. Returned public diagnostics use the buildset registry's +reachable host or IP and port, never the builder-local registry alias. + +`s2i_ci_content.images` contains every exact successful reference, including +base and both unmapped Cyborg images. The partial +`content_provider_os_custom_container_images` map contains only effective +keys joined to exact successful references. The legacy global OS registry URL +remains the neutral `null` sentinel, while its namespace/tag and gating-repo +fields remain empty or false because this selective provider does not publish a +complete OpenStack image namespace. `cifmw_build_images_output` +remains an empty mapping and is not repurposed for service images. + +Intended references are written before build mutation. Post-run cleanup +removes only those exact Podman pullback and Buildah build tags, verifies exact +absence, and removes a buildset registry only when its ownership marker is +valid. +Per-image parallel logs and registry/result manifests are retained under +`zuul-output/logs/container-build/`. + +The provider pauses while dependent jobs run. Private onboarding may attach a +trivial child that prints the returned registry paths and maps. That debug job +does not pull images, patch an OpenStackVersion resource, deploy OpenStack, or +invoke a downstream repository's playbooks. Downstream consumption is separate +work. + ## Build architecture ### Base image (`openstack-base`) diff --git a/molecule/provider-contract/cleanup.yml b/molecule/provider-contract/cleanup.yml new file mode 100644 index 00000000..9b147fe0 --- /dev/null +++ b/molecule/provider-contract/cleanup.yml @@ -0,0 +1,13 @@ +--- +- name: Remove provider contract fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_root: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract + + tasks: + - name: Remove provider contract fixture state + ansible.builtin.file: + path: "{{ s2i_test_root }}" + state: absent diff --git a/molecule/provider-contract/converge.yml b/molecule/provider-contract/converge.yml new file mode 100644 index 00000000..bb0c1ba1 --- /dev/null +++ b/molecule/provider-contract/converge.yml @@ -0,0 +1,126 @@ +--- +- name: Validate the fake normalized registry + ansible.builtin.import_playbook: >- + ../../playbooks/container-ci/shared/validate-registry.yaml + vars: + s2i_ci_output_dir: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract/zuul-output + +- name: Prepare exact cleanup fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + s2i_test_images: + - registry.test:5000/openstack/openstack-base:test + - registry.test:5000/openstack/openstack-watcher-base:test + s2i_ci_container_repo: "{{ s2i_test_repo_root }}" + s2i_ci_selected_images: + - base + - cyborg/cyborg + - watcher/watcher-base + s2i_ci_image_mappings: + cyborg/cyborg: + - futureCyborgImage + + tasks: + - name: Initialize metadata expansion fixtures + ansible.builtin.set_fact: + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Load tracked image mappings + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/load-image-mappings.yaml + + - name: Expand tracked and inventory image metadata + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-deployment-keys.yaml + + - name: Require tracked and replacement metadata expansion + ansible.builtin.assert: + that: + - s2i_ci_effective_image_mappings['base'] == [] + - s2i_ci_effective_image_mappings['cyborg/cyborg'] == + ['futureCyborgImage'] + - s2i_ci_effective_image_mappings['watcher/watcher-base'] | length == 3 + - s2i_ci_mapping_sources['cyborg/cyborg'] == 'inventory' + - s2i_ci_mapping_sources['watcher/watcher-base'] == 'tracked' + + - name: Initialize exact reference expansion fixtures + ansible.builtin.set_fact: + s2i_ci_target_references: + base: registry.test:5000/openstack/openstack-base:test + cyborg/cyborg: registry.test:5000/openstack/openstack-cyborg:test + watcher/watcher-base: >- + registry.test:5000/openstack/openstack-watcher-base:test + s2i_ci_custom_container_images: {} + + - name: Expand exact metadata references + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + + - name: Require consolidated Watcher reference expansion + ansible.builtin.assert: + that: + - s2i_ci_custom_container_images.watcherAPIImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.watcherApplierImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.watcherDecisionEngineImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.futureCyborgImage == + s2i_ci_target_references['cyborg/cyborg'] + + - name: Read fake container storage after registry validation + ansible.builtin.slurp: + src: "{{ s2i_test_root }}/state.json" + register: s2i_test_state_data + + - name: Decode fake container storage + ansible.builtin.set_fact: + s2i_test_state: >- + {{ s2i_test_state_data.content | b64decode | from_json }} + + - name: Add exact workflow tags to both storage clients + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/state.json" + mode: "0600" + content: | + {{ s2i_test_state | + combine({ + 'podman': s2i_test_state.podman + s2i_test_images, + 'buildah': s2i_test_images + }) | to_nice_json }} + + - name: Write intended exact cleanup metadata + ansible.builtin.copy: + dest: >- + {{ s2i_test_root }}/zuul-output/logs/container-build/intended-images.json + mode: "0644" + content: | + {{ { + 'registry': 'registry.test:5000', + 'namespace': 'openstack', + 'images': s2i_test_images, + 'selected_images': ['base', 'watcher/watcher-base'], + 'target': 'watcher/watcher-base' + } | to_nice_json }} + +- name: Exercise exact shared cleanup + ansible.builtin.import_playbook: >- + ../../playbooks/container-ci/shared/cleanup-images.yaml + vars: + s2i_ci_output_dir: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract/zuul-output diff --git a/molecule/provider-contract/duplicate-metadata.yml b/molecule/provider-contract/duplicate-metadata.yml new file mode 100644 index 00000000..faedb612 --- /dev/null +++ b/molecule/provider-contract/duplicate-metadata.yml @@ -0,0 +1,16 @@ +--- +- name: Reject duplicate deployment keys + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_effective_image_mappings: + base: + - duplicateImage + watcher/watcher-base: + - duplicateImage + + tasks: + - name: Validate duplicate fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-deployment-keys.yaml diff --git a/molecule/provider-contract/files/fake-container-client.py b/molecule/provider-contract/files/fake-container-client.py new file mode 100755 index 00000000..ade2a2cb --- /dev/null +++ b/molecule/provider-contract/files/fake-container-client.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib +import sys + + +state_path = pathlib.Path(os.environ["S2I_FAKE_STATE"]) +client = pathlib.Path(sys.argv[0]).name +arguments = sys.argv[1:] +state = json.loads(state_path.read_text(encoding="utf-8")) +state["commands"].append([client, *arguments]) +images = state[client] +return_code = 0 + +if client == "podman": + if arguments[:1] == ["pull"]: + images.append(arguments[-1]) + elif arguments[:1] == ["tag"]: + images.append(arguments[-1]) + elif arguments[:1] == ["push"]: + pass + elif arguments[:1] == ["untag"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + elif arguments[:2] in (["image", "inspect"], ["image", "exists"]): + return_code = 0 if arguments[-1] in images else 1 + elif arguments[:2] == ["image", "rm"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + else: + return_code = 1 + elif arguments[:1] == ["images"]: + print("\n".join(images)) + else: + return_code = 2 +elif client == "buildah": + if arguments[:1] == ["inspect"]: + return_code = 0 if arguments[-1] in images else 125 + elif arguments[:1] == ["rmi"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + else: + return_code = 125 + elif arguments[:1] == ["images"]: + print("\n".join(images)) + else: + return_code = 2 +else: + return_code = 2 + +state[client] = list(dict.fromkeys(images)) +state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") +sys.exit(return_code) diff --git a/molecule/provider-contract/malformed-mapping.yml b/molecule/provider-contract/malformed-mapping.yml new file mode 100644 index 00000000..60c20335 --- /dev/null +++ b/molecule/provider-contract/malformed-mapping.yml @@ -0,0 +1,18 @@ +--- +- name: Reject malformed mapping value + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_container_repo: "{{ s2i_test_repo_root }}" + s2i_ci_image_mappings: + base: not-a-list + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + tasks: + - name: Validate malformed mapping fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/image-metadata-item.yaml + vars: + s2i_ci_image: base diff --git a/molecule/provider-contract/molecule.yml b/molecule/provider-contract/molecule.yml new file mode 100644 index 00000000..d3eed451 --- /dev/null +++ b/molecule/provider-contract/molecule.yml @@ -0,0 +1,34 @@ +--- +driver: + name: default + options: + managed: false + ansible_connection_options: + ansible_connection: local + +platforms: + - name: localhost + groups: + - builder + +provisioner: + name: ansible + env: + ANSIBLE_LOCAL_TEMP: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/ansible/local-tmp" + PATH: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/molecule/provider-contract/bin:${PATH}" + S2I_FAKE_STATE: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/molecule/provider-contract/state.json" + inventory: + host_vars: + localhost: + ansible_connection: local + +verifier: + name: ansible + +scenario: + test_sequence: + - syntax + - prepare + - converge + - verify + - cleanup diff --git a/molecule/provider-contract/prepare.yml b/molecule/provider-contract/prepare.yml new file mode 100644 index 00000000..3874ab96 --- /dev/null +++ b/molecule/provider-contract/prepare.yml @@ -0,0 +1,57 @@ +--- +- name: Prepare provider contract fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + + tasks: + - name: Reset provider contract fixture state + ansible.builtin.file: + path: "{{ s2i_test_root }}" + state: absent + + - name: Create provider contract fixture directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ s2i_test_root }}/bin" + - "{{ s2i_test_root }}/zuul-output/logs/container-build" + + - name: Install the fake container client + ansible.builtin.copy: + src: fake-container-client.py + dest: "{{ s2i_test_root }}/bin/fake-container-client.py" + mode: "0755" + + - name: Link fake Podman and Buildah clients + ansible.builtin.file: + src: "{{ s2i_test_root }}/bin/fake-container-client.py" + dest: "{{ s2i_test_root }}/bin/{{ item }}" + state: link + loop: + - podman + - buildah + + - name: Initialize fake container storage + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/state.json" + mode: "0600" + content: | + {{ {'podman': [], 'buildah': [], 'commands': []} | to_nice_json }} + + - name: Write normalized registry connection + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/zuul-output/.s2i-ci-registry-connection.json" + mode: "0600" + content: | + {{ { + 'endpoint': 'registry.test:5000', + 'public_host': '192.0.2.10', + 'public_port': 5000, + 'auth_file': '', + 'cert_dir': '' + } | to_nice_json }} diff --git a/molecule/provider-contract/unknown-mapping.yml b/molecule/provider-contract/unknown-mapping.yml new file mode 100644 index 00000000..73dec15b --- /dev/null +++ b/molecule/provider-contract/unknown-mapping.yml @@ -0,0 +1,17 @@ +--- +- name: Reject unknown mapping target + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_selected_images: + - base + - watcher/watcher-base + s2i_ci_image_mappings: + cyborg/cyborg: + - unknownImage + + tasks: + - name: Validate unknown mapping fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-mapping-overrides.yaml diff --git a/molecule/provider-contract/verify.yml b/molecule/provider-contract/verify.yml new file mode 100644 index 00000000..0243d40a --- /dev/null +++ b/molecule/provider-contract/verify.yml @@ -0,0 +1,89 @@ +--- +- name: Verify provider registry and cleanup contracts + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + s2i_test_validation_image: >- + registry.test:5000/s2i-validation/ubi10-minimal:latest + s2i_test_images: + - registry.test:5000/openstack/openstack-base:test + - registry.test:5000/openstack/openstack-watcher-base:test + + tasks: + - name: Exercise invalid metadata contracts + ansible.builtin.command: + argv: + - ansible-playbook + - -i + - builder, + - -c + - local + - "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/{{ item }}" + loop: + - duplicate-metadata.yml + - unknown-mapping.yml + - malformed-mapping.yml + register: s2i_test_invalid_metadata + changed_when: false + failed_when: false + + - name: Require invalid metadata to fail deterministically + ansible.builtin.assert: + that: + - s2i_test_invalid_metadata.results | + map(attribute='rc') | reject('equalto', 0) | list | length == 3 + - >- + 'A deployment key may be assigned to only one image' in + (s2i_test_invalid_metadata.results[0].stdout + + s2i_test_invalid_metadata.results[0].stderr) + - >- + 'Image mapping overrides must name selected image targets' in + (s2i_test_invalid_metadata.results[1].stdout + + s2i_test_invalid_metadata.results[1].stderr) + - >- + 'Invalid effective image mapping for base' in + (s2i_test_invalid_metadata.results[2].stdout + + s2i_test_invalid_metadata.results[2].stderr) + + - name: Read final fake container storage + ansible.builtin.slurp: + src: "{{ s2i_test_root }}/state.json" + register: s2i_test_state_data + + - name: Decode final fake container storage + ansible.builtin.set_fact: + s2i_test_state: >- + {{ s2i_test_state_data.content | b64decode | from_json }} + + - name: Validate exact cleanup and retained source image + ansible.builtin.assert: + that: + - s2i_test_images | intersect(s2i_test_state.podman) | length == 0 + - s2i_test_images | intersect(s2i_test_state.buildah) | length == 0 + - s2i_test_validation_image not in s2i_test_state.podman + - >- + 'registry.access.redhat.com/ubi10/ubi-minimal:latest' in + s2i_test_state.podman + - >- + ['podman', 'push', '--remove-signatures', + 'registry.test:5000/s2i-validation/ubi10-minimal:latest'] in + s2i_test_state.commands + - >- + ['podman', 'image', 'rm', '--force', s2i_test_images[0]] in + s2i_test_state.commands + - >- + ['buildah', 'rmi', s2i_test_images[0]] in s2i_test_state.commands + + - name: Check public registry state artifact + ansible.builtin.stat: + path: >- + {{ s2i_test_root }}/zuul-output/logs/container-build/registry-state.json + register: s2i_test_registry_state + + - name: Require registry state artifact + ansible.builtin.assert: + that: + - s2i_test_registry_state.stat.exists + - s2i_test_registry_state.stat.mode == '0644' diff --git a/playbooks/container-ci/shared/cleanup-images.yaml b/playbooks/container-ci/shared/cleanup-images.yaml new file mode 100644 index 00000000..2100cba9 --- /dev/null +++ b/playbooks/container-ci/shared/cleanup-images.yaml @@ -0,0 +1,208 @@ +--- +- name: Remove exact images recorded by the workflow + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_cleanup_images: true + + tasks: + - name: Check intended image cleanup metadata + when: s2i_ci_cleanup_images | bool + ansible.builtin.stat: + path: "{{ s2i_ci_log_dir }}/intended-images.json" + register: s2i_ci_intended_images_metadata + + - name: Load intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.slurp: + src: "{{ s2i_ci_log_dir }}/intended-images.json" + register: s2i_ci_intended_images_data + + - name: Decode intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.set_fact: + s2i_ci_cleanup_metadata: >- + {{ s2i_ci_intended_images_data.content | b64decode | from_json }} + + - name: Validate intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.assert: + that: + - s2i_ci_cleanup_metadata is mapping + - s2i_ci_cleanup_metadata.registry is string + - s2i_ci_cleanup_metadata.registry is match('^[A-Za-z0-9.:-]+$') + - s2i_ci_cleanup_metadata.namespace is string + - s2i_ci_cleanup_metadata.namespace is match('^[A-Za-z0-9_.-]+$') + - s2i_ci_cleanup_metadata.images is sequence + - s2i_ci_cleanup_metadata.images is not string + - s2i_ci_cleanup_metadata.images | unique | list | length == + s2i_ci_cleanup_metadata.images | length + - s2i_ci_cleanup_metadata.images | + select('match', '^' + s2i_ci_cleanup_metadata.registry | + regex_escape + '/' + s2i_ci_cleanup_metadata.namespace | + regex_escape + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_cleanup_metadata.images | length + fail_msg: Refusing unsafe or malformed intended image cleanup metadata + + - name: Record exact intended images for cleanup + ansible.builtin.set_fact: + s2i_ci_cleanup_images_exact: >- + {{ s2i_ci_cleanup_metadata.images | reverse | list }} + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + + - name: Default exact intended image list + ansible.builtin.set_fact: + s2i_ci_cleanup_images_exact: [] + when: + - not s2i_ci_cleanup_images | bool or + not s2i_ci_intended_images_metadata.stat.exists + + - name: Check Podman pullback tags + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + register: s2i_ci_podman_image_checks + changed_when: false + failed_when: s2i_ci_podman_image_checks.rc not in [0, 1] + + - name: Remove exact Podman pullback tags + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "{{ item.item }}" + loop: "{{ s2i_ci_podman_image_checks.results }}" + when: item.rc == 0 + changed_when: true + + - name: Check Buildah-created image tags + ansible.builtin.command: + argv: + - buildah + - inspect + - --type + - image + - "{{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + register: s2i_ci_buildah_image_checks + changed_when: false + failed_when: s2i_ci_buildah_image_checks.rc not in [0, 125] + + - name: Remove exact Buildah-created image tags + ansible.builtin.command: + argv: + - buildah + - rmi + - "{{ item.item }}" + loop: "{{ s2i_ci_buildah_image_checks.results }}" + when: item.rc == 0 + changed_when: true + + - name: List remaining Podman image references + ansible.builtin.command: + argv: + - podman + - images + - --format + - "{{ '{{.Repository}}:{{.Tag}}' }}" + register: s2i_ci_remaining_podman_images + changed_when: false + + - name: List remaining Buildah image references + ansible.builtin.command: + argv: + - buildah + - images + - --format + - "{{ '{{.Name}}:{{.Tag}}' }}" + register: s2i_ci_remaining_buildah_images + changed_when: false + + - name: Require exact workflow references to be absent + ansible.builtin.assert: + that: + - item not in s2i_ci_remaining_podman_images.stdout_lines + - item not in s2i_ci_remaining_buildah_images.stdout_lines + fail_msg: "Image cleanup left exact workflow reference {{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + + - name: Check registry validation metadata + ansible.builtin.stat: + path: "{{ s2i_ci_log_dir }}/registry-state.json" + register: s2i_ci_registry_state_metadata + + - name: Load registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.slurp: + src: "{{ s2i_ci_log_dir }}/registry-state.json" + register: s2i_ci_registry_state_data + + - name: Decode registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.set_fact: + s2i_ci_registry_state: >- + {{ s2i_ci_registry_state_data.content | b64decode | from_json }} + + - name: Validate registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.assert: + that: + - s2i_ci_registry_state is mapping + - s2i_ci_registry_state.endpoint is string + - s2i_ci_registry_state.endpoint is match('^[A-Za-z0-9.:-]+$') + - s2i_ci_registry_state.validation_image is string + - s2i_ci_registry_state.validation_image.startswith( + s2i_ci_registry_state.endpoint + '/') + + - name: Check the Podman registry validation tag + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ s2i_ci_registry_state.validation_image }}" + register: s2i_ci_validation_image_check + changed_when: false + failed_when: s2i_ci_validation_image_check.rc not in [0, 1] + + - name: Remove the exact Podman registry validation tag + when: + - s2i_ci_registry_state_metadata.stat.exists + - s2i_ci_validation_image_check.rc == 0 + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "{{ s2i_ci_registry_state.validation_image }}" + changed_when: true + + - name: Require registry validation tag to be absent + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ s2i_ci_registry_state.validation_image }}" + register: s2i_ci_validation_image_absent + changed_when: false + failed_when: s2i_ci_validation_image_absent.rc != 1 diff --git a/playbooks/container-ci/shared/deployment-map-item.yaml b/playbooks/container-ci/shared/deployment-map-item.yaml new file mode 100644 index 00000000..c38a4e07 --- /dev/null +++ b/playbooks/container-ci/shared/deployment-map-item.yaml @@ -0,0 +1,7 @@ +--- +- name: Expand image reference for each deployment key + ansible.builtin.set_fact: + s2i_ci_custom_container_images: >- + {{ s2i_ci_custom_container_images | + combine({item: s2i_ci_target_references[s2i_ci_image]}) }} + loop: "{{ s2i_ci_effective_image_mappings[s2i_ci_image] }}" diff --git a/playbooks/container-ci/shared/image-metadata-item.yaml b/playbooks/container-ci/shared/image-metadata-item.yaml new file mode 100644 index 00000000..c9119f47 --- /dev/null +++ b/playbooks/container-ci/shared/image-metadata-item.yaml @@ -0,0 +1,42 @@ +--- +- name: Check the selected image Containerfile + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/containers/{{ s2i_ci_image }}/Containerfile" + register: s2i_ci_image_containerfile + +- name: Require the selected image Containerfile + ansible.builtin.assert: + that: + - s2i_ci_image_containerfile.stat.isreg | default(false) + fail_msg: "Selected image {{ s2i_ci_image }} requires a Containerfile" + +- name: Select effective image deployment keys + ansible.builtin.set_fact: + s2i_ci_effective_image_keys: >- + {{ s2i_ci_image_mappings[s2i_ci_image] + if s2i_ci_image in s2i_ci_image_mappings + else s2i_ci_tracked_image_mappings.get(s2i_ci_image, []) }} + s2i_ci_effective_mapping_source: >- + {{ 'inventory' if s2i_ci_image in s2i_ci_image_mappings else 'tracked' }} + +- name: Validate effective image deployment keys + ansible.builtin.assert: + that: + - s2i_ci_effective_image_keys is sequence + - s2i_ci_effective_image_keys is not string + - s2i_ci_effective_image_keys | select('string') | list | length == + s2i_ci_effective_image_keys | length + - s2i_ci_effective_image_keys | + reject('match', '^[A-Za-z][A-Za-z0-9]*$') | list | length == 0 + - s2i_ci_effective_image_keys | unique | list | length == + s2i_ci_effective_image_keys | length + fail_msg: "Invalid effective image mapping for {{ s2i_ci_image }}" + +- name: Record effective image metadata + ansible.builtin.set_fact: + s2i_ci_effective_image_mappings: >- + {{ s2i_ci_effective_image_mappings | + combine({s2i_ci_image: s2i_ci_effective_image_keys | list}) }} + s2i_ci_mapping_sources: >- + {{ s2i_ci_mapping_sources | + combine({s2i_ci_image: s2i_ci_effective_mapping_source}) }} diff --git a/playbooks/container-ci/shared/load-image-mappings.yaml b/playbooks/container-ci/shared/load-image-mappings.yaml new file mode 100644 index 00000000..4b44834f --- /dev/null +++ b/playbooks/container-ci/shared/load-image-mappings.yaml @@ -0,0 +1,67 @@ +--- +- name: Check the tracked image mapping manifest + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/containers/image-mappings.yaml" + register: s2i_ci_image_mapping_manifest + +- name: Require the tracked image mapping manifest + ansible.builtin.assert: + that: + - s2i_ci_image_mapping_manifest.stat.isreg | default(false) + fail_msg: containers/image-mappings.yaml is required + +- name: Read the tracked image mapping manifest + ansible.builtin.slurp: + src: "{{ s2i_ci_container_repo }}/containers/image-mappings.yaml" + register: s2i_ci_image_mapping_data + +- name: Decode the tracked image mapping manifest + ansible.builtin.set_fact: + s2i_ci_image_mapping_document: >- + {{ s2i_ci_image_mapping_data.content | b64decode | from_yaml }} + +- name: Validate the tracked image mapping schema + ansible.builtin.assert: + that: + - s2i_ci_image_mapping_document is mapping + - s2i_ci_image_mapping_document.openstack_version is mapping + - s2i_ci_image_mapping_document.openstack_version.custom_container_images is mapping + fail_msg: Malformed containers/image-mappings.yaml + +- name: Record tracked image mappings + ansible.builtin.set_fact: + s2i_ci_tracked_image_mappings: >- + {{ s2i_ci_image_mapping_document.openstack_version.custom_container_images }} + s2i_ci_tracked_deployment_keys: [] + +- name: Resolve all available image targets + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - resolve + - all + args: + chdir: "{{ s2i_ci_container_repo }}" + register: s2i_ci_available_images + changed_when: false + +- name: Reject mappings for unknown image targets + ansible.builtin.assert: + that: + - s2i_ci_tracked_image_mappings.keys() | + difference(s2i_ci_available_images.stdout_lines) | length == 0 + fail_msg: containers/image-mappings.yaml names an unknown image target + +- name: Validate every tracked image mapping + ansible.builtin.include_tasks: validate-tracked-mapping-item.yaml + loop: "{{ s2i_ci_tracked_image_mappings.keys() | list }}" + loop_control: + loop_var: s2i_ci_tracked_mapping_target + label: "{{ s2i_ci_tracked_mapping_target }}" + +- name: Reject globally duplicated tracked deployment keys + ansible.builtin.assert: + that: + - s2i_ci_tracked_deployment_keys | unique | list | length == + s2i_ci_tracked_deployment_keys | length + fail_msg: A tracked deployment key may be assigned to only one image diff --git a/playbooks/container-ci/shared/prepare-host.yaml b/playbooks/container-ci/shared/prepare-host.yaml new file mode 100644 index 00000000..64104d47 --- /dev/null +++ b/playbooks/container-ci/shared/prepare-host.yaml @@ -0,0 +1,46 @@ +--- +- name: Prepare the shared container image build tools + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + + tasks: + - name: Ensure the container build log directory exists + ansible.builtin.file: + path: "{{ s2i_ci_log_dir }}" + state: directory + mode: "0755" + + - name: Update the apt package cache + when: + - s2i_ci_install_host_packages | default(true) | bool + - ansible_facts.os_family == "Debian" + become: true + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install container build host packages + when: s2i_ci_install_host_packages | default(true) | bool + become: true + ansible.builtin.package: + name: + - buildah + - ca-certificates + - git + - openssl + state: present + + - name: Ensure Podman is installed and usable + when: s2i_ci_install_host_packages | default(true) | bool + ansible.builtin.include_role: + name: ensure-podman + vars: + ensure_podman_rootless: false + ensure_podman_validate: true + + - name: Validate Buildah + when: s2i_ci_install_host_packages | default(true) | bool + ansible.builtin.command: buildah version + changed_when: false diff --git a/playbooks/container-ci/shared/published-image-item.yaml b/playbooks/container-ci/shared/published-image-item.yaml new file mode 100644 index 00000000..1b30ee8a --- /dev/null +++ b/playbooks/container-ci/shared/published-image-item.yaml @@ -0,0 +1,10 @@ +--- +- name: Record exact reference for selected image + ansible.builtin.set_fact: + s2i_ci_target_references: >- + {{ s2i_ci_target_references | + combine({s2i_ci_image: s2i_ci_published_images[s2i_ci_image_index]}) }} + s2i_ci_local_target_references: >- + {{ s2i_ci_local_target_references | + combine({s2i_ci_image: + s2i_ci_local_published_images[s2i_ci_image_index]}) }} diff --git a/playbooks/container-ci/shared/run.yaml b/playbooks/container-ci/shared/run.yaml new file mode 100644 index 00000000..9e8a1081 --- /dev/null +++ b/playbooks/container-ci/shared/run.yaml @@ -0,0 +1,361 @@ +--- +- name: Build and publish selected container images + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_workspace_root: "{{ zuul_user_dir | default(ansible_user_dir) }}" + + tasks: + - name: Default optional selective provider inputs + ansible.builtin.set_fact: + s2i_ci_namespace: "{{ s2i_ci_namespace | default('openstack') }}" + s2i_ci_tag: >- + {{ s2i_ci_tag | default( + (zuul.change | default('build') | string) ~ '-' ~ + (zuul.patchset | default('latest') | string) + ) }} + s2i_ci_stream: "{{ s2i_ci_stream | default('master') }}" + s2i_ci_parallel: >- + {{ s2i_ci_parallel | + default(ansible_facts.processor_vcpus | default(1)) }} + s2i_ci_images: "{{ s2i_ci_images | default('all') }}" + s2i_ci_image_mappings: "{{ s2i_ci_image_mappings | default({}) }}" + + - name: Validate the selected container project + ansible.builtin.assert: + that: + - s2i_ci_container_project in zuul.projects + fail_msg: The selected container project is not in the Zuul workspace + + - name: Resolve the inventory-described container repository + ansible.builtin.set_fact: + s2i_ci_container_repo: >- + {{ s2i_ci_workspace_root }}/{{ + zuul.projects[s2i_ci_container_project].src_dir }} + + - name: Inspect the container build entry point on the builder + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/build.sh" + follow: true + register: s2i_ci_build_entry_point + + - name: Canonicalize the builder workspace root + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_workspace_root }}"] + register: s2i_ci_workspace_root_realpath + changed_when: false + failed_when: false + + - name: Canonicalize the container repository on the builder + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_container_repo }}"] + register: s2i_ci_container_repo_realpath + changed_when: false + failed_when: false + + - name: Validate selective provider inputs + ansible.builtin.assert: + that: + - s2i_ci_build_entry_point.stat.isreg | default(false) + - s2i_ci_workspace_root_realpath.rc == 0 + - s2i_ci_container_repo_realpath.rc == 0 + - s2i_ci_container_repo_realpath.stdout.startswith( + s2i_ci_workspace_root_realpath.stdout + '/') + - >- + s2i_ci_images == 'all' or + (s2i_ci_images is sequence and s2i_ci_images is not string and + s2i_ci_images | length > 0 and + s2i_ci_images | select('string') | list | length == + s2i_ci_images | length and + s2i_ci_images | + reject('match', '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$') | + list | length == 0 and + s2i_ci_images | unique | list | length == s2i_ci_images | length) + - s2i_ci_image_mappings is mapping + - s2i_ci_tag is string + - "',' not in s2i_ci_tag" + fail_msg: Selective provider inputs are invalid + + - name: Record the requested target expression + ansible.builtin.set_fact: + s2i_ci_target_expression: >- + {{ 'all' if s2i_ci_images == 'all' + else ((['base'] + s2i_ci_images) | unique | list | join(',')) }} + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Resolve selected image targets + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - resolve + - "{{ s2i_ci_target_expression }}" + args: + chdir: "{{ s2i_ci_container_repo }}" + register: s2i_ci_resolved_images + changed_when: false + + - name: Record normalized selected targets + ansible.builtin.set_fact: + s2i_ci_selected_images: "{{ s2i_ci_resolved_images.stdout_lines }}" + + - name: Validate optional mapping overrides + ansible.builtin.include_tasks: validate-mapping-overrides.yaml + + - name: Load tracked image mappings + ansible.builtin.include_tasks: load-image-mappings.yaml + + - name: Load and validate selected image metadata + ansible.builtin.include_tasks: image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: validate-deployment-keys.yaml + + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + no_log: true + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + no_log: true + + - name: Validate normalized registry connection + ansible.builtin.assert: + that: + - s2i_ci_registry is mapping + - s2i_ci_registry.endpoint is string + - s2i_ci_registry.endpoint | length > 0 + - s2i_ci_registry.public_host is string + - s2i_ci_registry.public_host | length > 0 + - s2i_ci_registry.public_port | int > 0 + - s2i_ci_registry.auth_file | default('') is string + - s2i_ci_registry.cert_dir | default('') is string + + - name: Resolve intended exact image references before mutation + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_intended_refs + changed_when: false + + - name: Record intended exact image references + ansible.builtin.set_fact: + s2i_ci_intended_images: >- + {{ s2i_ci_intended_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Validate intended exact image references + ansible.builtin.assert: + that: + - s2i_ci_intended_images | length == s2i_ci_selected_images | length + - s2i_ci_intended_images | unique | list | length == + s2i_ci_intended_images | length + - s2i_ci_intended_images | + select('match', '^' + s2i_ci_registry.endpoint | regex_escape + + '/' + s2i_ci_namespace + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_intended_images | length + fail_msg: Intended image references are incomplete or unsafe + + - name: Persist intended references for partial-failure cleanup + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/intended-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_registry.endpoint, + 'namespace': s2i_ci_namespace, + 'images': s2i_ci_intended_images, + 'selected_images': s2i_ci_selected_images, + 'target': s2i_ci_target_expression + } | to_nice_json }} + + - name: Build selected images from committed source pins + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - build-parallel + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + PARALLEL: "{{ s2i_ci_parallel | string }}" + BUILD_LOGS_DIR: "{{ s2i_ci_log_dir }}" + register: s2i_ci_build + changed_when: s2i_ci_build.rc == 0 + + - name: Save build command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/build.log" + mode: "0644" + content: | + {{ s2i_ci_build.stdout }} + {{ s2i_ci_build.stderr }} + + - name: Push selected images to the buildset registry + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - push + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + REGISTRY_AUTH_FILE: "{{ s2i_ci_registry.auth_file | default('') }}" + REGISTRY_CERT_DIR: "{{ s2i_ci_registry.cert_dir | default('') }}" + register: s2i_ci_push + changed_when: s2i_ci_push.rc == 0 + + - name: Save push command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/push.log" + mode: "0644" + content: | + {{ s2i_ci_push.stdout }} + {{ s2i_ci_push.stderr }} + + - name: Resolve successfully published image references + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_refs + changed_when: false + + - name: Record exact builder-local published image references + ansible.builtin.set_fact: + s2i_ci_local_published_images: >- + {{ s2i_ci_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Require intended and published references to match + ansible.builtin.assert: + that: + - s2i_ci_local_published_images == s2i_ci_intended_images + fail_msg: Published references differ from the pre-recorded intent + + - name: Record the public buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_public_registry_endpoint: >- + {{ s2i_ci_registry.public_host }}:{{ s2i_ci_registry.public_port }} + + - name: Convert published references to the public buildset endpoint + ansible.builtin.set_fact: + s2i_ci_published_images: >- + {{ s2i_ci_local_published_images | + map('regex_replace', + '^' + (s2i_ci_registry.endpoint | regex_escape), + s2i_ci_public_registry_endpoint) | list }} + + - name: Validate public exact published references + ansible.builtin.assert: + that: + - s2i_ci_published_images | length == + s2i_ci_local_published_images | length + - s2i_ci_published_images | + select('match', '^' + + (s2i_ci_public_registry_endpoint | regex_escape) + '/') | + list | length == s2i_ci_published_images | length + - s2i_ci_published_images | + select('search', s2i_ci_registry.endpoint | regex_escape) | + list | length == 0 + fail_msg: Public references must not expose the builder-local alias + + - name: Pull published images back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry.auth_file] + if s2i_ci_registry.auth_file | default('') | length > 0 else []) + + (['--cert-dir', s2i_ci_registry.cert_dir] + if s2i_ci_registry.cert_dir | default('') | length > 0 else []) + + [item] }} + loop: "{{ s2i_ci_local_published_images }}" + changed_when: true + + - name: Initialize exact reference expansion + ansible.builtin.set_fact: + s2i_ci_target_references: {} + s2i_ci_local_target_references: {} + s2i_ci_custom_container_images: {} + + - name: Map selected targets to exact references + ansible.builtin.include_tasks: published-image-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + index_var: s2i_ci_image_index + label: "{{ s2i_ci_image }}" + + - name: Expand exact references into deployment keys + ansible.builtin.include_tasks: deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate consolidated Watcher process entry points + ansible.builtin.command: + argv: + - podman + - run + - --rm + - --user + - watcher + - --entrypoint + - /bin/sh + - "{{ s2i_ci_local_target_references['watcher/watcher-base'] }}" + - -ec + - >- + for command in watcher-api watcher-applier watcher-decision-engine; + do command -v "$command"; "$command" --help >/dev/null; done + when: "'watcher/watcher-base' in s2i_ci_selected_images" + changed_when: false + + - name: Save published image metadata + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/published-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_public_registry_endpoint, + 'namespace': s2i_ci_namespace, + 'tag': s2i_ci_tag, + 'target': s2i_ci_target_expression, + 'stream': s2i_ci_stream, + 'selected_images': s2i_ci_selected_images, + 'images': s2i_ci_published_images, + 'image_mappings': s2i_ci_effective_image_mappings, + 'mapping_sources': s2i_ci_mapping_sources, + 'custom_container_images': + s2i_ci_custom_container_images + } | to_nice_json }} diff --git a/playbooks/container-ci/shared/validate-deployment-keys.yaml b/playbooks/container-ci/shared/validate-deployment-keys.yaml new file mode 100644 index 00000000..f9750707 --- /dev/null +++ b/playbooks/container-ci/shared/validate-deployment-keys.yaml @@ -0,0 +1,12 @@ +--- +- name: Record all effective deployment keys + ansible.builtin.set_fact: + s2i_ci_all_deployment_keys: >- + {{ s2i_ci_effective_image_mappings.values() | flatten | list }} + +- name: Reject duplicate deployment keys + ansible.builtin.assert: + that: + - s2i_ci_all_deployment_keys | unique | list | length == + s2i_ci_all_deployment_keys | length + fail_msg: A deployment key may be assigned to only one image diff --git a/playbooks/container-ci/shared/validate-mapping-overrides.yaml b/playbooks/container-ci/shared/validate-mapping-overrides.yaml new file mode 100644 index 00000000..081c990a --- /dev/null +++ b/playbooks/container-ci/shared/validate-mapping-overrides.yaml @@ -0,0 +1,8 @@ +--- +- name: Reject mapping overrides for unselected images + ansible.builtin.assert: + that: + - s2i_ci_image_mappings is mapping + - s2i_ci_image_mappings.keys() | + difference(s2i_ci_selected_images) | length == 0 + fail_msg: Image mapping overrides must name selected image targets diff --git a/playbooks/container-ci/shared/validate-registry.yaml b/playbooks/container-ci/shared/validate-registry.yaml new file mode 100644 index 00000000..cad25f04 --- /dev/null +++ b/playbooks/container-ci/shared/validate-registry.yaml @@ -0,0 +1,109 @@ +--- +- name: Validate the prepared image registry connection + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_validation_source_image: registry.access.redhat.com/ubi10/ubi-minimal:latest + s2i_ci_validation_repository: s2i-validation/ubi10-minimal:latest + + tasks: + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry_connection: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + + - name: Require normalized registry connection data + ansible.builtin.assert: + that: + - s2i_ci_registry_connection is mapping + - s2i_ci_registry_connection.endpoint is string + - s2i_ci_registry_connection.endpoint | length > 0 + - s2i_ci_registry_connection.public_host is string + - s2i_ci_registry_connection.public_host | length > 0 + - s2i_ci_registry_connection.public_port | int > 0 + - s2i_ci_registry_connection.auth_file | default('') is string + - s2i_ci_registry_connection.cert_dir | default('') is string + + - name: Set normalized registry command parameters + ansible.builtin.set_fact: + s2i_ci_registry_endpoint: "{{ s2i_ci_registry_connection.endpoint }}" + s2i_ci_registry_auth_file: >- + {{ s2i_ci_registry_connection.auth_file | default('') }} + s2i_ci_registry_cert_dir: >- + {{ s2i_ci_registry_connection.cert_dir | default('') }} + + - name: Pull the UBI 10 base image + ansible.builtin.command: + argv: + - podman + - pull + - "{{ s2i_ci_validation_source_image }}" + register: s2i_ci_ubi_pull + changed_when: "'Copying blob' in s2i_ci_ubi_pull.stdout" + + - name: Tag the UBI 10 image for registry validation + ansible.builtin.command: + argv: + - podman + - tag + - "{{ s2i_ci_validation_source_image }}" + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: true + + - name: Push the UBI 10 validation image + ansible.builtin.command: + argv: >- + {{ ['podman', 'push', '--remove-signatures'] + + (['--authfile', s2i_ci_registry_auth_file] + if s2i_ci_registry_auth_file | length > 0 else []) + + (['--cert-dir', s2i_ci_registry_cert_dir] + if s2i_ci_registry_cert_dir | length > 0 else []) + + [s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository] }} + changed_when: true + + - name: Remove the validation tag before pulling it back + ansible.builtin.command: + argv: + - podman + - untag + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: true + + - name: Pull the UBI 10 validation image back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry_auth_file] + if s2i_ci_registry_auth_file | length > 0 else []) + + (['--cert-dir', s2i_ci_registry_cert_dir] + if s2i_ci_registry_cert_dir | length > 0 else []) + + [s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository] }} + changed_when: false + + - name: Inspect the UBI 10 validation image from the registry + ansible.builtin.command: + argv: + - podman + - image + - inspect + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: false + + - name: Record non-secret registry state + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/registry-state.json" + mode: "0644" + content: | + {{ { + 'endpoint': s2i_ci_registry_endpoint, + 'validation_source': s2i_ci_validation_source_image, + 'validation_image': s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository + } | to_nice_json }} diff --git a/playbooks/container-ci/shared/validate-tracked-mapping-item.yaml b/playbooks/container-ci/shared/validate-tracked-mapping-item.yaml new file mode 100644 index 00000000..95f885bc --- /dev/null +++ b/playbooks/container-ci/shared/validate-tracked-mapping-item.yaml @@ -0,0 +1,24 @@ +--- +- name: Select tracked deployment keys for validation + ansible.builtin.set_fact: + s2i_ci_tracked_mapping_keys: >- + {{ s2i_ci_tracked_image_mappings[s2i_ci_tracked_mapping_target] }} + +- name: Validate tracked deployment keys + ansible.builtin.assert: + that: + - s2i_ci_tracked_mapping_keys is sequence + - s2i_ci_tracked_mapping_keys is not string + - s2i_ci_tracked_mapping_keys | select('string') | list | length == + s2i_ci_tracked_mapping_keys | length + - s2i_ci_tracked_mapping_keys | + reject('match', '^[A-Za-z][A-Za-z0-9]*$') | list | length == 0 + - s2i_ci_tracked_mapping_keys | unique | list | length == + s2i_ci_tracked_mapping_keys | length + fail_msg: >- + Invalid tracked image mapping for {{ s2i_ci_tracked_mapping_target }} + +- name: Accumulate tracked deployment keys + ansible.builtin.set_fact: + s2i_ci_tracked_deployment_keys: >- + {{ s2i_ci_tracked_deployment_keys + s2i_ci_tracked_mapping_keys }} diff --git a/playbooks/container-ci/zuul/content-provider-return.yaml b/playbooks/container-ci/zuul/content-provider-return.yaml new file mode 100644 index 00000000..f4e1abe3 --- /dev/null +++ b/playbooks/container-ci/zuul/content-provider-return.yaml @@ -0,0 +1,32 @@ +--- +- name: Return selective image metadata and pause the provider + zuul_return: + data: + zuul: + pause: true + s2i_ci_content: + registry: "{{ s2i_ci_public_registry_endpoint }}" + registry_host: "{{ s2i_ci_registry.public_host }}" + registry_port: "{{ s2i_ci_registry.public_port }}" + namespace: "{{ s2i_ci_namespace }}" + tag: "{{ s2i_ci_tag }}" + target: "{{ s2i_ci_target_expression }}" + stream: "{{ s2i_ci_stream }}" + selected_images: "{{ s2i_ci_selected_images }}" + images: "{{ s2i_ci_published_images }}" + image_mappings: "{{ s2i_ci_effective_image_mappings }}" + mapping_sources: "{{ s2i_ci_mapping_sources }}" + custom_container_images: "{{ s2i_ci_custom_container_images }}" + content_provider_os_custom_container_images: >- + {{ s2i_ci_custom_container_images }} + content_provider_os_registry_url: "null" + content_provider_os_registry_namespace: "" + content_provider_os_registry_tag: "" + content_provider_dlrn_md5_hash: "" + content_provider_gating_repo_available: false + content_provider_gating_repo_url: "" + content_provider_registry_available: true + content_provider_registry_ip: "{{ s2i_ci_registry.public_host }}" + content_provider_registry_ip_port: >- + {{ s2i_ci_public_registry_endpoint }} + cifmw_build_images_output: {} diff --git a/playbooks/container-ci/zuul/post.yaml b/playbooks/container-ci/zuul/post.yaml new file mode 100644 index 00000000..6b63478d --- /dev/null +++ b/playbooks/container-ci/zuul/post.yaml @@ -0,0 +1,129 @@ +--- +- name: Collect Zuul container logs + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + + tasks: + - name: Ensure the container build log directory exists + ansible.builtin.file: + path: "{{ s2i_ci_log_dir }}" + state: directory + mode: "0755" + failed_when: false + + - name: Collect buildset registry logs + block: + - name: Run the trusted container log collection role + ansible.builtin.include_role: + name: collect-container-logs + vars: + container_command: podman + rescue: + - name: Report non-fatal Zuul container log collection failure + ansible.builtin.debug: + msg: Container log collection failed; registry cleanup will continue + +- name: Remove exact workflow images + ansible.builtin.import_playbook: ../shared/cleanup-images.yaml + +- name: Remove the project-owned Zuul buildset registry + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_registry_port: 5000 + + tasks: + - name: Check buildset registry ownership + ansible.builtin.stat: + path: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + register: s2i_ci_registry_owner_marker + + - name: Default buildset registry ownership to unloaded + ansible.builtin.set_fact: + s2i_ci_registry_owner_loaded: false + s2i_ci_registry_owner_is_valid: false + + - name: Load buildset registry ownership + when: s2i_ci_registry_owner_marker.stat.exists | default(false) + block: + - name: Read the buildset registry ownership marker + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + register: s2i_ci_registry_owner_data + + - name: Decode the buildset registry ownership marker + ansible.builtin.set_fact: + s2i_ci_registry_owner: >- + {{ s2i_ci_registry_owner_data.content | b64decode | from_json }} + + - name: Record successful ownership loading + ansible.builtin.set_fact: + s2i_ci_registry_owner_loaded: true + rescue: + - name: Refuse unreadable registry ownership metadata + ansible.builtin.fail: + msg: Refusing to clean a registry whose ownership marker is invalid + + - name: Validate buildset registry ownership + when: s2i_ci_registry_owner_loaded | bool + ansible.builtin.assert: + that: + - s2i_ci_registry_owner.port | default(-1) | int == + s2i_ci_registry_port | int + - s2i_ci_registry_owner.container | default('') == + ('buildset_registry' if s2i_ci_registry_port | int == 5000 + else 'buildset_registry_' + s2i_ci_registry_port | string) + fail_msg: Refusing to clean a registry with an invalid ownership marker + + - name: Record valid buildset registry ownership + when: s2i_ci_registry_owner_loaded | bool + ansible.builtin.set_fact: + s2i_ci_registry_owner_is_valid: true + + - name: Remove the project-owned buildset registry container + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ s2i_ci_registry_owner.container }}" + register: s2i_ci_removed_buildset_registry + changed_when: s2i_ci_removed_buildset_registry.stdout | length > 0 + + - name: Stop the project-owned buildset registry tunnel + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.shell: | + pids=$(pgrep -f '^socat -d -d TCP6-LISTEN:{{ s2i_ci_registry_owner.port }},fork TCP:127.0.0.1:1{{ s2i_ci_registry_owner.port }}$' || true) + if [ -n "$pids" ]; then + printf '%s\n' "$pids" + kill $pids + fi + args: + executable: /bin/bash + register: s2i_ci_stopped_registry_tunnel + changed_when: s2i_ci_stopped_registry_tunnel.stdout | length > 0 + + - name: Remove project-owned buildset registry state + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.file: + path: "{{ s2i_ci_registry_root }}" + state: absent + + - name: Remove persisted buildset registry connection data + ansible.builtin.file: + path: "{{ s2i_ci_registry_connection_file }}" + state: absent diff --git a/playbooks/container-ci/zuul/pre.yaml b/playbooks/container-ci/zuul/pre.yaml new file mode 100644 index 00000000..49f52bae --- /dev/null +++ b/playbooks/container-ci/zuul/pre.yaml @@ -0,0 +1,127 @@ +--- +- name: Prepare shared build tools + ansible.builtin.import_playbook: ../shared/prepare-host.yaml + +- name: Prepare the Zuul buildset registry + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_registry_port: 5000 + s2i_ci_registry_image: quay.io/zuul-ci/zuul-registry@sha256:0bc02a2eed546daa570ebb18829579057cefceefd9ee15660dcf5a1a752d8f78 + + tasks: + - name: Record whether this job owns the buildset registry + ansible.builtin.set_fact: + s2i_ci_owns_buildset_registry: "{{ buildset_registry is not defined }}" + + - name: Discard a stale ownership marker for an inherited registry + when: not s2i_ci_owns_buildset_registry | bool + ansible.builtin.file: + path: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + state: absent + + - name: Remove a stale project-owned buildset registry container + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.command: + argv: [podman, rm, --force, buildset_registry] + register: s2i_ci_stale_buildset_registry + changed_when: s2i_ci_stale_buildset_registry.stdout | length > 0 + failed_when: s2i_ci_stale_buildset_registry.rc not in [0, 1] + + - name: Start the Zuul buildset registry + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.include_role: + name: run-buildset-registry + vars: + buildset_registry_root: "{{ s2i_ci_registry_root }}" + buildset_registry_port: "{{ s2i_ci_registry_port }}" + buildset_registry_image: "{{ s2i_ci_registry_image }}" + container_command: podman + + - name: Record ownership of the started buildset registry + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.copy: + dest: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + mode: "0600" + content: | + {{ { + 'container': ('buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string), + 'port': s2i_ci_registry_port + } | to_nice_json }} + + - name: Remove the unlabeled buildset registry container + when: + - s2i_ci_owns_buildset_registry | bool + - ansible_selinux.status | default("disabled") == "enabled" + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + changed_when: true + + - name: Start the buildset registry with SELinux-labeled volumes + when: + - s2i_ci_owns_buildset_registry | bool + - ansible_selinux.status | default("disabled") == "enabled" + ansible.builtin.command: + argv: + - podman + - run + - --detach + - "--name={{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + - --restart=always + - "--publish=1{{ s2i_ci_registry_port }}:5000" + - "--volume={{ s2i_ci_registry_root }}/tls:/tls:Z" + - "--volume={{ s2i_ci_registry_root }}/conf:/conf:Z" + - "{{ s2i_ci_registry_image }}" + - zuul-registry + - -d + changed_when: true + + - name: Wait for the buildset registry authentication endpoint + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.uri: + url: "https://127.0.0.1:{{ s2i_ci_registry_port }}/v2/" + validate_certs: false + status_code: 401 + register: s2i_ci_buildset_registry_api + retries: 30 + delay: 1 + until: s2i_ci_buildset_registry_api.status == 401 + + - name: Configure the host to use the Zuul buildset registry + ansible.builtin.include_role: + name: use-buildset-registry + vars: + buildset_registry_namespaces: + - [docker.io, https://registry-1.docker.io] + - [quay.io, https://quay.io] + - [gcr.io, https://gcr.io] + - [registry.k8s.io, https://registry.k8s.io] + - [registry.access.redhat.com, https://registry.access.redhat.com] + + - name: Set the Zuul buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_registry_endpoint: >- + {{ buildset_registry_alias }}:{{ buildset_registry.port }} + + - name: Write the normalized registry connection + ansible.builtin.copy: + dest: "{{ s2i_ci_registry_connection_file }}" + mode: "0600" + content: | + {{ { + 'endpoint': s2i_ci_registry_endpoint, + 'public_host': buildset_registry.host, + 'public_port': buildset_registry.port, + 'auth_file': '', + 'cert_dir': '' + } | to_nice_json }} + +- name: Validate the normalized buildset registry + ansible.builtin.import_playbook: ../shared/validate-registry.yaml diff --git a/playbooks/container-ci/zuul/run.yaml b/playbooks/container-ci/zuul/run.yaml new file mode 100644 index 00000000..e8afdfdf --- /dev/null +++ b/playbooks/container-ci/zuul/run.yaml @@ -0,0 +1,12 @@ +--- +- name: Run shared selective image publication + ansible.builtin.import_playbook: ../shared/run.yaml + +- name: Return published image metadata to Zuul + hosts: builder + gather_facts: false + + tasks: + - name: Return content-provider metadata to Zuul + when: s2i_ci_content_provider | default(false) | bool + ansible.builtin.include_tasks: content-provider-return.yaml diff --git a/tests/test_provider_architecture.py b/tests/test_provider_architecture.py new file mode 100644 index 00000000..011d5856 --- /dev/null +++ b/tests/test_provider_architecture.py @@ -0,0 +1,254 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pathlib +import re +import unittest + + +class ProviderArchitectureTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + self.container_ci = self.repo_root / "playbooks" / "container-ci" + + def _read(self, path): + return (self.repo_root / path).read_text(encoding="utf-8") + + def _image_mappings(self): + mappings = {} + current_target = None + for line in self._read("containers/image-mappings.yaml").splitlines(): + target = re.match(r"^ (\S+):$", line) + if target: + current_target = target.group(1) + mappings[current_target] = [] + continue + key = re.match(r"^ - (\S+)$", line) + if key and current_target: + mappings[current_target].append(key.group(1)) + return mappings + + def test_deployment_mappings_are_centralized(self): + self.assertTrue( + (self.repo_root / "containers/image-mappings.yaml").is_file() + ) + self.assertFalse( + list((self.repo_root / "containers").glob("**/image.yaml")) + ) + + def test_tracked_deployment_mappings_are_intentional(self): + self.assertEqual( + { + "glance/glance-api": ["glanceAPIImage"], + "manila/manila-api": ["manilaAPIImage"], + "manila/manila-scheduler": ["manilaSchedulerImage"], + "manila/manila-share": ["manilaShareImage"], + "watcher/watcher-base": [ + "watcherAPIImage", + "watcherApplierImage", + "watcherDecisionEngineImage", + ], + }, + self._image_mappings(), + ) + + def test_container_ci_mutations_target_only_builder(self): + playbooks = sorted(self.container_ci.glob("**/*.yaml")) + + self.assertTrue(playbooks) + for playbook in playbooks: + content = playbook.read_text(encoding="utf-8") + self.assertNotIn("hosts: all", content, playbook) + self.assertNotIn("hosts: localhost", content, playbook) + self.assertNotIn("delegate_to: localhost", content, playbook) + self.assertNotIn("local_action:", content, playbook) + for host_pattern in re.findall(r"^ hosts: (.+)$", content, re.M): + self.assertEqual("builder", host_pattern, playbook) + + def test_exact_cleanup_does_not_hide_failures(self): + cleanup = self._read( + "playbooks/container-ci/shared/cleanup-images.yaml" + ) + + self.assertNotIn("failed_when: false", cleanup) + self.assertIn("podman", cleanup) + self.assertIn("buildah", cleanup) + self.assertIn( + "Require exact workflow references to be absent", cleanup + ) + + def test_shared_and_zuul_ownership_is_separated(self): + shared = { + path.name for path in (self.container_ci / "shared").glob("*") + } + zuul = {path.name for path in (self.container_ci / "zuul").glob("*")} + + self.assertTrue( + { + "prepare-host.yaml", + "validate-registry.yaml", + "run.yaml", + "cleanup-images.yaml", + }.issubset(shared) + ) + self.assertTrue( + { + "pre.yaml", + "run.yaml", + "post.yaml", + "content-provider-return.yaml", + }.issubset(zuul) + ) + self.assertNotIn("reset-static-node.yaml", zuul) + self.assertNotIn("content-provider-return.yaml", shared) + + def test_provider_job_keeps_explicit_builder_contract(self): + zuul = self._read("zuul.d/jobs.yaml") + + self.assertIn("name: builder", zuul) + self.assertEqual( + 2, + zuul.count("nodeset: s2i-openstack-containers-image-builder"), + ) + self.assertIn("s2i_ci_images: all", zuul) + self.assertNotIn("- project:", zuul) + self.assertNotIn("abstract: true", zuul) + + def test_upstream_github_check_runs_configured_jobs(self): + layout = self._read("zuul.d/projects.yaml") + jobs = re.findall(r"^ - (\S+):$", layout, re.M) + + self.assertEqual( + [ + "s2i-openstack-containers-molecule", + "s2i-openstack-container-content-provider", + ], + jobs, + ) + self.assertIn("irrelevant-files:", layout) + self.assertNotIn("noop", layout) + + def test_provider_validates_repository_on_builder(self): + run = self._read("playbooks/container-ci/shared/run.yaml") + + self.assertIn("ansible.builtin.stat:", run) + self.assertIn("s2i_ci_build_entry_point.stat.isreg", run) + self.assertIn("argv: [realpath, --canonicalize-existing", run) + self.assertNotIn(" is file", run) + self.assertNotIn(" | realpath", run) + + def test_central_mapping_validates_all_tracked_entries(self): + loader = self._read( + "playbooks/container-ci/shared/load-image-mappings.yaml" + ) + item = self._read( + "playbooks/container-ci/shared/validate-tracked-mapping-item.yaml" + ) + + self.assertIn("resolve", loader) + self.assertIn( + "difference(s2i_ci_available_images.stdout_lines)", loader + ) + self.assertIn("Validate every tracked image mapping", loader) + self.assertIn("globally duplicated tracked deployment keys", loader) + self.assertIn("s2i_ci_tracked_mapping_keys is sequence", item) + self.assertIn("reject('match', '^[A-Za-z][A-Za-z0-9]*$')", item) + + def test_inventory_provider_inputs_are_not_masked_by_play_vars(self): + shared_run = self._read("playbooks/container-ci/shared/run.yaml") + zuul_run = self._read("playbooks/container-ci/zuul/run.yaml") + prepare_host = self._read( + "playbooks/container-ci/shared/prepare-host.yaml" + ) + + shared_play_vars = shared_run.split(" tasks:", 1)[0] + for variable in ( + "s2i_ci_namespace:", + "s2i_ci_tag:", + "s2i_ci_stream:", + "s2i_ci_parallel:", + "s2i_ci_images:", + "s2i_ci_image_mappings:", + ): + self.assertNotIn(variable, shared_play_vars) + self.assertNotIn("s2i_ci_content_provider: false", zuul_run) + self.assertNotIn("s2i_ci_install_host_packages: true", prepare_host) + self.assertIn("s2i_ci_images | default('all')", shared_run) + self.assertIn("s2i_ci_target_expression", shared_run) + self.assertIn("build.sh", shared_run) + self.assertIn("resolve", shared_run) + self.assertIn( + "s2i_ci_content_provider | default(false) | bool", zuul_run + ) + + def test_return_contract_is_selective_and_secret_free(self): + returned = self._read( + "playbooks/container-ci/zuul/content-provider-return.yaml" + ) + + for field in ( + "s2i_ci_content:", + "content_provider_os_custom_container_images:", + 'content_provider_os_registry_url: "null"', + "content_provider_dlrn_md5_hash:", + "content_provider_gating_repo_available: false", + "content_provider_gating_repo_url:", + "content_provider_registry_ip:", + "content_provider_registry_ip_port:", + "cifmw_build_images_output: {}", + "pause: true", + ): + self.assertIn(field, returned) + for secret in ("password", "username", "auth_file", "cert_dir"): + self.assertNotIn(secret, returned) + self.assertIn("s2i_ci_public_registry_endpoint", returned) + self.assertNotIn("s2i_ci_registry.endpoint", returned) + + def test_watcher_image_is_process_neutral(self): + containerfile = self._read( + "containers/watcher/watcher-base/Containerfile" + ) + bindeps = set( + line + for line in self._read( + "containers/watcher/watcher-base/bindeps.txt" + ).splitlines() + if line and not line.startswith("#") + ) + + self.assertIn( + "Consolidated Watcher API, applier, and decision-engine", + containerfile, + ) + self.assertTrue( + { + "httpd", + "python3-mod_wsgi", + "libffi", + "libxml2", + "libxslt", + }.issubset(bindeps) + ) + + def test_c2_has_no_oib_or_local_adapter(self): + self.assertFalse((self.repo_root / "openstack_image_builder").exists()) + self.assertFalse((self.container_ci / "local").exists()) + all_content = "\n".join( + path.read_text(encoding="utf-8") + for path in self.container_ci.glob("**/*.yaml") + ) + for forbidden in ("S2I_CONTEXTS_ROOT", "ERROR_ON_CLONE", "oib"): + self.assertNotIn(forbidden, all_content) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_provider_shell.py b/tests/test_provider_shell.py new file mode 100644 index 00000000..3fb076ff --- /dev/null +++ b/tests/test_provider_shell.py @@ -0,0 +1,200 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pathlib +import subprocess +import tempfile +import time +import unittest + + +class ProviderShellTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + temporary_root = self.repo_root / ".tmp" + temporary_root.mkdir(exist_ok=True) + self.temporary_directory = tempfile.TemporaryDirectory( + dir=temporary_root, prefix="provider-shell." + ) + self.addCleanup(self.temporary_directory.cleanup) + self.root = pathlib.Path(self.temporary_directory.name) + (self.root / "build.sh").symlink_to(self.repo_root / "build.sh") + self.bin_dir = self.root / "bin" + self.bin_dir.mkdir() + self.logs_dir = self.root / "logs" + self._create_images() + self._create_fake_buildah() + + def _create_images(self): + for target in ("base", "alpha/one", "beta/two"): + image_root = self.root / "containers" / target + image_root.mkdir(parents=True) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + project = target.split("/", maxsplit=1)[0] + project_root = self.root / "containers" / project + (project_root / "requirements.lock.master").touch() + if "/" in target: + (project_root / "src" / project).mkdir( + parents=True, exist_ok=True + ) + + def _create_fake_buildah(self): + fake = self.bin_dir / "buildah" + fake.write_text( + """#!/usr/bin/env python3 +import os +import pathlib +import sys +import time + +args = sys.argv[1:] +if args[0] == "bud": + containerfile = pathlib.Path(args[args.index("-f") + 1]) + image = containerfile.parent.name + if containerfile.parent.parent.name != "containers": + image = f"{containerfile.parent.parent.name}/{image}" + if image != "base": + print(f"LIVE {image}", flush=True) + time.sleep(0.75) + if os.environ.get("FAIL_IMAGE") and image.endswith(os.environ["FAIL_IMAGE"]): + print(f"FAIL {image}", flush=True) + sys.exit(9) + print(f"DONE {image}", flush=True) + sys.exit(0) +if args[0] == "inspect": + sys.exit(0) +if args[0] == "push": + sys.exit(0) +sys.exit(2) +""", + encoding="utf-8", + ) + fake.chmod(0o755) + + def _environment(self): + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "STREAM": "master", + "REGISTRY": "registry.test:5000", + "NAMESPACE": "openstack", + "TAG": "test", + "PARALLEL": "2", + "BUILD_LOGS_DIR": str(self.logs_dir), + } + ) + return environment + + def _run(self, *arguments, environment=None): + return subprocess.run( + [str(self.root / "build.sh"), *arguments], + cwd=self.root, + env=environment or self._environment(), + check=False, + capture_output=True, + text=True, + ) + + def test_explicit_union_is_ordered_and_includes_base(self): + result = self._run("refs", "beta/two,alpha/one,beta/two") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + [ + "registry.test:5000/openstack/openstack-base:test", + "registry.test:5000/openstack/openstack-two:test", + "registry.test:5000/openstack/openstack-one:test", + ], + result.stdout.splitlines(), + ) + + def test_single_target_semantics_are_unchanged(self): + result = self._run("refs", "alpha/one") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["registry.test:5000/openstack/openstack-one:test"], + result.stdout.splitlines(), + ) + + def test_resolve_all_returns_machine_readable_targets(self): + result = self._run("resolve", "all") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["base", "alpha/one", "beta/two"], result.stdout.splitlines() + ) + + def test_explicit_union_rejects_unknown_target(self): + result = self._run("refs", "alpha/one,unknown/image") + + self.assertNotEqual(0, result.returncode) + self.assertIn("Unknown image or project", result.stderr) + + def test_parallel_output_is_live_and_logs_are_retained(self): + process = subprocess.Popen( + [ + str(self.root / "build.sh"), + "build-parallel", + "alpha/one,beta/two", + ], + cwd=self.root, + env=self._environment(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.addCleanup( + lambda: process.kill() if process.poll() is None else None + ) + self.addCleanup(process.stdout.close) + output = [] + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + line = process.stdout.readline() + output.append(line) + if "LIVE" in line: + break + + self.assertIn("LIVE", "".join(output)) + self.assertIsNone( + process.poll(), "build exited before live output arrived" + ) + output.append(process.stdout.read()) + process.wait(timeout=10) + self.assertEqual(0, process.returncode, "".join(output)) + self.assertIn("[alpha/one] LIVE alpha/one", "".join(output)) + self.assertIn("[beta/two] LIVE beta/two", "".join(output)) + self.assertTrue((self.logs_dir / "base.log").is_file()) + self.assertTrue((self.logs_dir / "alpha_one.log").is_file()) + self.assertTrue((self.logs_dir / "beta_two.log").is_file()) + self.assertEqual(1, "".join(output).count("LIVE alpha/one")) + + def test_parallel_failure_propagates(self): + environment = self._environment() + environment["FAIL_IMAGE"] = "two" + + result = self._run( + "build-parallel", "alpha/one,beta/two", environment=environment + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("FAIL beta/two", result.stdout + result.stderr) + self.assertIn("stopping remaining builds", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tox.ini b/tox.ini index bec1fbe4..1c4494d0 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ passenv = SKIP_HASH_UPDATE PIP_NO_BINARY REGISTRY_AUTH_FILE + REGISTRY_CERT_DIR PARALLEL BUILD_LOGS_DIR UPDATE_LOCKFILES_TARGETS @@ -52,6 +53,22 @@ setenv = commands = pre-commit run -a --show-diff-on-failure +[testenv:molecule] +description = Exercise provider registry and cleanup contracts with Molecule +deps = + ansible-core==2.18.7 + molecule==26.6.0 +setenv = + PYTHONPYCACHEPREFIX = {toxinidir}/.tmp/python-cache + MOLECULE_EPHEMERAL_DIRECTORY = {toxinidir}/.tmp/molecule/ephemeral/{envname} + ANSIBLE_HOME = {toxinidir}/.tmp/ansible/home + ANSIBLE_COLLECTIONS_PATH = {toxinidir}/.tmp/ansible/collections + ANSIBLE_LOCAL_TEMP = {toxinidir}/.tmp/ansible/local-tmp + ANSIBLE_ROLES_PATH = {toxinidir}/.tmp/ansible/roles +commands = + molecule reset -s provider-contract + molecule test -s provider-contract + [testenv:update-sources] description = Refresh source pins and generated lockfiles setenv = diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml new file mode 100644 index 00000000..99fe2b35 --- /dev/null +++ b/zuul.d/jobs.yaml @@ -0,0 +1,38 @@ +--- +- job: + name: s2i-openstack-containers-molecule + parent: tox + description: Run provider registry and cleanup contract scenarios. + nodeset: s2i-openstack-containers-image-builder + vars: + python_version: "3.12" + tox_envlist: molecule + +- nodeset: + name: s2i-openstack-containers-image-builder + nodes: + - name: builder + label: cloud-centos-10-stream + +- job: + name: s2i-openstack-container-content-provider + parent: base + description: | + Build and publish OpenStack service container images from committed + source pins for dependent jobs. This repository builds all maintained + images by default; child jobs can select a subset and add related + required projects. Speculative source consumption remains follow-up work. + timeout: 7200 + nodeset: s2i-openstack-containers-image-builder + required-projects: + - github.com/openstack-k8s-operators/s2i-openstack-containers + - opendev.org/zuul/zuul-jobs + pre-run: playbooks/container-ci/zuul/pre.yaml + run: playbooks/container-ci/zuul/run.yaml + post-run: playbooks/container-ci/zuul/post.yaml + vars: + s2i_ci_container_project: >- + github.com/openstack-k8s-operators/s2i-openstack-containers + s2i_ci_images: all + s2i_ci_stream: master + s2i_ci_content_provider: true diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index ca302dea..4bd2a3be 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -1,8 +1,12 @@ --- -# Minimal RDO github-check wiring so Zuul loads in-repo config from main. -# Expand with real jobs once webhook/status checks are confirmed. - project: name: openstack-k8s-operators/s2i-openstack-containers github-check: jobs: - - noop + - s2i-openstack-containers-molecule: + irrelevant-files: &documentation-only + - ^docs/.* + - ^.*\.md$ + - ^LICENSE.*$ + - s2i-openstack-container-content-provider: + irrelevant-files: *documentation-only