From ef6a8e18306d6541d766fbb468f7d691d97ed548 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 17:33:05 +0200 Subject: [PATCH 1/9] feat: declare base image build dependencies in weblog_metadata.yml --- docs/understand/weblogs/weblog-metadata.md | 20 +++ utils/_context/weblog_metadata.py | 17 +++ utils/build/docker/nodejs/weblog_metadata.yml | 23 ++++ utils/build/docker/php/weblog_metadata.yml | 114 ++++++++++++++++++ utils/build/docker/python/weblog_metadata.yml | 18 +++ 5 files changed, 192 insertions(+) create mode 100644 utils/build/docker/php/weblog_metadata.yml diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md index 2730d9e6875..7a121c3131b 100644 --- a/docs/understand/weblogs/weblog-metadata.md +++ b/docs/understand/weblogs/weblog-metadata.md @@ -45,3 +45,23 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`. `WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges: 1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata). 2. Explicit overrides from `weblog_metadata.yml`. + +## Base image dependencies + +A `weblog_metadata.yml` may also declare a top-level `base_image_dependencies` section, unrelated +to the per-weblog entries above, used by the `build_base_images` CI job +(`utils/scripts/build_base_images.py`) to know when a weblog base image needs to be rebuilt: + +```yaml +base_image_dependencies: + : + - + - ... +``` + +For each target listed there, the job computes a content hash from the resolved +`docker-bake.hcl` target config, the target's Dockerfile, and every git-tracked file under the +listed paths, then pushes the base image to Docker Hub tagged `-` if that tag +doesn't already exist. It never overwrites an existing tag, so weblog Dockerfiles that `FROM` a +base image must have their tag updated by hand after a new one is pushed (run the script with +`--dry-run` to find the current tag for each target). diff --git a/utils/_context/weblog_metadata.py b/utils/_context/weblog_metadata.py index 7d72d1a2bfb..5523d712053 100644 --- a/utils/_context/weblog_metadata.py +++ b/utils/_context/weblog_metadata.py @@ -68,8 +68,25 @@ def _load_explicit_metadata(library: str) -> dict[str, "WeblogMetaData"]: with path.open() as f: data: dict = yaml.safe_load(f) or {} + data.pop("base_image_dependencies", None) + return {name: WeblogMetaData(name=name, library=library, **kwargs) for name, kwargs in data.items()} + @staticmethod + def load_base_image_dependencies(library: str) -> dict[str, list[str]]: + """Returns the `base_image_dependencies` section of weblog_metadata.yml: a mapping of + docker-bake.hcl target name to the list of paths (files or directories) that base image + depends on, used to compute a content-hash tag for the base image build job. + """ + path = Path(f"utils/build/docker/{library}/weblog_metadata.yml") + if not path.exists(): + return {} + + with path.open() as f: + data: dict = yaml.safe_load(f) or {} + + return data.get("base_image_dependencies", {}) or {} + @staticmethod def load(library: str) -> list["WeblogMetaData"]: metadata = WeblogMetaData._load_explicit_metadata(library) diff --git a/utils/build/docker/nodejs/weblog_metadata.yml b/utils/build/docker/nodejs/weblog_metadata.yml index 802e518616e..4a7da1a482b 100644 --- a/utils/build/docker/nodejs/weblog_metadata.yml +++ b/utils/build/docker/nodejs/weblog_metadata.yml @@ -21,3 +21,26 @@ anthropic-js: google_genai-js: build_mode: none framework_versions: ["1.34.0"] + +# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by +# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. +base_image_dependencies: + express4: + - utils/build/docker/nodejs/express + - utils/build/docker/nodejs/express4/package.json + - utils/build/docker/nodejs/express4/bun.lock + - utils/build/docker/nodejs/nft-prune.mjs + express5: + - utils/build/docker/nodejs/express + - utils/build/docker/nodejs/express5/package.json + - utils/build/docker/nodejs/express5/bun.lock + - utils/build/docker/nodejs/nft-prune.mjs + express4-typescript: + - utils/build/docker/nodejs/express4-typescript + - utils/build/docker/nodejs/nft-prune.mjs + fastify: + - utils/build/docker/nodejs/fastify + - utils/build/docker/nodejs/nft-prune.mjs + nextjs: + - utils/build/docker/nodejs/nextjs + - utils/build/docker/nodejs/nft-prune.mjs diff --git a/utils/build/docker/php/weblog_metadata.yml b/utils/build/docker/php/weblog_metadata.yml new file mode 100644 index 00000000000..91ffc08aa5d --- /dev/null +++ b/utils/build/docker/php/weblog_metadata.yml @@ -0,0 +1,114 @@ +# docs/understand/weblogs/weblog-metadata.md + +# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by +# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. +base_image_dependencies: + apache-mod-7_0: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_1: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_2: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_3: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_4: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_0: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_1: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_2: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_0-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_1-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_2-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_3-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_4-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_0-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_1-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_2-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_0: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_1: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_2: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_3: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_4: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_0: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_1: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_2: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_5: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common diff --git a/utils/build/docker/python/weblog_metadata.yml b/utils/build/docker/python/weblog_metadata.yml index c0b67e68fa7..c8e71bb0297 100644 --- a/utils/build/docker/python/weblog_metadata.yml +++ b/utils/build/docker/python/weblog_metadata.yml @@ -9,3 +9,21 @@ anthropic-py: google_genai-py: build_mode: none framework_versions: ["1.55.0"] + +# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by +# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. +base_image_dependencies: + django-poc: + - utils/build/docker/python/django/requirements-django-poc.txt + django-py3_13: + - utils/build/docker/python/django/requirements-django-py3.13.txt + python3_12: + - utils/build/docker/python/django/requirements-python3.12.txt + fastapi: + - utils/build/docker/python/fastapi/requirements-fastapi.txt + flask-poc: + - utils/build/docker/python/flask/requirements-flask-poc.txt + uwsgi-poc: + - utils/build/docker/python/flask/requirements-uwsgi-poc.txt + tornado: + - utils/build/docker/python/tornado/requirements-tornado.txt From 8dccc014e6757408ae61c9206f97f3b1209b314e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 17:33:15 +0200 Subject: [PATCH 2/9] refactor: drop static -vN suffix from base image bake tags Base images will now be tagged with a content hash appended by utils/scripts/build_base_images.py. --- utils/build/docker/nodejs/docker-bake.hcl | 10 ++--- utils/build/docker/php/docker-bake.hcl | 50 +++++++++++------------ utils/build/docker/python/docker-bake.hcl | 14 +++---- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/utils/build/docker/nodejs/docker-bake.hcl b/utils/build/docker/nodejs/docker-bake.hcl index a8430a0ed51..45c1d071caf 100644 --- a/utils/build/docker/nodejs/docker-bake.hcl +++ b/utils/build/docker/nodejs/docker-bake.hcl @@ -18,33 +18,33 @@ target "express4" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/express4.base.Dockerfile" - tags = ["datadog/system-tests:express4.base-v3"] + tags = ["datadog/system-tests:express4.base"] } target "express5" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/express5.base.Dockerfile" - tags = ["datadog/system-tests:express5.base-v3"] + tags = ["datadog/system-tests:express5.base"] } target "fastify" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/fastify.base.Dockerfile" - tags = ["datadog/system-tests:fastify.base-v3"] + tags = ["datadog/system-tests:fastify.base"] } target "express4-typescript" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/express4-typescript.base.Dockerfile" - tags = ["datadog/system-tests:express4-typescript.base-v3"] + tags = ["datadog/system-tests:express4-typescript.base"] } target "nextjs" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/nextjs.base.Dockerfile" - tags = ["datadog/system-tests:nextjs.base-v3"] + tags = ["datadog/system-tests:nextjs.base"] } diff --git a/utils/build/docker/php/docker-bake.hcl b/utils/build/docker/php/docker-bake.hcl index 880178b38b9..47308716f3e 100644 --- a/utils/build/docker/php/docker-bake.hcl +++ b/utils/build/docker/php/docker-bake.hcl @@ -35,158 +35,158 @@ group "default" { target "apache-mod-7_0" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.0.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.0.base"] } target "apache-mod-7_1" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.1.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.1.base"] } target "apache-mod-7_2" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.2.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.2.base"] } target "apache-mod-7_3" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.3.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.3.base"] } target "apache-mod-7_4" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.4.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.4.base"] } target "apache-mod-8_0" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.0.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.0.base"] } target "apache-mod-8_1" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.1.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.1.base"] } target "apache-mod-8_2" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.2.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.2.base"] } target "apache-mod-7_0-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.0-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.0-zts.base"] } target "apache-mod-7_1-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.1-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.1-zts.base"] } target "apache-mod-7_2-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.2-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.2-zts.base"] } target "apache-mod-7_3-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.3-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.3-zts.base"] } target "apache-mod-7_4-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.4-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.4-zts.base"] } target "apache-mod-8_0-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.0-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.0-zts.base"] } target "apache-mod-8_1-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.1-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.1-zts.base"] } target "apache-mod-8_2-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.2-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.2-zts.base"] } target "php-fpm-7_0" { context = "utils/build/docker/php/" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.0" } - tags = ["datadog/system-tests:php-fpm-7.0.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.0.base"] } target "php-fpm-7_1" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.1" } - tags = ["datadog/system-tests:php-fpm-7.1.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.1.base"] } target "php-fpm-7_2" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.2" } - tags = ["datadog/system-tests:php-fpm-7.2.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.2.base"] } target "php-fpm-7_3" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.3" } - tags = ["datadog/system-tests:php-fpm-7.3.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.3.base"] } target "php-fpm-7_4" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.4" } - tags = ["datadog/system-tests:php-fpm-7.4.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.4.base"] } target "php-fpm-8_0" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.0" } - tags = ["datadog/system-tests:php-fpm-8.0.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.0.base"] } target "php-fpm-8_1" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.1" } - tags = ["datadog/system-tests:php-fpm-8.1.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.1.base"] } target "php-fpm-8_2" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.2" } - tags = ["datadog/system-tests:php-fpm-8.2.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.2.base"] } target "php-fpm-8_5" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.5" } - tags = ["datadog/system-tests:php-fpm-8.5.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.5.base"] } diff --git a/utils/build/docker/python/docker-bake.hcl b/utils/build/docker/python/docker-bake.hcl index 3e3f0e8af99..0cf888f82d1 100644 --- a/utils/build/docker/python/docker-bake.hcl +++ b/utils/build/docker/python/docker-bake.hcl @@ -15,41 +15,41 @@ group "default" { target "django-py3_13" { context = "." dockerfile = "utils/build/docker/python/django-py3.13.base.Dockerfile" - tags = ["datadog/system-tests:django-py3.13.base-v10"] + tags = ["datadog/system-tests:django-py3.13.base"] } target "fastapi" { context = "." dockerfile = "utils/build/docker/python/fastapi.base.Dockerfile" - tags = ["datadog/system-tests:fastapi.base-v9"] + tags = ["datadog/system-tests:fastapi.base"] } target "python3_12" { context = "." dockerfile = "utils/build/docker/python/python3.12.base.Dockerfile" - tags = ["datadog/system-tests:python3.12.base-v13"] + tags = ["datadog/system-tests:python3.12.base"] } target "django-poc" { context = "." dockerfile = "utils/build/docker/python/django-poc.base.Dockerfile" - tags = ["datadog/system-tests:django-poc.base-v11"] + tags = ["datadog/system-tests:django-poc.base"] } target "flask-poc" { context = "." dockerfile = "utils/build/docker/python/flask-poc.base.Dockerfile" - tags = ["datadog/system-tests:flask-poc.base-v14"] + tags = ["datadog/system-tests:flask-poc.base"] } target "uwsgi-poc" { context = "." dockerfile = "utils/build/docker/python/uwsgi-poc.base.Dockerfile" - tags = ["datadog/system-tests:uwsgi-poc.base-v10"] + tags = ["datadog/system-tests:uwsgi-poc.base"] } target "tornado" { context = "." dockerfile = "utils/build/docker/python/tornado.base.Dockerfile" - tags = ["datadog/system-tests:tornado.base-v2"] + tags = ["datadog/system-tests:tornado.base"] } From f4aa6a13de0de1b601ac60b69f7d39c5c3999809 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 18:16:56 +0200 Subject: [PATCH 3/9] feat: add CI job to build and push base images on change Adds build_base_images.py, which for every docker-bake.hcl target listed under base_image_dependencies in a library's weblog_metadata.yml, hashes the target's bake config, Dockerfile, and declared dependencies, and pushes the base image to Docker Hub tagged with that hash if missing. Wires it up as a build_base_images job in .gitlab-ci.yml, running on every push. It is idempotent: existing tags are never overwritten, so it is safe to run ahead of merge. --- .gitlab-ci.yml | 33 ++++++ utils/scripts/build_base_images.py | 170 +++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 utils/scripts/build_base_images.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c92374ff41..67c2ccbc58d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -473,3 +473,36 @@ mirror_images: - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' + +# ────────────────────────────────────────────── +# Rebuild weblog base images whose content changed. +# +# For every library with a `base_image_dependencies` section in +# utils/build/docker//weblog_metadata.yml, computes a content-hash tag per +# docker-bake.hcl target and pushes it if missing (see utils/scripts/build_base_images.py). +# Runs on every push, on every branch: it is idempotent (only pushes new tags, never +# overwrites existing ones), so it is safe to build ahead of merge. Since new tags never +# replace old ones, a weblog Dockerfile's FROM clause must be updated by hand to pick up +# a newly pushed base image (run the script with --dry-run to find the new tag). +# ────────────────────────────────────────────── + +build_base_images: + image: $CI_IMAGE + tags: + - docker-in-docker:amd64 + needs: + - job: build_ci_image + artifacts: false + optional: true + stage: system-tests-utils + allow_failure: true + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ln -sf /system-tests/venv venv + - source venv/bin/activate + script: + - python utils/scripts/build_base_images.py + rules: + - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py new file mode 100644 index 00000000000..4419fa903ff --- /dev/null +++ b/utils/scripts/build_base_images.py @@ -0,0 +1,170 @@ +"""Rebuild and push weblog base images whose content-hash tag is missing from Docker Hub. + +Run from a system-tests checkout, with the runner venv active: + + python utils/scripts/build_base_images.py + +For every library with a `utils/build/docker//weblog_metadata.yml` +declaring a `base_image_dependencies` section, and for every docker-bake.hcl target +listed there: + + 1. Resolve the target's bake config (context/dockerfile/args) via + `docker buildx bake --print`. + 2. Compute a content hash from: the resolved bake config (tags excluded), the + content of the target's Dockerfile, and the content of every git-tracked file + under the paths listed in `base_image_dependencies`. + 3. Take the base tag declared in the bake file (e.g. "datadog/system-tests:express4.base") + and append "-" to get the final tag. + 4. Skip the build if that tag already exists on Docker Hub (`docker manifest inspect`); + otherwise build and push it with that tag. + +This is idempotent and safe to run on every push: it never overwrites an existing +tag, it only creates new ones when the relevant files change. + +Pass --dry-run to only print the computed tag and whether it already exists, +without ever building or pushing (useful to find the tag to put in a weblog +Dockerfile's FROM clause after dependencies change). +""" + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +# Make `utils` importable and resolve paths regardless of the caller's cwd, so a +# plain `python utils/scripts/build_base_images.py` works. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from utils._context.weblog_metadata import WeblogMetaData # noqa: E402 +from utils.const import COMPONENT_GROUPS # noqa: E402 + + +def _bake_file(library: str) -> Path: + return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" + + +def _bake_config(bake_file: Path, target: str) -> dict: + """Resolved bake config (context, dockerfile, args, tags) for a single target.""" + result = subprocess.run( + ["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout)["target"][target] + + +def _tracked_files(path: str) -> list[Path]: + """Every git-tracked file under `path` (a file or a directory), sorted.""" + result = subprocess.run( + ["git", "ls-files", path], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + return sorted(REPO_ROOT / line for line in result.stdout.splitlines() if line) + + +def compute_hash(bake_config: dict, dependencies: list[str]) -> str: + """Content hash for a base image target: bake config (minus tags) + Dockerfile + content + content of every git-tracked file under `dependencies`. + """ + digest = hashlib.sha256() + + config_without_tags = {k: v for k, v in bake_config.items() if k != "tags"} + digest.update(json.dumps(config_without_tags, sort_keys=True).encode()) + + dockerfile = REPO_ROOT / bake_config["context"] / bake_config["dockerfile"] + digest.update(dockerfile.read_bytes()) + + files: list[Path] = [] + for dep in dependencies: + files.extend(_tracked_files(dep)) + + for file in sorted(set(files)): + digest.update(str(file.relative_to(REPO_ROOT)).encode()) + digest.update(file.read_bytes()) + + return digest.hexdigest()[:12] + + +def image_exists(tag: str) -> bool: + result = subprocess.run( + ["docker", "manifest", "inspect", tag], + cwd=REPO_ROOT, + check=False, + capture_output=True, + ) + return result.returncode == 0 + + +def build_and_push(bake_file: Path, target: str, tag: str) -> None: + print(f"Building and pushing {tag}") + subprocess.run( + [ + "docker", + "buildx", + "bake", + "--push", + "--progress=plain", + "--set", + f"{target}.tags={tag}", + "-f", + str(bake_file), + target, + ], + cwd=REPO_ROOT, + check=True, + ) + + +def process_library(library: str, *, dry_run: bool) -> None: + dependencies_by_target = WeblogMetaData.load_base_image_dependencies(library) + if not dependencies_by_target: + return + + bake_file = _bake_file(library) + if not bake_file.exists(): + print(f"Warning: {library} declares base_image_dependencies but has no docker-bake.hcl, skipping") + return + + for target, dependencies in dependencies_by_target.items(): + bake_config = _bake_config(bake_file, target) + base_tag = bake_config["tags"][0] + content_hash = compute_hash(bake_config, dependencies) + tag = f"{base_tag}-{content_hash}" + + if dry_run: + state = "exists" if image_exists(tag) else "missing" + print(f"{library}/{target}: {tag} ({state})") + continue + + if image_exists(tag): + print(f"{tag} already exists, skipping") + continue + + build_and_push(bake_file, target, tag) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Rebuild and push weblog base images with a content-hash tag") + parser.add_argument("--library", help="Only process this library (default: all libraries)") + parser.add_argument( + "--dry-run", + action="store_true", + help="Only print the computed tag and whether it exists on Docker Hub; never build or push", + ) + args = parser.parse_args() + + libraries = [args.library] if args.library else sorted(COMPONENT_GROUPS.all) + for library in libraries: + process_library(library, dry_run=args.dry_run) + + +if __name__ == "__main__": + main() From 611f535c855518a8fdd04679959d16771a04202f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 18:23:34 +0200 Subject: [PATCH 4/9] Waiting for base image build for e2e gitlab --- .gitlab-ci.yml | 3 +-- utils/ci/gitlab/main.yml | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67c2ccbc58d..9786039e299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -494,8 +494,7 @@ build_base_images: - job: build_ci_image artifacts: false optional: true - stage: system-tests-utils - allow_failure: true + stage: e2e before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 2c596c89566..475fcf4e638 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -155,6 +155,9 @@ system_tests_build_pipeline: - job: mirror_images optional: true artifacts: false + - job: build_base_images + optional: true + artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$SYSTEM_TESTS_FORCE_EXECUTE" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$SYSTEM_TESTS_SKIP_EMPTY_SCENARIO" From 0beb1871acf8b98c4a9b10776d1db8db1823a620 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 19:54:37 +0200 Subject: [PATCH 5/9] fix: surface docker/git command errors in build_base_images.py Previously image_exists() silently treated any docker manifest inspect failure as "tag doesn't exist", masking real errors (auth, network). Print stderr on failure there, and in a new _run() helper used by the other docker/git subprocess calls, so failures are visible instead of only raising a bare CalledProcessError. --- utils/scripts/build_base_images.py | 42 +++++++++++++++++------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py index 4419fa903ff..844d58eaebe 100644 --- a/utils/scripts/build_base_images.py +++ b/utils/scripts/build_base_images.py @@ -46,27 +46,28 @@ def _bake_file(library: str) -> Path: return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" +def _run(cmd: list[str]) -> subprocess.CompletedProcess: + """Run `cmd`, printing stderr (and stdout, if any) before raising on failure.""" + result = subprocess.run(cmd, cwd=REPO_ROOT, check=False, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: command failed: {' '.join(cmd)}") + if result.stdout: + print(result.stdout) + if result.stderr: + print(result.stderr) + result.check_returncode() + return result + + def _bake_config(bake_file: Path, target: str) -> dict: """Resolved bake config (context, dockerfile, args, tags) for a single target.""" - result = subprocess.run( - ["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target], - cwd=REPO_ROOT, - check=True, - capture_output=True, - text=True, - ) + result = _run(["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target]) return json.loads(result.stdout)["target"][target] def _tracked_files(path: str) -> list[Path]: """Every git-tracked file under `path` (a file or a directory), sorted.""" - result = subprocess.run( - ["git", "ls-files", path], - cwd=REPO_ROOT, - check=True, - capture_output=True, - text=True, - ) + result = _run(["git", "ls-files", path]) return sorted(REPO_ROOT / line for line in result.stdout.splitlines() if line) @@ -94,18 +95,25 @@ def compute_hash(bake_config: dict, dependencies: list[str]) -> str: def image_exists(tag: str) -> bool: + """Whether `tag` exists on the registry. `docker manifest inspect` exits non-zero both + when the tag genuinely doesn't exist and on unrelated failures (auth, network); print + the error either way so a real failure isn't silently mistaken for a missing tag. + """ result = subprocess.run( ["docker", "manifest", "inspect", tag], cwd=REPO_ROOT, check=False, capture_output=True, + text=True, ) + if result.returncode != 0 and result.stderr: + print(result.stderr.strip()) return result.returncode == 0 def build_and_push(bake_file: Path, target: str, tag: str) -> None: print(f"Building and pushing {tag}") - subprocess.run( + _run( [ "docker", "buildx", @@ -117,9 +125,7 @@ def build_and_push(bake_file: Path, target: str, tag: str) -> None: "-f", str(bake_file), target, - ], - cwd=REPO_ROOT, - check=True, + ] ) From 0797a090cd7d132ce298bdffb2b692983140a482 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 20:26:55 +0200 Subject: [PATCH 6/9] fix(php): install mbstring extension in php-fpm base images composer install failed for php-fpm targets because stripe/stripe-php requires ext-mbstring, which wasn't installed. apache-mod targets were unaffected since they use the official PHP image, which bundles it. --- utils/build/docker/php/php-fpm/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/docker/php/php-fpm/build.sh b/utils/build/docker/php/php-fpm/build.sh index c73bd0d6f3e..8a00683520c 100644 --- a/utils/build/docker/php/php-fpm/build.sh +++ b/utils/build/docker/php/php-fpm/build.sh @@ -31,7 +31,7 @@ nala update printf '#!/bin/sh\n\nexit 101\n' > /usr/sbin/policy-rc.d chmod +x /usr/sbin/policy-rc.d -nala install -y --no-install-recommends tzdata publicsuffix curl apache2 libapache2-mod-fcgid jq ca-certificates git unzip php$PHP_VERSION-fpm php$PHP_VERSION-curl apache2 php$PHP_VERSION-mysql php$PHP_VERSION-pgsql php$PHP_VERSION-xml php$PHP_VERSION-mongodb +nala install -y --no-install-recommends tzdata publicsuffix curl apache2 libapache2-mod-fcgid jq ca-certificates git unzip php$PHP_VERSION-fpm php$PHP_VERSION-curl apache2 php$PHP_VERSION-mysql php$PHP_VERSION-pgsql php$PHP_VERSION-xml php$PHP_VERSION-mongodb php$PHP_VERSION-mbstring rm -rf /usr/sbin/policy-rc.d rm -rf /var/lib/apt/lists/* From b8f1ceb4edb50e131d52d5b0c69f6b9ecb18b8a9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 7 Jul 2026 13:48:46 +0200 Subject: [PATCH 7/9] feat: replace GitHub weblog base image build with a poll for GitLab's push GitHub Actions has no dependency mechanism on GitLab CI jobs, so instead of building weblog base images itself (via the removed, already-broken utils/script/build_base_image.py and PR labels), it now waits for the tag declared in the weblog's Dockerfile to be pushed by GitLab's build_base_images job, polling Docker Hub every 30s with a 15 minute timeout (utils/scripts/wait_for_base_image.py). Drops the _build_weblog_base_images label-gated flag entirely: waiting is cheap (a single manifest inspect when the tag already exists) so it now runs unconditionally for every weblog that has a base image tag. --- .github/workflows/ci.yml | 1 - .../workflows/compute-workflow-parameters.yml | 7 -- .github/workflows/run-end-to-end.yml | 8 +- .github/workflows/system-tests.yml | 14 +--- docs/CI/github-actions.md | 1 - docs/edit/update-docker-images.md | 25 ++++-- docs/understand/weblogs/weblog-metadata.md | 4 + tests/test_the_test/test_ci_orchestrator.py | 17 ++-- utils/_context/weblog_metadata.py | 14 ---- utils/scripts/build-base-image.py | 41 ---------- .../scripts/ci_orchestrators/workflow_data.py | 22 ++---- utils/scripts/compute-workflow-parameters.py | 9 --- utils/scripts/wait_for_base_image.py | 78 +++++++++++++++++++ 13 files changed, 117 insertions(+), 124 deletions(-) delete mode 100755 utils/scripts/build-base-image.py create mode 100755 utils/scripts/wait_for_base_image.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e50b1d1255..b49270477c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,6 @@ jobs: _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }} _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }} _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }} - _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }} _enable_replay_scenarios: true _system_tests_dev_mode: ${{ matrix.version == 'dev' }} _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }} diff --git a/.github/workflows/compute-workflow-parameters.yml b/.github/workflows/compute-workflow-parameters.yml index f5ac934999a..0405493a000 100644 --- a/.github/workflows/compute-workflow-parameters.yml +++ b/.github/workflows/compute-workflow-parameters.yml @@ -52,11 +52,6 @@ on: default: '' required: false type: string - _build_weblog_base_images: - description: "Shall we build weblog base images" - default: false - required: false - type: boolean # Map the workflow outputs to job outputs outputs: @@ -142,7 +137,6 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ - --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --output $GITHUB_OUTPUT - name: log run: | @@ -156,7 +150,6 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ - --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --format json | jq - name: Extract library target branch diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 56564d9680b..1fc7d6e89f5 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -63,7 +63,7 @@ on: required: false type: boolean _build_weblog_base_image: - description: "Shall we build the weblog base image (only valid if weblog build is local)" + description: "Shall we wait for the weblog base image to be pushed by GitLab CI before building the weblog" default: false required: false type: boolean @@ -146,11 +146,9 @@ jobs: if: inputs._build_lambda_proxy_image run: ./build.sh -i lambda-proxy - - name: Build weblog base image + - name: Wait for weblog base image if: inputs._build_weblog_base_image - run: | - source venv/bin/activate - ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} + run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30 - name: Pull images uses: ./.github/actions/pull_images diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index a783c6f9ca1..9ea0bc6961a 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -44,11 +44,6 @@ on: default: 'custom' required: false type: string - _build_weblog_base_images: - description: "Shall we rebuild weblog base images" - default: false - required: false - type: boolean _build_buddies_images: description: "Shall we build buddies images" default: false @@ -154,7 +149,6 @@ jobs: binaries_artifact: ${{ inputs.binaries_artifact }} _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }} _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }} - _build_weblog_base_images: ${{ inputs._build_weblog_base_images }} parametric: needs: @@ -215,11 +209,9 @@ jobs: run: ls -la binaries/ - name: Export github token to a file run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt" - - name: Build weblog base images - if: matrix.weblog.build_base_image - run: | - source venv/bin/activate - ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} + - name: Wait for weblog base image + if: matrix.weblog.build_weblog_base_image + run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30 - name: Build weblog id: build run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt" diff --git a/docs/CI/github-actions.md b/docs/CI/github-actions.md index 75a2fc61bce..2272e4f7787 100644 --- a/docs/CI/github-actions.md +++ b/docs/CI/github-actions.md @@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI | `_build_buddies_images` | Shall we build buddies images | boolean | false | false | | `_build_proxy_image` | Shall we build proxy image | boolean | false | false | | `_build_lambda_proxy_image` | Shall we build the lambda-proxy image | boolean | false | false | -| `_build_weblog_base_images` | Shall we build weblog base images | boolean | false | false | | `_enable_replay_scenarios` | Enable replay scenarios, should only be used in system-tests CI | boolean | false | false | diff --git a/docs/edit/update-docker-images.md b/docs/edit/update-docker-images.md index a2ae67d46e4..6ebe066acf2 100644 --- a/docs/edit/update-docker-images.md +++ b/docs/edit/update-docker-images.md @@ -1,11 +1,20 @@ Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/). -If you need to update them, you will need to follow those +For weblog base images (nodejs, python, php), the build and push are fully automated: -1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code) -2. create your PR, and add the relevant label to rebuild the image in the CI - * `build-python-base-images` for python weblogs - * `build-php-base-images` for PHP weblogs - * `build-nodejs-base-images` for Node.js weblogs - * `build-proxy-image` for proxy image -3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack) +1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every + branch. For each `docker-bake.hcl` target declared in a library's `weblog_metadata.yml` + `base_image_dependencies` section (see `docs/understand/weblogs/weblog-metadata.md`), it computes a + content hash of the target's dependencies and, if a base image tagged with that hash doesn't already + exist on Docker Hub, builds and pushes it as `-`. +2. If you changed a file listed in `base_image_dependencies`, a new tag will automatically be pushed by + that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to point to the new tag (you can + find it by running `python utils/scripts/build_base_images.py --dry-run` locally). +3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for + the tag referenced in the weblog's `FROM` line to become available before building the weblog, via + `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag + before (or shortly after) your PR's GitHub CI run starts. + +For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a +rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push +your image to hub.docker.com (`#apm-shared-testing` on slack). diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md index 7a121c3131b..2359444cf3b 100644 --- a/docs/understand/weblogs/weblog-metadata.md +++ b/docs/understand/weblogs/weblog-metadata.md @@ -65,3 +65,7 @@ listed paths, then pushes the base image to Docker Hub tagged `- bool: """The run_end_to_end job builds the weblog locally (weblog_build_required).""" return self.build_mode != BuildMode.none - @property - def base_dockerfile(self) -> Path | None: - """Returns the path of the base image docker file if exists, else None""" - image_name = self.base_image_tag - - if image_name is None: - return None - - file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0] - assert file_prefix.endswith(".base") - - path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile") - return path if path.exists() else None - @property def base_image_tag(self) -> str | None: """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" diff --git a/utils/scripts/build-base-image.py b/utils/scripts/build-base-image.py deleted file mode 100755 index f52d9c3ab8b..00000000000 --- a/utils/scripts/build-base-image.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import subprocess -import sys - -from utils._context.weblog_metadata import WeblogMetaData - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build a weblog base image") - parser.add_argument("library", help="Library name (e.g. nodejs, python)") - parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") - args = parser.parse_args() - - weblogs = WeblogMetaData.load(args.library) - names = [w.name for w in weblogs] - - if args.weblog not in names: - print(f"Error: weblog '{args.weblog}' not found for library '{args.library}'") - print(f"Available weblogs: {', '.join(sorted(names))}") - sys.exit(1) - - weblog = next(w for w in weblogs if w.name == args.weblog) - dockerfile = weblog.base_dockerfile - if dockerfile is None: - print(f"Error: no base Dockerfile found for {args.weblog}") - sys.exit(1) - - image_tag = weblog.base_image_tag - if image_tag is None: - print(f"Error: could not extract base image tag for {args.weblog}") - sys.exit(1) - - print(f"Building base image: {image_tag}") - subprocess.run(["docker", "build", "--progress=plain", "-f", str(dockerfile), "-t", image_tag, "."], check=True) - - print(f"\nDone. To push:\n docker push {image_tag}") - - -if __name__ == "__main__": - main() diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 4cdc6100b4e..ba5cb662714 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -206,8 +206,6 @@ def __init__( weblog_instance: int, scenarios_times: dict[str, float], build_time: float, - *, - build_base_images: bool, ): self.library = library self.weblog = weblog @@ -224,9 +222,6 @@ def __init__( # split mechanism self.build_time = build_time - self.build_base_images = build_base_images - """ Shall the end-to-end scenario rebuild the weblog base image for fully baked weblog """ - def serialize(self) -> dict: return { "runs_on": "ubuntu-latest", @@ -237,9 +232,10 @@ def serialize(self) -> dict: "scenarios": sorted(self.scenarios), "expected_job_time": self.expected_job_time + self.build_time, "binaries_artifact": self.weblog.artifact_name, + # only local weblogs build their base image inline; prebuild weblogs wait for it + # once in the dedicated build_end_to_end job (see _get_weblog_build_job). "build_weblog_base_image": self.weblog.build_mode == BuildMode.local - and self.build_base_images - and self.weblog.base_dockerfile is not None, + and self.weblog.base_image_tag is not None, } @property @@ -275,7 +271,6 @@ def split_for_parallel_execution(self, desired_execution_time: float) -> list["J weblog_instance=i + 1, scenarios_times={scenario: self._scenarios_times[scenario] for scenario in scenarios}, build_time=self.build_time, - build_base_images=self.build_base_images, ) ) @@ -314,8 +309,6 @@ def get_endtoend_definitions( maximum_parallel_jobs: int, unique_id: str, binaries_artifact: str, - *, - build_base_images: bool = False, ) -> dict: scenarios = scenario_map.get("endtoend", []) @@ -353,7 +346,6 @@ def get_endtoend_definitions( weblog_instance=1, scenarios_times=scenarios_times, build_time=_get_build_time(library, weblog, time_stats["build"]), - build_base_images=build_base_images, ) ) @@ -372,20 +364,18 @@ def get_endtoend_definitions( "endtoend_defs": { "parallel_enable": len(jobs) > 0, "parallel_weblogs": [ - _get_weblog_build_job(weblog, build_base_images=build_base_images) - for weblog in weblogs - if weblog.build_mode == BuildMode.prebuild + _get_weblog_build_job(weblog) for weblog in weblogs if weblog.build_mode == BuildMode.prebuild ], "parallel_jobs": [job.serialize() for job in jobs], } } -def _get_weblog_build_job(weblog: Weblog, *, build_base_images: bool) -> dict: +def _get_weblog_build_job(weblog: Weblog) -> dict: return { "name": weblog.name, "artifact_name": weblog.artifact_name, - "build_base_images": build_base_images and weblog.base_dockerfile is not None, + "build_weblog_base_image": weblog.base_image_tag is not None, } diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index c004f0621db..07836492c02 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,7 +43,6 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, - build_weblog_base_images: bool = False, ): # this data struture is a dict where: # the key is the workflow identifier @@ -88,7 +87,6 @@ def __init__( maximum_parallel_jobs=256, unique_id=self.unique_id, binaries_artifact=self.binaries_artifact, - build_base_images=build_weblog_base_images, ) self.data["parametric"] = { @@ -288,12 +286,6 @@ def _get_workflow_map( "--system-tests-dev-mode", type=str, help="true if running in system-tests CI, with the dev mode", default="" ) parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) - parser.add_argument( - "--build-weblog-base-images", - type=str, - help="Rebuild weblog base images", - default="", - ) args = parser.parse_args() @@ -313,5 +305,4 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, - build_weblog_base_images=args.build_weblog_base_images == "true", ).export(export_format=args.format, output=args.output) diff --git a/utils/scripts/wait_for_base_image.py b/utils/scripts/wait_for_base_image.py new file mode 100755 index 00000000000..6aeb658451e --- /dev/null +++ b/utils/scripts/wait_for_base_image.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Wait for a weblog's base image to be available on Docker Hub. + +Weblog base images (e.g. `datadog/system-tests:express4.base-`) are built and +pushed by a dedicated GitLab CI job (see `utils/scripts/build_base_images.py`), not by +GitHub Actions. There is no direct dependency mechanism between the two CI systems, so +this script simply polls `docker manifest inspect` for the tag declared in the weblog's +Dockerfile until it appears, or a timeout is reached. + +It never builds or pushes anything: updating the tag in the weblog's Dockerfile after a +base image dependency changes remains the contributor's responsibility (see +docs/edit/update-docker-images.md). + +Deliberately depends only on the standard library (no `utils` package, no venv) so it +can run as a plain CI step before the runner virtualenv is built. +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + + +def _base_image_tag(library: str, weblog: str) -> str | None: + """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" + dockerfile = Path(f"utils/build/docker/{library}/{weblog}.Dockerfile") + if not dockerfile.exists(): + print(f"Error: no Dockerfile found for weblog '{weblog}' in library '{library}'") + sys.exit(1) + + for line in dockerfile.read_text().splitlines(): + if line.startswith("FROM "): + image_name = line.split()[1] + return image_name if image_name.startswith("datadog/system-tests:") else None + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description="Wait for a weblog's base image to exist on Docker Hub") + parser.add_argument("library", help="Library name (e.g. nodejs, python)") + parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") + parser.add_argument("--timeout", type=int, default=900, help="Max time to wait, in seconds (default: 900)") + parser.add_argument("--poll-interval", type=int, default=30, help="Time between polls, in seconds (default: 30)") + args = parser.parse_args() + + image_tag = _base_image_tag(args.library, args.weblog) + + if image_tag is None: + print(f"{args.weblog} does not use a system-tests base image, nothing to wait for") + return + + print(f"Waiting for {image_tag} to be available on Docker Hub (timeout: {args.timeout}s)") + + deadline = time.monotonic() + args.timeout + while True: + result = subprocess.run( + ["docker", "manifest", "inspect", image_tag], + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + print(f"{image_tag} is available") + return + + if time.monotonic() >= deadline: + print(f"Error: timed out waiting for {image_tag}") + if result.stderr: + print(result.stderr.strip()) + sys.exit(1) + + print(f"{image_tag} not found yet, retrying in {args.poll_interval}s...") + time.sleep(args.poll_interval) + + +if __name__ == "__main__": + main() From ee95888ed65a6cca8bbce0c1d59066c8fc950be5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Jul 2026 17:49:39 +0200 Subject: [PATCH 8/9] Building from a temporary context dir --- .gitignore | 3 + docs/edit/update-docker-images.md | 16 +- docs/understand/weblogs/weblog-metadata.md | 47 +- tests/test_the_test/test_build_base_images.py | 487 ++++++++++++++++++ utils/_context/weblog_metadata.py | 17 - utils/build/docker/nodejs/docker-bake.hcl | 20 +- .../express4-typescript.base.Dockerfile | 9 +- .../docker/nodejs/express4.base.Dockerfile | 9 +- .../docker/nodejs/express5.base.Dockerfile | 9 +- .../docker/nodejs/fastify.base.Dockerfile | 22 +- .../docker/nodejs/nextjs.base.Dockerfile | 9 +- utils/build/docker/nodejs/weblog_metadata.yml | 23 - .../docker/php/apache-mod.base.Dockerfile | 10 +- utils/build/docker/php/docker-bake.hcl | 70 ++- .../build/docker/php/php-fpm.base.Dockerfile | 10 +- utils/build/docker/php/weblog_metadata.yml | 113 ---- .../docker/python/django-poc.base.Dockerfile | 4 +- .../python/django-py3.13.base.Dockerfile | 4 +- utils/build/docker/python/docker-bake.hcl | 36 +- .../docker/python/fastapi.base.Dockerfile | 4 +- .../docker/python/flask-poc.base.Dockerfile | 4 +- .../docker/python/python3.12.base.Dockerfile | 4 +- .../docker/python/tornado.base.Dockerfile | 4 +- .../docker/python/uwsgi-poc.base.Dockerfile | 4 +- utils/build/docker/python/weblog_metadata.yml | 18 - utils/scripts/build_base_images.py | 349 +++++++++++-- 26 files changed, 964 insertions(+), 341 deletions(-) create mode 100644 tests/test_the_test/test_build_base_images.py diff --git a/.gitignore b/.gitignore index b8cc70f86c0..51e07b11d40 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,9 @@ version-check-output/ tmp/ +# Materialized build context for utils/scripts/build_base_images.py +/.base_image_build/ + # Go go.work* diff --git a/docs/edit/update-docker-images.md b/docs/edit/update-docker-images.md index 6ebe066acf2..cc31659f7ba 100644 --- a/docs/edit/update-docker-images.md +++ b/docs/edit/update-docker-images.md @@ -3,13 +3,15 @@ Some of images used in system-tests are prebuild and used threw [hub.docker.com/ For weblog base images (nodejs, python, php), the build and push are fully automated: 1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every - branch. For each `docker-bake.hcl` target declared in a library's `weblog_metadata.yml` - `base_image_dependencies` section (see `docs/understand/weblogs/weblog-metadata.md`), it computes a - content hash of the target's dependencies and, if a base image tagged with that hash doesn't already - exist on Docker Hub, builds and pushes it as `-`. -2. If you changed a file listed in `base_image_dependencies`, a new tag will automatically be pushed by - that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to point to the new tag (you can - find it by running `python utils/scripts/build_base_images.py --dry-run` locally). + branch. For each target in a library's `docker-bake.hcl`, it derives the target's dependencies from the + `COPY` instructions in its `.base.Dockerfile` (see `docs/understand/weblogs/weblog-metadata.md` + for the Dockerfile rules this relies on), computes a content hash of those dependencies, and, if a base + image tagged with that hash doesn't already exist on Docker Hub, builds and pushes it as + `-`. +2. If you changed a file referenced by a `.base.Dockerfile`'s `COPY` instructions, a new tag will + automatically be pushed by that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to + point to the new tag (you can find it by running `python utils/scripts/build_base_images.py --dry-run` + locally). 3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for the tag referenced in the weblog's `FROM` line to become available before building the weblog, via `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md index 2359444cf3b..c07f8b767b4 100644 --- a/docs/understand/weblogs/weblog-metadata.md +++ b/docs/understand/weblogs/weblog-metadata.md @@ -48,23 +48,36 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`. ## Base image dependencies -A `weblog_metadata.yml` may also declare a top-level `base_image_dependencies` section, unrelated -to the per-weblog entries above, used by the `build_base_images` CI job -(`utils/scripts/build_base_images.py`) to know when a weblog base image needs to be rebuilt: - -```yaml -base_image_dependencies: - : - - - - ... -``` - -For each target listed there, the job computes a content hash from the resolved -`docker-bake.hcl` target config, the target's Dockerfile, and every git-tracked file under the -listed paths, then pushes the base image to Docker Hub tagged `-` if that tag -doesn't already exist. It never overwrites an existing tag, so weblog Dockerfiles that `FROM` a -base image must have their tag updated by hand after a new one is pushed (run the script with -`--dry-run` to find the current tag for each target). +Base images (built by the `build_base_images` CI job, `utils/scripts/build_base_images.py`) are +declared in each library's `utils/build/docker//docker-bake.hcl`, one target per base +image. There is no separate dependency list to maintain: for each target, the job parses the +target's own `.base.Dockerfile` and treats every `COPY` source as a dependency. This works +because base Dockerfiles are required to follow a few rules that make that derivation +unambiguous: + +- No `ADD` — use `COPY` for everything (no glob sources, no whole-directory copies, no remote + URLs). +- Every `COPY` has exactly one source: `COPY [flags] `. +- The bake target's `context` is always the Dockerfile's own directory, so every `COPY` source + is a plain path relative to that directory. +- No `RUN --mount` — a bind/cache/secret mount reads from a path the script can't see, so it + would silently escape the derived dependency list. + +(`COPY --from=` is unaffected: it isn't a local repository path, so it's skipped.) + +The job computes a content hash from the resolved `docker-bake.hcl` target config, the target's +Dockerfile, and every git-tracked file under each derived dependency path, then pushes the base +image to Docker Hub tagged `-` if that tag doesn't already exist. It never +overwrites an existing tag, so weblog Dockerfiles that `FROM` a base image must have their tag +updated by hand after a new one is pushed (run the script with `--dry-run` to find the current +tag for each target). + +As a safety net, before building, every derived dependency is hardlinked (or copied, if +hardlinking isn't possible) into an isolated build context under `.base_image_build/`, and the +image is built from that directory instead of the real one. This way, if the Dockerfile +references a file the parser failed to recognize as a dependency, the build fails loudly +("file not found") instead of silently succeeding against the full checkout — which would leave +the tag's content hash stale without anyone noticing. GitHub Actions never builds these base images itself: `utils/scripts/wait_for_base_image.py` polls Docker Hub for the tag currently referenced in the weblog's `FROM` line (with a timeout) diff --git a/tests/test_the_test/test_build_base_images.py b/tests/test_the_test/test_build_base_images.py new file mode 100644 index 00000000000..7fa984431ec --- /dev/null +++ b/tests/test_the_test/test_build_base_images.py @@ -0,0 +1,487 @@ +from pathlib import Path +import subprocess +import tempfile +import textwrap + +import pytest + +from utils import scenarios +from utils.scripts import build_base_images +from utils.scripts.build_base_images import ( + _dependency_paths, + _files_under, + compute_hash, + materialize_build_context, + parse_copy_dependencies, +) + + +def _write_dockerfile(tmp_path: Path, content: str) -> Path: + dockerfile = tmp_path / "some.base.Dockerfile" + dockerfile.write_text(textwrap.dedent(content)) + return dockerfile + + +@scenarios.test_the_test +class Test_ParseCopyDependencies: + """Unit tests for utils.scripts.build_base_images.parse_copy_dependencies, which derives + a base image's dependencies from the `COPY` instructions of its Dockerfile. + """ + + def test_single_copy(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_multiple_copy_instructions(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express /usr/app + COPY express4/package.json ./ + COPY express4/bun.lock ./ + COPY nft-prune.mjs ./ + """, + ) + assert parse_copy_dependencies(dockerfile) == [ + "express", + "express4/package.json", + "express4/bun.lock", + "nft-prune.mjs", + ] + + def test_copy_with_flags_is_parsed(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY --chown=node:node app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_copy_from_external_image_is_skipped(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY --from=oven/bun:1.3.13-alpine /usr/local/bin/bun /usr/local/bin/bun + COPY app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_copy_from_build_stage_is_skipped(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM golang:1.22 AS build + RUN echo build + + FROM alpine + COPY --from=build /app/weblog /app/weblog + COPY app.sh . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.sh"] + + def test_line_continuation_is_joined(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY \\ + app.js \\ + . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_lowercase_instruction_is_parsed(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + copy app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_comments_and_blank_lines_are_ignored(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + + # This is a comment + COPY app.js . + + # docker build ... + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_run_without_mount_is_allowed(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js . + RUN node --version + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_no_copy_instructions_returns_empty_list(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + RUN node --version + """, + ) + assert parse_copy_dependencies(dockerfile) == [] + + def test_add_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + ADD app.js . + """, + ) + with pytest.raises(ValueError, match="ADD is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_add_with_glob_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + ADD binaries* /binaries/ + """, + ) + with pytest.raises(ValueError, match="ADD is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_run_mount_bind_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM golang:1.22 + RUN --mount=type=bind,source=.,target=/src \\ + go build ./... + """, + ) + with pytest.raises(ValueError, match="RUN --mount is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_run_mount_cache_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM python:3.11 + RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt + """, + ) + with pytest.raises(ValueError, match="RUN --mount is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_multi_source_copy_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js dsm.js rasp.js ./ + """, + ) + with pytest.raises(ValueError, match="exactly one source"): + parse_copy_dependencies(dockerfile) + + def test_copy_with_no_destination_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js + """, + ) + with pytest.raises(ValueError, match="exactly one source"): + parse_copy_dependencies(dockerfile) + + +@scenarios.test_the_test +class Test_DependencyPaths: + """Unit tests for utils.scripts.build_base_images._dependency_paths, which resolves each + COPY source parsed by parse_copy_dependencies() against `context_root` (the Dockerfile's own + directory), expanding glob wildcards along the way (see the module docstring, "Wildcard + sources"), then flattens each match to its constituent non-gitignored files via + `_files_under` (see Test_FilesUnder). Every returned path is relative to `context_root`, + which is also the only boundary a COPY source is allowed to stay within (see the "escapes" + test below) — REPO_ROOT plays no part in this resolution, so these tests don't need to touch + it at all. + """ + + @pytest.fixture(autouse=True) + def _git_repo(self, tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + + def test_literal_source_resolves_to_itself(self, tmp_path: Path): + (tmp_path / "app.js").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js . + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [Path("app.js")] + + def test_literal_directory_source_flattens_to_its_files(self, tmp_path: Path): + (tmp_path / "express").mkdir() + (tmp_path / "express" / "app.js").write_text("") + (tmp_path / "express" / "auth.js").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express /usr/app + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [ + Path("express/app.js"), + Path("express/auth.js"), + ] + + def test_wildcard_source_expands_to_every_match(self, tmp_path: Path): + (tmp_path / "package.json").write_text("") + (tmp_path / "package-lock.json").write_text("") + (tmp_path / "app.js").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY *.json . + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [ + Path("package-lock.json"), + Path("package.json"), + ] + + def test_overlapping_sources_are_deduplicated(self, tmp_path: Path): + # A directory dependency and one of its own files listed separately (as in + # `COPY express4-typescript /usr/app` + `COPY express4-typescript/package.json ./` in a + # real base Dockerfile) overlap: the file must only appear once in the final list. + (tmp_path / "express").mkdir() + (tmp_path / "express" / "app.js").write_text("") + (tmp_path / "express" / "package.json").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express /usr/app + COPY express/package.json ./ + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [ + Path("express/app.js"), + Path("express/package.json"), + ] + + def test_wildcard_source_matching_nothing_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY *.json . + """, + ) + with pytest.raises(ValueError, match="matched no files"): + _dependency_paths(tmp_path, dockerfile) + + def test_literal_source_matching_nothing_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY does-not-exist.txt . + """, + ) + with pytest.raises(ValueError, match="matched no files"): + _dependency_paths(tmp_path, dockerfile) + + def test_source_escaping_context_root_is_rejected(self, tmp_path: Path): + # A COPY source that reaches into a sibling library's directory (e.g. a nodejs base + # Dockerfile depending on something under utils/build/docker/python/) must be rejected, + # regardless of whether the resolved path stays inside the repository or not. + context_root = tmp_path / "nodejs" + sibling_dir = tmp_path / "python" + context_root.mkdir() + sibling_dir.mkdir() + (sibling_dir / "weblog_metadata.yml").write_text("") + + dockerfile = _write_dockerfile( + context_root, + """\ + FROM node:18-alpine + COPY ../python/weblog_metadata.yml . + """, + ) + with pytest.raises(ValueError, match="escapes the Dockerfile's context directory"): + _dependency_paths(context_root, dockerfile) + + +@scenarios.test_the_test +class Test_FilesUnder: + """Unit tests for utils.scripts.build_base_images._files_under, which lists the files under + a dependency path that should actually be hashed/hardlinked: every git-tracked file, plus + every untracked-but-not-gitignored file, but never anything gitignored (e.g. a locally + installed, never-committed node_modules/ sitting under a dependency directory). Filtering is + based purely on .gitignore rules, not on whether a file is committed or staged. + """ + + @pytest.fixture(autouse=True) + def _git_repo(self, tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + + def test_tracked_file_is_included(self, tmp_path: Path): + (tmp_path / "app.js").write_text("") + subprocess.run(["git", "add", "app.js"], cwd=tmp_path, check=True) + + assert _files_under(tmp_path, Path("app.js")) == [Path("app.js")] + + def test_untracked_but_not_gitignored_file_is_included(self, tmp_path: Path): + # Never `git add`-ed, and not covered by any .gitignore rule: this script must not + # require content to be committed, or even staged, to be picked up. + (tmp_path / "app.js").write_text("") + + assert _files_under(tmp_path, Path("app.js")) == [Path("app.js")] + + def test_gitignored_file_under_a_directory_dependency_is_excluded(self, tmp_path: Path): + (tmp_path / ".gitignore").write_text("node_modules/\n") + (tmp_path / "express").mkdir() + (tmp_path / "express" / "app.js").write_text("") + (tmp_path / "express" / "node_modules").mkdir() + (tmp_path / "express" / "node_modules" / "some_dep.js").write_text("") + subprocess.run(["git", "add", "express/app.js", ".gitignore"], cwd=tmp_path, check=True) + + assert _files_under(tmp_path, Path("express")) == [Path("express/app.js")] + + def test_dependency_entirely_gitignored_raises_with_an_explicit_message(self, tmp_path: Path): + (tmp_path / ".gitignore").write_text("secret.txt\n") + (tmp_path / "secret.txt").write_text("") + + with pytest.raises(ValueError, match="every file under it is gitignored"): + _files_under(tmp_path, Path("secret.txt")) + + def test_directory_dependency_entirely_gitignored_raises_with_an_explicit_message(self, tmp_path: Path): + (tmp_path / ".gitignore").write_text("ignored_dir/\n") + (tmp_path / "ignored_dir").mkdir() + (tmp_path / "ignored_dir" / "x.txt").write_text("") + + with pytest.raises(ValueError, match="every file under it is gitignored"): + _files_under(tmp_path, Path("ignored_dir")) + + def test_outside_a_git_repository_raises(self): + # This test needs a directory that is guaranteed not to be inside any git repository + # (unlike tmp_path, which _git_repo's autouse fixture always `git init`s), so it uses a + # fully independent temporary directory instead. + with tempfile.TemporaryDirectory() as outside_any_repo_str: + outside_any_repo = Path(outside_any_repo_str) + (outside_any_repo / "app.js").write_text("") + + with pytest.raises(subprocess.CalledProcessError): + _files_under(outside_any_repo, Path("app.js")) + + +@scenarios.test_the_test +class Test_MaterializeBuildContext: + """Unit tests for utils.scripts.build_base_images.materialize_build_context, which + hardlinks every dependency (plus the Dockerfile itself) into an isolated build directory. + """ + + @pytest.fixture(autouse=True) + def _git_repo(self, tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + + def test_overlapping_dependencies_are_deduplicated(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + # Regression test: a directory dependency and one of its own files listed separately as + # a second dependency (as in `COPY express4-typescript /usr/app` + + # `COPY express4-typescript/package.json ./` in a real base Dockerfile) overlap, so the + # same source file would otherwise be hardlinked into the same destination twice. + monkeypatch.setattr(build_base_images, "BUILD_CONTEXT_ROOT", tmp_path / "build") + + (tmp_path / "express4-typescript").mkdir() + (tmp_path / "express4-typescript" / "package.json").write_text("{}") + (tmp_path / "express4-typescript" / "app.ts").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express4-typescript /usr/app + COPY express4-typescript/package.json ./ + """, + ) + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + + dependencies = _dependency_paths(tmp_path, dockerfile) + build_dir = materialize_build_context("somelib", "sometarget", tmp_path, dockerfile, dependencies) + + materialized = sorted(p.relative_to(build_dir) for p in build_dir.rglob("*") if p.is_file()) + assert materialized == [ + Path("express4-typescript/app.ts"), + Path("express4-typescript/package.json"), + Path(dockerfile.name), + ] + + +@scenarios.test_the_test +class Test_ComputeHash: + """Unit tests for utils.scripts.build_base_images.compute_hash, which hashes the resolved + bake config (minus tags) together with the path and content of every file in an already + materialized build directory (see materialize_build_context) — not a separate read of the + dependency paths, so there is nothing git- or context_root-related to set up here. + """ + + def test_same_directory_and_config_produce_the_same_hash(self, tmp_path: Path): + (tmp_path / "app.js").write_text("console.log(1);") + bake_config = {"context": ".", "dockerfile": "x.base.Dockerfile", "tags": ["x"]} + + assert compute_hash(tmp_path, bake_config) == compute_hash(tmp_path, bake_config) + + def test_different_file_content_changes_the_hash(self, tmp_path_factory: pytest.TempPathFactory): + dir_a = tmp_path_factory.mktemp("a") + dir_b = tmp_path_factory.mktemp("b") + (dir_a / "app.js").write_text("console.log(1);") + (dir_b / "app.js").write_text("console.log(2);") + bake_config = {"tags": ["x"]} + + assert compute_hash(dir_a, bake_config) != compute_hash(dir_b, bake_config) + + def test_different_file_name_changes_the_hash(self, tmp_path_factory: pytest.TempPathFactory): + dir_a = tmp_path_factory.mktemp("a") + dir_b = tmp_path_factory.mktemp("b") + (dir_a / "app.js").write_text("same content") + (dir_b / "index.js").write_text("same content") + bake_config = {"tags": ["x"]} + + assert compute_hash(dir_a, bake_config) != compute_hash(dir_b, bake_config) + + def test_different_bake_config_changes_the_hash(self, tmp_path: Path): + (tmp_path / "app.js").write_text("console.log(1);") + + assert compute_hash(tmp_path, {"args": {"PHP_VERSION": "8.0"}, "tags": ["x"]}) != compute_hash( + tmp_path, {"args": {"PHP_VERSION": "8.1"}, "tags": ["x"]} + ) + + def test_tags_are_excluded_from_the_hash(self, tmp_path: Path): + (tmp_path / "app.js").write_text("console.log(1);") + + assert compute_hash(tmp_path, {"tags": ["x"]}) == compute_hash(tmp_path, {"tags": ["y"]}) diff --git a/utils/_context/weblog_metadata.py b/utils/_context/weblog_metadata.py index d4e45859579..f8881148f6c 100644 --- a/utils/_context/weblog_metadata.py +++ b/utils/_context/weblog_metadata.py @@ -54,25 +54,8 @@ def _load_explicit_metadata(library: str) -> dict[str, "WeblogMetaData"]: with path.open() as f: data: dict = yaml.safe_load(f) or {} - data.pop("base_image_dependencies", None) - return {name: WeblogMetaData(name=name, library=library, **kwargs) for name, kwargs in data.items()} - @staticmethod - def load_base_image_dependencies(library: str) -> dict[str, list[str]]: - """Returns the `base_image_dependencies` section of weblog_metadata.yml: a mapping of - docker-bake.hcl target name to the list of paths (files or directories) that base image - depends on, used to compute a content-hash tag for the base image build job. - """ - path = Path(f"utils/build/docker/{library}/weblog_metadata.yml") - if not path.exists(): - return {} - - with path.open() as f: - data: dict = yaml.safe_load(f) or {} - - return data.get("base_image_dependencies", {}) or {} - @staticmethod def load(library: str) -> list["WeblogMetaData"]: metadata = WeblogMetaData._load_explicit_metadata(library) diff --git a/utils/build/docker/nodejs/docker-bake.hcl b/utils/build/docker/nodejs/docker-bake.hcl index 45c1d071caf..5a46688c07b 100644 --- a/utils/build/docker/nodejs/docker-bake.hcl +++ b/utils/build/docker/nodejs/docker-bake.hcl @@ -1,4 +1,8 @@ # Docker Buildx bake file for Node.js base images +# +# `context` is always this directory: base image Dockerfiles only COPY files from +# here, so paths in the Dockerfile are relative to it (see build_base_images.py, +# which derives base_image_dependencies from these COPY instructions). group "default" { targets = [ @@ -11,40 +15,36 @@ group "default" { } target "_common" { + context = "." provenance = false } target "express4" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/express4.base.Dockerfile" + dockerfile = "express4.base.Dockerfile" tags = ["datadog/system-tests:express4.base"] } target "express5" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/express5.base.Dockerfile" + dockerfile = "express5.base.Dockerfile" tags = ["datadog/system-tests:express5.base"] } target "fastify" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/fastify.base.Dockerfile" + dockerfile = "fastify.base.Dockerfile" tags = ["datadog/system-tests:fastify.base"] } target "express4-typescript" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/express4-typescript.base.Dockerfile" + dockerfile = "express4-typescript.base.Dockerfile" tags = ["datadog/system-tests:express4-typescript.base"] } target "nextjs" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/nextjs.base.Dockerfile" + dockerfile = "nextjs.base.Dockerfile" tags = ["datadog/system-tests:nextjs.base"] } diff --git a/utils/build/docker/nodejs/express4-typescript.base.Dockerfile b/utils/build/docker/nodejs/express4-typescript.base.Dockerfile index 2396c0ca98a..dbae3fde995 100644 --- a/utils/build/docker/nodejs/express4-typescript.base.Dockerfile +++ b/utils/build/docker/nodejs/express4-typescript.base.Dockerfile @@ -8,12 +8,13 @@ RUN node --version && npm --version && bun --version && curl --version WORKDIR /usr/app -COPY utils/build/docker/nodejs/express4-typescript /usr/app -COPY utils/build/docker/nodejs/express4-typescript/package.json utils/build/docker/nodejs/express4-typescript/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY express4-typescript /usr/app +COPY express4-typescript/package.json ./ +COPY express4-typescript/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs --keep-types app.ts node_modules/typescript/bin/tsc \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/express4-typescript.base.Dockerfile -t datadog/system-tests:express4-typescript.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/express4-typescript.base.Dockerfile -t datadog/system-tests:express4-typescript.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:express4-typescript.base-v3 diff --git a/utils/build/docker/nodejs/express4.base.Dockerfile b/utils/build/docker/nodejs/express4.base.Dockerfile index 8c24260b4e5..3d9003ecfad 100644 --- a/utils/build/docker/nodejs/express4.base.Dockerfile +++ b/utils/build/docker/nodejs/express4.base.Dockerfile @@ -10,12 +10,13 @@ WORKDIR /usr/app ENV NODE_ENV=production -COPY utils/build/docker/nodejs/express /usr/app -COPY utils/build/docker/nodejs/express4/package.json utils/build/docker/nodejs/express4/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY express /usr/app +COPY express4/package.json ./ +COPY express4/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs app.js \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/express4.base.Dockerfile -t datadog/system-tests:express4.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/express4.base.Dockerfile -t datadog/system-tests:express4.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:express4.base-v3 diff --git a/utils/build/docker/nodejs/express5.base.Dockerfile b/utils/build/docker/nodejs/express5.base.Dockerfile index 9eee044cbab..cd90f59d5d9 100644 --- a/utils/build/docker/nodejs/express5.base.Dockerfile +++ b/utils/build/docker/nodejs/express5.base.Dockerfile @@ -10,12 +10,13 @@ WORKDIR /usr/app ENV NODE_ENV=production -COPY utils/build/docker/nodejs/express /usr/app -COPY utils/build/docker/nodejs/express5/package.json utils/build/docker/nodejs/express5/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY express /usr/app +COPY express5/package.json ./ +COPY express5/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs app.js \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/express5.base.Dockerfile -t datadog/system-tests:express5.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/express5.base.Dockerfile -t datadog/system-tests:express5.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:express5.base-v3 diff --git a/utils/build/docker/nodejs/fastify.base.Dockerfile b/utils/build/docker/nodejs/fastify.base.Dockerfile index 095430dc098..479ca6f3aa0 100644 --- a/utils/build/docker/nodejs/fastify.base.Dockerfile +++ b/utils/build/docker/nodejs/fastify.base.Dockerfile @@ -10,19 +10,19 @@ WORKDIR /usr/app ENV NODE_ENV=production -COPY utils/build/docker/nodejs/fastify/app.js \ - utils/build/docker/nodejs/fastify/dsm.js \ - utils/build/docker/nodejs/fastify/rasp.js \ - ./ -COPY utils/build/docker/nodejs/fastify/debugger ./debugger -COPY utils/build/docker/nodejs/fastify/iast ./iast -COPY utils/build/docker/nodejs/fastify/integrations ./integrations -COPY utils/build/docker/nodejs/fastify/resources ./resources -COPY utils/build/docker/nodejs/fastify/package.json utils/build/docker/nodejs/fastify/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY fastify/app.js ./ +COPY fastify/dsm.js ./ +COPY fastify/rasp.js ./ +COPY fastify/debugger ./debugger +COPY fastify/iast ./iast +COPY fastify/integrations ./integrations +COPY fastify/resources ./resources +COPY fastify/package.json ./ +COPY fastify/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs app.js \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/fastify.base.Dockerfile -t datadog/system-tests:fastify.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/fastify.base.Dockerfile -t datadog/system-tests:fastify.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:fastify.base-v3 diff --git a/utils/build/docker/nodejs/nextjs.base.Dockerfile b/utils/build/docker/nodejs/nextjs.base.Dockerfile index 7aae00023a5..4a69ff707b0 100644 --- a/utils/build/docker/nodejs/nextjs.base.Dockerfile +++ b/utils/build/docker/nodejs/nextjs.base.Dockerfile @@ -8,9 +8,10 @@ RUN node --version && npm --version && bun --version && curl --version WORKDIR /usr/app -COPY utils/build/docker/nodejs/nextjs/package.json utils/build/docker/nodejs/nextjs/bun.lock ./ -COPY utils/build/docker/nodejs/nextjs /usr/app -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY nextjs/package.json ./ +COPY nextjs/bun.lock ./ +COPY nextjs /usr/app +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && bun run build \ && node nft-prune.mjs \ @@ -24,5 +25,5 @@ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ node_modules/next/dist/compiled/terser \ && rm -rf .next/cache /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/nextjs.base.Dockerfile -t datadog/system-tests:nextjs.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/nextjs.base.Dockerfile -t datadog/system-tests:nextjs.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:nextjs.base-v3 diff --git a/utils/build/docker/nodejs/weblog_metadata.yml b/utils/build/docker/nodejs/weblog_metadata.yml index 4a7da1a482b..802e518616e 100644 --- a/utils/build/docker/nodejs/weblog_metadata.yml +++ b/utils/build/docker/nodejs/weblog_metadata.yml @@ -21,26 +21,3 @@ anthropic-js: google_genai-js: build_mode: none framework_versions: ["1.34.0"] - -# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by -# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. -base_image_dependencies: - express4: - - utils/build/docker/nodejs/express - - utils/build/docker/nodejs/express4/package.json - - utils/build/docker/nodejs/express4/bun.lock - - utils/build/docker/nodejs/nft-prune.mjs - express5: - - utils/build/docker/nodejs/express - - utils/build/docker/nodejs/express5/package.json - - utils/build/docker/nodejs/express5/bun.lock - - utils/build/docker/nodejs/nft-prune.mjs - express4-typescript: - - utils/build/docker/nodejs/express4-typescript - - utils/build/docker/nodejs/nft-prune.mjs - fastify: - - utils/build/docker/nodejs/fastify - - utils/build/docker/nodejs/nft-prune.mjs - nextjs: - - utils/build/docker/nodejs/nextjs - - utils/build/docker/nodejs/nft-prune.mjs diff --git a/utils/build/docker/php/apache-mod.base.Dockerfile b/utils/build/docker/php/apache-mod.base.Dockerfile index 13af5872d0e..fb45008228c 100644 --- a/utils/build/docker/php/apache-mod.base.Dockerfile +++ b/utils/build/docker/php/apache-mod.base.Dockerfile @@ -10,8 +10,11 @@ ENV DD_TRACE_HEADER_TAGS=user-agent EXPOSE 7777/tcp -ADD binaries* /binaries/ -ADD utils/build/docker/php /tmp/php +# build.sh only reads from these three subdirectories (see apache-mod/build.sh); keep +# this list in sync with it, since it drives the base-image content hash. +COPY apache-mod /tmp/php/apache-mod +COPY weblogs /tmp/php/weblogs +COPY common /tmp/php/common RUN chmod +x /tmp/php/apache-mod/build.sh RUN /tmp/php/apache-mod/build.sh @@ -22,3 +25,6 @@ ENTRYPOINT [] RUN echo "#!/bin/bash\ndumb-init /entrypoint.sh" > app.sh RUN chmod +x app.sh CMD [ "./app.sh" ] + +# docker build --progress=plain -f utils/build/docker/php/apache-mod.base.Dockerfile -t datadog/system-tests:apache-mod-8.0.base utils/build/docker/php +# docker push datadog/system-tests:apache-mod-8.0.base diff --git a/utils/build/docker/php/docker-bake.hcl b/utils/build/docker/php/docker-bake.hcl index 47308716f3e..39557ee91f4 100644 --- a/utils/build/docker/php/docker-bake.hcl +++ b/utils/build/docker/php/docker-bake.hcl @@ -32,160 +32,180 @@ group "default" { ] } +target "_common" { + context = "." +} + target "apache-mod-7_0" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-7.0.base"] } target "apache-mod-7_1" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-7.1.base"] } target "apache-mod-7_2" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-7.2.base"] } target "apache-mod-7_3" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-7.3.base"] } target "apache-mod-7_4" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-7.4.base"] } target "apache-mod-8_0" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-8.0.base"] } target "apache-mod-8_1" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-8.1.base"] } target "apache-mod-8_2" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release" } tags = ["datadog/system-tests:apache-mod-8.2.base"] } target "apache-mod-7_0-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-7.0-zts.base"] } target "apache-mod-7_1-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-7.1-zts.base"] } target "apache-mod-7_2-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-7.2-zts.base"] } target "apache-mod-7_3-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-7.3-zts.base"] } target "apache-mod-7_4-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-7.4-zts.base"] } target "apache-mod-8_0-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-8.0-zts.base"] } target "apache-mod-8_1-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-8.1-zts.base"] } target "apache-mod-8_2-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release-zts" } tags = ["datadog/system-tests:apache-mod-8.2-zts.base"] } target "php-fpm-7_0" { - context = "utils/build/docker/php/" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.0" } tags = ["datadog/system-tests:php-fpm-7.0.base"] } target "php-fpm-7_1" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.1" } tags = ["datadog/system-tests:php-fpm-7.1.base"] } target "php-fpm-7_2" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.2" } tags = ["datadog/system-tests:php-fpm-7.2.base"] } target "php-fpm-7_3" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.3" } tags = ["datadog/system-tests:php-fpm-7.3.base"] } target "php-fpm-7_4" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.4" } tags = ["datadog/system-tests:php-fpm-7.4.base"] } target "php-fpm-8_0" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.0" } tags = ["datadog/system-tests:php-fpm-8.0.base"] } target "php-fpm-8_1" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.1" } tags = ["datadog/system-tests:php-fpm-8.1.base"] } target "php-fpm-8_2" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.2" } tags = ["datadog/system-tests:php-fpm-8.2.base"] } target "php-fpm-8_5" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.5" } tags = ["datadog/system-tests:php-fpm-8.5.base"] diff --git a/utils/build/docker/php/php-fpm.base.Dockerfile b/utils/build/docker/php/php-fpm.base.Dockerfile index ac981cc100f..2b79f6f912e 100644 --- a/utils/build/docker/php/php-fpm.base.Dockerfile +++ b/utils/build/docker/php/php-fpm.base.Dockerfile @@ -3,7 +3,12 @@ FROM ubuntu:24.04 ARG PHP_VERSION ENV PHP_VERSION=${PHP_VERSION} -ADD . /tmp/php +# build.sh only reads from these four subdirectories (see php-fpm/build.sh); keep this +# list in sync with it, since it drives the base-image content hash. +COPY apt-sources.d /tmp/php/apt-sources.d +COPY weblogs /tmp/php/weblogs +COPY php-fpm /tmp/php/php-fpm +COPY common /tmp/php/common RUN chmod +x /tmp/php/php-fpm/build.sh RUN /tmp/php/php-fpm/build.sh $PHP_VERSION @@ -13,3 +18,6 @@ ENV DD_TRACE_ENABLED=1 ENV DD_TRACE_GENERATE_ROOT_SPAN=1 ENV DD_TRACE_AGENT_FLUSH_AFTER_N_REQUESTS=0 ENV DD_TRACE_HEADER_TAGS=user-agent + +# docker build --progress=plain --build-arg PHP_VERSION=8.2 -f utils/build/docker/php/php-fpm.base.Dockerfile -t datadog/system-tests:php-fpm-8.2.base utils/build/docker/php +# docker push datadog/system-tests:php-fpm-8.2.base diff --git a/utils/build/docker/php/weblog_metadata.yml b/utils/build/docker/php/weblog_metadata.yml index 91ffc08aa5d..69bb692b872 100644 --- a/utils/build/docker/php/weblog_metadata.yml +++ b/utils/build/docker/php/weblog_metadata.yml @@ -1,114 +1 @@ # docs/understand/weblogs/weblog-metadata.md - -# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by -# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. -base_image_dependencies: - apache-mod-7_0: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_1: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_2: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_3: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_4: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-8_0: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-8_1: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-8_2: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_0-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_1-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_2-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_3-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-7_4-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-8_0-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-8_1-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - apache-mod-8_2-zts: - - utils/build/docker/php/apache-mod - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-7_0: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-7_1: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-7_2: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-7_3: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-7_4: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-8_0: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-8_1: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-8_2: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common - php-fpm-8_5: - - utils/build/docker/php/php-fpm - - utils/build/docker/php/apt-sources.d - - utils/build/docker/php/weblogs - - utils/build/docker/php/common diff --git a/utils/build/docker/python/django-poc.base.Dockerfile b/utils/build/docker/python/django-poc.base.Dockerfile index e2a360ce3aa..3c7a7936958 100644 --- a/utils/build/docker/python/django-poc.base.Dockerfile +++ b/utils/build/docker/python/django-poc.base.Dockerfile @@ -8,8 +8,8 @@ RUN python --version && curl --version # install python deps ENV PIP_ROOT_USER_ACTION=ignore -COPY utils/build/docker/python/django/requirements-django-poc.txt /tmp/django-requirements.txt +COPY django/requirements-django-poc.txt /tmp/django-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/django-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v11 . +# docker build --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v11 utils/build/docker/python # docker push datadog/system-tests:django-poc.base-v11 diff --git a/utils/build/docker/python/django-py3.13.base.Dockerfile b/utils/build/docker/python/django-py3.13.base.Dockerfile index 47a08691905..a077a7a06b9 100644 --- a/utils/build/docker/python/django-py3.13.base.Dockerfile +++ b/utils/build/docker/python/django-py3.13.base.Dockerfile @@ -7,8 +7,8 @@ RUN apt-get update && apt-get install -y curl RUN python --version && curl --version # install python deps ENV PIP_ROOT_USER_ACTION=ignore -COPY utils/build/docker/python/django/requirements-django-py3.13.txt /tmp/django-requirements.txt +COPY django/requirements-django-py3.13.txt /tmp/django-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/django-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/django-py3.13.base.Dockerfile -t datadog/system-tests:django-py3.13.base-v10 . +# docker build --progress=plain -f utils/build/docker/python/django-py3.13.base.Dockerfile -t datadog/system-tests:django-py3.13.base-v10 utils/build/docker/python # docker push datadog/system-tests:django-py3.13.base-v10 diff --git a/utils/build/docker/python/docker-bake.hcl b/utils/build/docker/python/docker-bake.hcl index 0cf888f82d1..418d625e204 100644 --- a/utils/build/docker/python/docker-bake.hcl +++ b/utils/build/docker/python/docker-bake.hcl @@ -1,4 +1,8 @@ # Docker Buildx bake file for python base images +# +# `context` is always this directory: base image Dockerfiles only COPY files from +# here, so paths in the Dockerfile are relative to it (see build_base_images.py, +# which derives base_image_dependencies from these COPY instructions). group "default" { targets = [ @@ -12,44 +16,48 @@ group "default" { ] } +target "_common" { + context = "." +} + target "django-py3_13" { - context = "." - dockerfile = "utils/build/docker/python/django-py3.13.base.Dockerfile" + inherits = ["_common"] + dockerfile = "django-py3.13.base.Dockerfile" tags = ["datadog/system-tests:django-py3.13.base"] } target "fastapi" { - context = "." - dockerfile = "utils/build/docker/python/fastapi.base.Dockerfile" + inherits = ["_common"] + dockerfile = "fastapi.base.Dockerfile" tags = ["datadog/system-tests:fastapi.base"] } target "python3_12" { - context = "." - dockerfile = "utils/build/docker/python/python3.12.base.Dockerfile" + inherits = ["_common"] + dockerfile = "python3.12.base.Dockerfile" tags = ["datadog/system-tests:python3.12.base"] } target "django-poc" { - context = "." - dockerfile = "utils/build/docker/python/django-poc.base.Dockerfile" + inherits = ["_common"] + dockerfile = "django-poc.base.Dockerfile" tags = ["datadog/system-tests:django-poc.base"] } target "flask-poc" { - context = "." - dockerfile = "utils/build/docker/python/flask-poc.base.Dockerfile" + inherits = ["_common"] + dockerfile = "flask-poc.base.Dockerfile" tags = ["datadog/system-tests:flask-poc.base"] } target "uwsgi-poc" { - context = "." - dockerfile = "utils/build/docker/python/uwsgi-poc.base.Dockerfile" + inherits = ["_common"] + dockerfile = "uwsgi-poc.base.Dockerfile" tags = ["datadog/system-tests:uwsgi-poc.base"] } target "tornado" { - context = "." - dockerfile = "utils/build/docker/python/tornado.base.Dockerfile" + inherits = ["_common"] + dockerfile = "tornado.base.Dockerfile" tags = ["datadog/system-tests:tornado.base"] } diff --git a/utils/build/docker/python/fastapi.base.Dockerfile b/utils/build/docker/python/fastapi.base.Dockerfile index 87701020efd..e676ee7ffba 100644 --- a/utils/build/docker/python/fastapi.base.Dockerfile +++ b/utils/build/docker/python/fastapi.base.Dockerfile @@ -7,11 +7,11 @@ RUN apt-get update && apt-get install -y curl RUN python --version && curl --version # install python deps -COPY utils/build/docker/python/fastapi/requirements-fastapi.txt /tmp/fastapi-requirements.txt +COPY fastapi/requirements-fastapi.txt /tmp/fastapi-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/fastapi-requirements.txt RUN mkdir app WORKDIR /app -# docker build --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v8 . +# docker build --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v8 utils/build/docker/python # docker push datadog/system-tests:fastapi.base-v8 diff --git a/utils/build/docker/python/flask-poc.base.Dockerfile b/utils/build/docker/python/flask-poc.base.Dockerfile index 58dfa951615..1af788d1d77 100644 --- a/utils/build/docker/python/flask-poc.base.Dockerfile +++ b/utils/build/docker/python/flask-poc.base.Dockerfile @@ -18,8 +18,8 @@ RUN apt-get update \ RUN apt update && apt install -y pkg-config default-libmysqlclient-dev pkg-config # install python deps (flask is pinned below because tracer does not support >=2.3.0) -COPY utils/build/docker/python/flask/requirements-flask-poc.txt /tmp/flask-requirements.txt +COPY flask/requirements-flask-poc.txt /tmp/flask-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/flask-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v14 . +# docker build --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v14 utils/build/docker/python # docker push datadog/system-tests:flask-poc.base-v14 diff --git a/utils/build/docker/python/python3.12.base.Dockerfile b/utils/build/docker/python/python3.12.base.Dockerfile index 6c6343ab632..6b72290dc6e 100644 --- a/utils/build/docker/python/python3.12.base.Dockerfile +++ b/utils/build/docker/python/python3.12.base.Dockerfile @@ -8,9 +8,9 @@ RUN python --version && curl --version # install python deps ENV PIP_ROOT_USER_ACTION=ignore -COPY utils/build/docker/python/django/requirements-python3.12.txt /tmp/django-requirements.txt +COPY django/requirements-python3.12.txt /tmp/django-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/django-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v13 . +# docker build --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v13 utils/build/docker/python # docker push datadog/system-tests:python3.12.base-v13 diff --git a/utils/build/docker/python/tornado.base.Dockerfile b/utils/build/docker/python/tornado.base.Dockerfile index 2af2fff6569..d62563746d4 100644 --- a/utils/build/docker/python/tornado.base.Dockerfile +++ b/utils/build/docker/python/tornado.base.Dockerfile @@ -7,11 +7,11 @@ RUN apt-get update && apt-get install -y curl RUN python --version && curl --version # install python deps -COPY utils/build/docker/python/tornado/requirements-tornado.txt /tmp/tornado-requirements.txt +COPY tornado/requirements-tornado.txt /tmp/tornado-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/tornado-requirements.txt RUN mkdir app WORKDIR /app -# docker build --progress=plain -f utils/build/docker/python/tornado.base.Dockerfile -t datadog/system-tests:tornado.base-v1 . +# docker build --progress=plain -f utils/build/docker/python/tornado.base.Dockerfile -t datadog/system-tests:tornado.base-v1 utils/build/docker/python # docker push datadog/system-tests:tornado.base-v1 diff --git a/utils/build/docker/python/uwsgi-poc.base.Dockerfile b/utils/build/docker/python/uwsgi-poc.base.Dockerfile index 9a3739ecc9b..16fbc9ff4df 100644 --- a/utils/build/docker/python/uwsgi-poc.base.Dockerfile +++ b/utils/build/docker/python/uwsgi-poc.base.Dockerfile @@ -11,8 +11,8 @@ RUN apt update && apt install -y pkg-config default-libmysqlclient-dev pkg-confi # install python deps # Tracer does not support flask 2.3.0 or higher, pin the flask version in the requirements file. -COPY utils/build/docker/python/flask/requirements-uwsgi-poc.txt /tmp/flask-requirements.txt +COPY flask/requirements-uwsgi-poc.txt /tmp/flask-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/flask-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v10 . +# docker build --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v10 utils/build/docker/python # docker push datadog/system-tests:uwsgi-poc.base-v10 diff --git a/utils/build/docker/python/weblog_metadata.yml b/utils/build/docker/python/weblog_metadata.yml index c8e71bb0297..c0b67e68fa7 100644 --- a/utils/build/docker/python/weblog_metadata.yml +++ b/utils/build/docker/python/weblog_metadata.yml @@ -9,21 +9,3 @@ anthropic-py: google_genai-py: build_mode: none framework_versions: ["1.55.0"] - -# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by -# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. -base_image_dependencies: - django-poc: - - utils/build/docker/python/django/requirements-django-poc.txt - django-py3_13: - - utils/build/docker/python/django/requirements-django-py3.13.txt - python3_12: - - utils/build/docker/python/django/requirements-python3.12.txt - fastapi: - - utils/build/docker/python/fastapi/requirements-fastapi.txt - flask-poc: - - utils/build/docker/python/flask/requirements-flask-poc.txt - uwsgi-poc: - - utils/build/docker/python/flask/requirements-uwsgi-poc.txt - tornado: - - utils/build/docker/python/tornado/requirements-tornado.txt diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py index 844d58eaebe..ec17760516c 100644 --- a/utils/scripts/build_base_images.py +++ b/utils/scripts/build_base_images.py @@ -4,31 +4,87 @@ python utils/scripts/build_base_images.py -For every library with a `utils/build/docker//weblog_metadata.yml` -declaring a `base_image_dependencies` section, and for every docker-bake.hcl target -listed there: - - 1. Resolve the target's bake config (context/dockerfile/args) via - `docker buildx bake --print`. - 2. Compute a content hash from: the resolved bake config (tags excluded), the - content of the target's Dockerfile, and the content of every git-tracked file - under the paths listed in `base_image_dependencies`. - 3. Take the base tag declared in the bake file (e.g. "datadog/system-tests:express4.base") - and append "-" to get the final tag. - 4. Skip the build if that tag already exists on Docker Hub (`docker manifest inspect`); - otherwise build and push it with that tag. - -This is idempotent and safe to run on every push: it never overwrites an existing -tag, it only creates new ones when the relevant files change. - -Pass --dry-run to only print the computed tag and whether it already exists, -without ever building or pushing (useful to find the tag to put in a weblog -Dockerfile's FROM clause after dependencies change). +For every library with a `utils/build/docker//docker-bake.hcl` file, and for every +target in that file's "default" group: + + 1. Resolve the target's bake config (context/dockerfile/args) via `docker buildx bake --print`. + 2. Parse the target's Dockerfile for its `COPY` instructions to determine which local, + non-gitignored files (relative to the Dockerfile's own directory, which is required to be + the bake target's context — see "Base Dockerfile constraints" below) the image depends on. + 3. Materialize those files, plus the Dockerfile itself, into an isolated build context (see + "Safety net" below) — hardlinking is essentially free, so this always happens next, before + even computing the hash. + 4. Compute a content hash from the resolved bake config (tags excluded) and the path and + content of every file in that materialized build context. + 5. Take the base tag declared in the bake file (e.g. "datadog/system-tests:express4.base") and + append "-" to get the final tag. + 6. Skip the build if that tag already exists on Docker Hub (`docker manifest inspect`); + otherwise build and push the already-materialized build context with that tag. + +This is idempotent and safe to run on every push: it never overwrites an existing tag, it only +creates new ones when the relevant files change. + +Pass --dry-run to only print the computed tag and whether it already exists, without ever +building or pushing (useful to find the tag to put in a weblog Dockerfile's FROM clause after +dependencies change). + +Base Dockerfile constraints +---------------------------- +So that the dependency list above can be derived mechanically and unambiguously from the +Dockerfile alone, every `.base.Dockerfile` built by this script must follow these rules: + + - No `ADD`. Use `COPY` for everything (no whole-context-directory copies, no remote URLs). + `COPY` sources may contain glob wildcards (see "Wildcard sources" below). + - Every `COPY` instruction has exactly one source: `COPY [flags] `. Split + multi-source `COPY` instructions into one `COPY` per source. + - The bake target's `context` is always the Dockerfile's own directory, so every `COPY` + source is a plain path relative to that directory (`COPY app.js .`, not + `COPY utils/build/docker/nodejs/fastify/app.js .`). + - No `RUN --mount`. Bind/cache/secret mounts read from paths this script cannot see, so they + would silently escape the derived dependency list. + +`COPY --from=` is unaffected by these rules: multi-stage copies and copies from +an external image aren't local repository paths, so they're skipped when deriving dependencies +(buildx/buildkit resolves them independently). + +Wildcard sources +----------------- +A `COPY` source is resolved with `Path.glob()` against the Dockerfile's own directory, so a +plain path (no wildcard) and a glob pattern are handled the same way: a plain path simply +matches itself, a pattern is expanded to every file/directory it matches. A pattern matching +zero files raises (almost certainly a typo), but that is the only completeness check available: +a pattern matching *fewer* files than intended does not raise, because Docker's own wildcard +expansion, run again at build time against the materialized build context (see below), only +ever sees the files that same pattern already matched when we computed dependencies — it can't +disagree with a match we already computed by construction. So an incomplete wildcard match +doesn't fail the build the way a missing literal dependency does; it just silently ships an +incomplete image. The materialized build context's file list is printed before every build +specifically to make that failure mode easy to catch by eye. + +Safety net: isolated build context +----------------------------------- +Every detected dependency is hardlinked (falling back to a plain copy if hardlinking isn't +possible, e.g. across filesystems) into `.base_image_build///`, and the image +is built from *that* directory instead of the real one, after printing that directory's file +list. Hardlinking is only a directory-entry operation (no data is duplicated), so this is +essentially free. Its purpose is correctness, not performance: if the Dockerfile references a +file that the parser failed to detect as a dependency, the build fails loudly ("file not found") +instead of silently succeeding against the full repository checkout — which would let an +incomplete dependency list (and thus an under-hashed, stale-looking tag) go unnoticed. (This +guarantee does not extend to an incomplete wildcard match, see above — the printed file list is +the mitigation for that case.) + +The content hash is computed directly from this materialized directory (see `compute_hash`), +not from a second, independent read of the dependency paths: this way "the files the hash was +computed from" and "the files the build can actually see" are the same files by construction, +not two separate computations that are merely supposed to produce the same result. """ import argparse import hashlib import json +import os +import shutil import subprocess import sys from pathlib import Path @@ -38,17 +94,22 @@ REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT)) -from utils._context.weblog_metadata import WeblogMetaData # noqa: E402 from utils.const import COMPONENT_GROUPS # noqa: E402 +BUILD_CONTEXT_ROOT = REPO_ROOT / ".base_image_build" + +# A `COPY [flags] ` line has exactly one source and one destination path +# (see this module's docstring, "Base Dockerfile constraints"). +_SOURCE_AND_DEST_TOKEN_COUNT = 2 + def _bake_file(library: str) -> Path: return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" -def _run(cmd: list[str]) -> subprocess.CompletedProcess: +def _run(cmd: list[str], *, cwd: Path = REPO_ROOT) -> subprocess.CompletedProcess: """Run `cmd`, printing stderr (and stdout, if any) before raising on failure.""" - result = subprocess.run(cmd, cwd=REPO_ROOT, check=False, capture_output=True, text=True) + result = subprocess.run(cmd, cwd=cwd, check=False, capture_output=True, text=True) if result.returncode != 0: print(f"Error: command failed: {' '.join(cmd)}") if result.stdout: @@ -59,41 +120,211 @@ def _run(cmd: list[str]) -> subprocess.CompletedProcess: return result -def _bake_config(bake_file: Path, target: str) -> dict: - """Resolved bake config (context, dockerfile, args, tags) for a single target.""" - result = _run(["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target]) - return json.loads(result.stdout)["target"][target] +def _all_bake_configs(bake_file: Path) -> dict[str, dict]: + """Resolved bake config (context, dockerfile, args, tags) for every target in the bake + file's "default" group. + """ + result = _run(["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), "default"]) + return json.loads(result.stdout)["target"] + + +def _files_under(context_root: Path, path: Path) -> list[Path]: + """Every non-gitignored file under `path` (a file or a directory, relative to + `context_root`), itself relative to `context_root`, sorted. + + This deliberately filters on `.gitignore` rules alone, not on whether a file is committed + or even staged (`--cached --others --exclude-standard` includes both tracked files and + untracked-but-not-ignored ones): this script always runs against a real checkout, tracked + or not, so there is nothing to gain from requiring content to be in the git index, only + the risk of silently skipping a real, intended, not-yet-committed dependency. What must + still be filtered out is gitignored content that happens to sit on disk under a dependency + path (e.g. a locally installed, never-committed `node_modules/`), since that is never meant + to be part of the image's tracked dependency set. + + Raises if `path` exists on disk (so `_dependency_paths` already accepted it as a match) but + every file under it turns out to be gitignored, since that would otherwise silently produce + an empty dependency with no explanation. + """ + result = _run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "--", str(path)], cwd=context_root) + files = sorted(Path(line) for line in result.stdout.splitlines() if line) + if not files: + raise ValueError( + f"{context_root / path}: matched by a COPY source, but every file under it is " + "gitignored (or it is not inside a git repository at all)" + ) + return files + + +def _dockerfile_logical_lines(dockerfile: Path) -> list[str]: + """Dockerfile lines with backslash-continuations joined into one logical line each, and + full-line comments/blank lines dropped. + """ + logical_lines: list[str] = [] + buffer = "" + for raw_line in dockerfile.read_text().splitlines(): + line = raw_line.rstrip() + stripped = line.strip() + if not buffer and (not stripped or stripped.startswith("#")): + continue + if line.endswith("\\"): + buffer += line[:-1] + " " + continue + buffer += line + logical_lines.append(buffer.strip()) + buffer = "" + if buffer: # pragma: no cover - malformed Dockerfile ending on a continuation + logical_lines.append(buffer.strip()) + return logical_lines + + +def parse_copy_dependencies(dockerfile: Path) -> list[str]: + """Local `COPY` source paths (relative to `dockerfile`'s own directory) that a base + Dockerfile depends on. Enforces the constraints listed in this module's docstring: + no `ADD`, no `RUN --mount`, and every `COPY` has exactly one source. + + `COPY --from=` lines are recognized and skipped (not a local repo path). + """ + dependencies: list[str] = [] + for line in _dockerfile_logical_lines(dockerfile): + instruction, _, rest = line.partition(" ") + instruction = instruction.upper() + + if instruction == "ADD": + raise ValueError(f"{dockerfile}: ADD is not allowed in a base Dockerfile, use COPY instead: {line!r}") + + if instruction == "RUN" and "--mount" in rest: + raise ValueError(f"{dockerfile}: RUN --mount is not allowed in a base Dockerfile: {line!r}") + + if instruction != "COPY": + continue + + tokens = rest.split() + flags = [t for t in tokens if t.startswith("--")] + paths = [t for t in tokens if not t.startswith("--")] + + if any(flag.startswith("--from=") for flag in flags): + continue # copies from another build stage or an external image, not a local path + + if len(paths) != _SOURCE_AND_DEST_TOKEN_COUNT: + raise ValueError( + f"{dockerfile}: COPY must be of the form 'COPY [flags] ' " + f"(exactly one source), got: {line!r}" + ) + + source, _dest = paths + dependencies.append(source) + + return dependencies + +def _dependency_paths(context_root: Path, dockerfile: Path) -> list[Path]: + """Every non-gitignored file (relative to `context_root`) that `dockerfile` depends on, + sorted and deduplicated. `context_root` is `dockerfile`'s own directory, per this module's + Dockerfile constraints. This is the single place dependencies are resolved: both + `compute_hash` and `materialize_build_context` operate on exactly this list, so "the files + the hash was computed from" and "the files that get hardlinked" are the same set by + construction, not by two independent computations happening to agree. -def _tracked_files(path: str) -> list[Path]: - """Every git-tracked file under `path` (a file or a directory), sorted.""" - result = _run(["git", "ls-files", path]) - return sorted(REPO_ROOT / line for line in result.stdout.splitlines() if line) + Each COPY source is resolved with `Path.glob()`, so a plain path and a glob pattern are + handled uniformly (a plain path just matches itself). A source matching zero files raises, + since that's almost certainly a typo; see this module's docstring ("Wildcard sources") for + why a pattern matching *fewer* files than intended is not, and cannot be, detected here. + Every match must resolve to a path under `context_root` itself: per this module's Dockerfile + constraints, a base Dockerfile's `COPY` sources are only ever meant to reach within its own + directory (e.g. a `nodejs` base image has no business depending on a file under + `utils/build/docker/python/`), so a source escaping `context_root` is rejected even if it + stays inside the repository. -def compute_hash(bake_config: dict, dependencies: list[str]) -> str: - """Content hash for a base image target: bake config (minus tags) + Dockerfile - content + content of every git-tracked file under `dependencies`. + Each match is then flattened to its constituent files via `_files_under`, which also filters + out gitignored content (e.g. a locally installed, never-committed `node_modules/` sitting + under a dependency directory): two COPY sources can overlap (a directory and one of its own + files listed separately), so the final list is deduplicated. + """ + files: set[Path] = set() + for source in parse_copy_dependencies(dockerfile): + matches = sorted(context_root.glob(source)) + if not matches: + raise ValueError(f"{dockerfile}: COPY source {source!r} matched no files") + + for match in matches: + resolved = match.resolve() + try: + relative = resolved.relative_to(context_root) + except ValueError: + raise ValueError( + f"{dockerfile}: COPY source {source!r} escapes the Dockerfile's context directory ({context_root})" + ) from None + files.update(_files_under(context_root, relative)) + return sorted(files) + + +def compute_hash(build_dir: Path, bake_config: dict) -> str: + """Content hash for a base image target: bake config (minus tags) + path and content of + every file in `build_dir` (the isolated build context already materialized by + `materialize_build_context`, which includes the Dockerfile itself alongside its + dependencies). + + Hashing the materialized directory directly, rather than separately re-reading dependencies + from `context_root`, guarantees "the files the hash was computed from" and "the files the + build can actually see" are the exact same files by construction (same inodes, even), not + two independent reads that are merely supposed to agree. """ digest = hashlib.sha256() config_without_tags = {k: v for k, v in bake_config.items() if k != "tags"} digest.update(json.dumps(config_without_tags, sort_keys=True).encode()) - dockerfile = REPO_ROOT / bake_config["context"] / bake_config["dockerfile"] - digest.update(dockerfile.read_bytes()) - - files: list[Path] = [] - for dep in dependencies: - files.extend(_tracked_files(dep)) - - for file in sorted(set(files)): - digest.update(str(file.relative_to(REPO_ROOT)).encode()) + for file in sorted(p for p in build_dir.rglob("*") if p.is_file()): + digest.update(str(file.relative_to(build_dir)).encode()) digest.update(file.read_bytes()) return digest.hexdigest()[:12] +def _link_or_copy(source: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, dest) + except OSError: + # e.g. source and dest are on different filesystems + shutil.copy2(source, dest) + + +def materialize_build_context( + library: str, target: str, context_root: Path, dockerfile: Path, dependencies: list[Path] +) -> Path: + """Hardlink (falling back to a copy) every file in `dependencies` (relative to + `context_root`, already flattened and deduplicated by `_dependency_paths`), plus `dockerfile` + itself, into a fresh `.base_image_build///` directory, mirroring each + file's path relative to `context_root`. + + Building from this directory instead of the real one is a safety net: it makes "the files + the hash was computed from" and "the files the build can actually see" the same set by + construction, so a dependency the parser failed to detect causes a build failure (missing + file) instead of silently succeeding against the full repository checkout. + + That guarantee doesn't extend to a wildcard COPY source matching fewer files than intended + (see this module's docstring, "Wildcard sources"): the build can't detect that on its own, + so the resulting file tree is printed here to make it easy to catch by eye instead. + """ + build_dir = BUILD_CONTEXT_ROOT / library / target + shutil.rmtree(build_dir, ignore_errors=True) + build_dir.mkdir(parents=True) + + for file in dependencies: + _link_or_copy(context_root / file, build_dir / file) + + _link_or_copy(dockerfile, build_dir / dockerfile.relative_to(context_root)) + + print(f"Build context for {library}/{target} ({build_dir}):") + for file in sorted(build_dir.rglob("*")): + if file.is_file(): + print(f" {file.relative_to(build_dir)}") + + return build_dir + + def image_exists(tag: str) -> bool: """Whether `tag` exists on the registry. `docker manifest inspect` exits non-zero both when the tag genuinely doesn't exist and on unrelated failures (auth, network); print @@ -111,7 +342,7 @@ def image_exists(tag: str) -> bool: return result.returncode == 0 -def build_and_push(bake_file: Path, target: str, tag: str) -> None: +def build_and_push(bake_file: Path, target: str, tag: str, build_dir: Path, dockerfile_name: str) -> None: print(f"Building and pushing {tag}") _run( [ @@ -122,6 +353,10 @@ def build_and_push(bake_file: Path, target: str, tag: str) -> None: "--progress=plain", "--set", f"{target}.tags={tag}", + "--set", + f"{target}.context={build_dir}", + "--set", + f"{target}.dockerfile={dockerfile_name}", "-f", str(bake_file), target, @@ -130,19 +365,27 @@ def build_and_push(bake_file: Path, target: str, tag: str) -> None: def process_library(library: str, *, dry_run: bool) -> None: - dependencies_by_target = WeblogMetaData.load_base_image_dependencies(library) - if not dependencies_by_target: - return - bake_file = _bake_file(library) if not bake_file.exists(): - print(f"Warning: {library} declares base_image_dependencies but has no docker-bake.hcl, skipping") return - for target, dependencies in dependencies_by_target.items(): - bake_config = _bake_config(bake_file, target) + for target, bake_config in _all_bake_configs(bake_file).items(): + # REPO_ROOT is only ever used here, to locate the bake target's own context directory + # (`docker buildx bake --print` may report it as either absolute or relative). Every + # dependency-resolution/hash/materialize step downstream operates purely in terms of + # context_root, never REPO_ROOT again. + context_root = (REPO_ROOT / bake_config["context"]).resolve() + dockerfile = context_root / bake_config["dockerfile"] + + dependencies = _dependency_paths(context_root, dockerfile) + + # Materialize before hashing (hardlinking is essentially free, see this module's + # docstring), so the hash is always computed from the exact files the build will see, + # never from a separate read of context_root that is merely supposed to match. + build_dir = materialize_build_context(library, target, context_root, dockerfile, dependencies) + base_tag = bake_config["tags"][0] - content_hash = compute_hash(bake_config, dependencies) + content_hash = compute_hash(build_dir, bake_config) tag = f"{base_tag}-{content_hash}" if dry_run: @@ -154,7 +397,7 @@ def process_library(library: str, *, dry_run: bool) -> None: print(f"{tag} already exists, skipping") continue - build_and_push(bake_file, target, tag) + build_and_push(bake_file, target, tag, build_dir, bake_config["dockerfile"]) def main() -> None: From d2abdebb5c53b8e8dd41a77cf8156aa9f072122b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Jul 2026 11:48:00 +0200 Subject: [PATCH 9/9] cleanup --- utils/scripts/build_base_images.py | 208 ++++++++++------------------- 1 file changed, 71 insertions(+), 137 deletions(-) diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py index ec17760516c..88f5a2fd2cd 100644 --- a/utils/scripts/build_base_images.py +++ b/utils/scripts/build_base_images.py @@ -1,83 +1,59 @@ """Rebuild and push weblog base images whose content-hash tag is missing from Docker Hub. -Run from a system-tests checkout, with the runner venv active: +Usage (from a system-tests checkout, runner venv active): python utils/scripts/build_base_images.py -For every library with a `utils/build/docker//docker-bake.hcl` file, and for every -target in that file's "default" group: - - 1. Resolve the target's bake config (context/dockerfile/args) via `docker buildx bake --print`. - 2. Parse the target's Dockerfile for its `COPY` instructions to determine which local, - non-gitignored files (relative to the Dockerfile's own directory, which is required to be - the bake target's context — see "Base Dockerfile constraints" below) the image depends on. - 3. Materialize those files, plus the Dockerfile itself, into an isolated build context (see - "Safety net" below) — hardlinking is essentially free, so this always happens next, before - even computing the hash. - 4. Compute a content hash from the resolved bake config (tags excluded) and the path and - content of every file in that materialized build context. - 5. Take the base tag declared in the bake file (e.g. "datadog/system-tests:express4.base") and - append "-" to get the final tag. - 6. Skip the build if that tag already exists on Docker Hub (`docker manifest inspect`); - otherwise build and push the already-materialized build context with that tag. - -This is idempotent and safe to run on every push: it never overwrites an existing tag, it only -creates new ones when the relevant files change. - -Pass --dry-run to only print the computed tag and whether it already exists, without ever -building or pushing (useful to find the tag to put in a weblog Dockerfile's FROM clause after -dependencies change). +For every library with a `utils/build/docker//docker-bake.hcl` file, and every target +in that file's "default" group: + + 1. Resolve the target's bake config via `docker buildx bake --print`. + 2. Parse the Dockerfile's `COPY` instructions for local, non-gitignored dependencies. + 3. Materialize those files, plus the Dockerfile, into an isolated build context (see "Safety + net" below). + 4. Hash the bake config (tags excluded) plus that materialized context. + 5. Append "-" to the bake file's base tag (e.g. "datadog/system-tests:express4.base"). + 6. Skip if that tag already exists on Docker Hub (`docker manifest inspect`); otherwise build + and push. + +Idempotent: never overwrites an existing tag, only creates new ones when dependencies change. + +Pass --dry-run to only print the computed tag and whether it already exists, without building +or pushing (useful to find the tag for a weblog Dockerfile's FROM clause after deps change). Base Dockerfile constraints ---------------------------- -So that the dependency list above can be derived mechanically and unambiguously from the -Dockerfile alone, every `.base.Dockerfile` built by this script must follow these rules: - - - No `ADD`. Use `COPY` for everything (no whole-context-directory copies, no remote URLs). - `COPY` sources may contain glob wildcards (see "Wildcard sources" below). - - Every `COPY` instruction has exactly one source: `COPY [flags] `. Split - multi-source `COPY` instructions into one `COPY` per source. - - The bake target's `context` is always the Dockerfile's own directory, so every `COPY` - source is a plain path relative to that directory (`COPY app.js .`, not - `COPY utils/build/docker/nodejs/fastify/app.js .`). - - No `RUN --mount`. Bind/cache/secret mounts read from paths this script cannot see, so they - would silently escape the derived dependency list. - -`COPY --from=` is unaffected by these rules: multi-stage copies and copies from -an external image aren't local repository paths, so they're skipped when deriving dependencies -(buildx/buildkit resolves them independently). +So the dependency list above can be derived mechanically from the Dockerfile alone, every +`.base.Dockerfile` built by this script must follow these rules: + + - No `ADD`. Use `COPY` only (no whole-context copies, no remote URLs); wildcards are allowed + (see "Wildcard sources" below). + - Every `COPY` has exactly one source: `COPY [flags] `. + - The bake target's `context` is always the Dockerfile's own directory, so `COPY` sources are + plain paths relative to it (`COPY app.js .`, not `COPY utils/build/docker/nodejs/fastify/app.js .`). + - No `RUN --mount`: those mounts read paths this script cannot see, so they'd silently escape + the derived dependency list. + +`COPY --from=` is exempt from these rules and skipped when deriving dependencies +(not a local repository path). Wildcard sources ----------------- -A `COPY` source is resolved with `Path.glob()` against the Dockerfile's own directory, so a -plain path (no wildcard) and a glob pattern are handled the same way: a plain path simply -matches itself, a pattern is expanded to every file/directory it matches. A pattern matching -zero files raises (almost certainly a typo), but that is the only completeness check available: -a pattern matching *fewer* files than intended does not raise, because Docker's own wildcard -expansion, run again at build time against the materialized build context (see below), only -ever sees the files that same pattern already matched when we computed dependencies — it can't -disagree with a match we already computed by construction. So an incomplete wildcard match -doesn't fail the build the way a missing literal dependency does; it just silently ships an -incomplete image. The materialized build context's file list is printed before every build -specifically to make that failure mode easy to catch by eye. +A `COPY` source is resolved with `Path.glob()` against the Dockerfile's own directory (a plain +path just matches itself). A pattern matching zero files raises (likely a typo); a pattern +matching *fewer* files than intended cannot be detected and silently ships an incomplete image. +The materialized build context's file list is printed before every build to make that case easy +to catch by eye. Safety net: isolated build context ----------------------------------- -Every detected dependency is hardlinked (falling back to a plain copy if hardlinking isn't -possible, e.g. across filesystems) into `.base_image_build///`, and the image -is built from *that* directory instead of the real one, after printing that directory's file -list. Hardlinking is only a directory-entry operation (no data is duplicated), so this is -essentially free. Its purpose is correctness, not performance: if the Dockerfile references a -file that the parser failed to detect as a dependency, the build fails loudly ("file not found") -instead of silently succeeding against the full repository checkout — which would let an -incomplete dependency list (and thus an under-hashed, stale-looking tag) go unnoticed. (This -guarantee does not extend to an incomplete wildcard match, see above — the printed file list is -the mitigation for that case.) - -The content hash is computed directly from this materialized directory (see `compute_hash`), -not from a second, independent read of the dependency paths: this way "the files the hash was -computed from" and "the files the build can actually see" are the same files by construction, -not two separate computations that are merely supposed to produce the same result. +Every detected dependency is hardlinked (falling back to a copy, e.g. across filesystems) into +`.base_image_build///`, and the image is built from that directory instead of +the real one. This way, a Dockerfile reference the parser failed to detect as a dependency makes +the build fail loudly ("file not found") instead of silently succeeding against the full +repository checkout. The content hash is computed from this same materialized directory (see +`compute_hash`), not a separate read of the dependency paths, so the hashed files and the built +files are identical by construction. """ import argparse @@ -89,8 +65,7 @@ import sys from pathlib import Path -# Make `utils` importable and resolve paths regardless of the caller's cwd, so a -# plain `python utils/scripts/build_base_images.py` works. +# So `python utils/scripts/build_base_images.py` works regardless of the caller's cwd. REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT)) @@ -98,8 +73,6 @@ BUILD_CONTEXT_ROOT = REPO_ROOT / ".base_image_build" -# A `COPY [flags] ` line has exactly one source and one destination path -# (see this module's docstring, "Base Dockerfile constraints"). _SOURCE_AND_DEST_TOKEN_COUNT = 2 @@ -129,21 +102,14 @@ def _all_bake_configs(bake_file: Path) -> dict[str, dict]: def _files_under(context_root: Path, path: Path) -> list[Path]: - """Every non-gitignored file under `path` (a file or a directory, relative to - `context_root`), itself relative to `context_root`, sorted. - - This deliberately filters on `.gitignore` rules alone, not on whether a file is committed - or even staged (`--cached --others --exclude-standard` includes both tracked files and - untracked-but-not-ignored ones): this script always runs against a real checkout, tracked - or not, so there is nothing to gain from requiring content to be in the git index, only - the risk of silently skipping a real, intended, not-yet-committed dependency. What must - still be filtered out is gitignored content that happens to sit on disk under a dependency - path (e.g. a locally installed, never-committed `node_modules/`), since that is never meant - to be part of the image's tracked dependency set. - - Raises if `path` exists on disk (so `_dependency_paths` already accepted it as a match) but - every file under it turns out to be gitignored, since that would otherwise silently produce - an empty dependency with no explanation. + """Every non-gitignored file under `path` (a file or directory, relative to `context_root`), + itself relative to `context_root`, sorted. + + Filters on `.gitignore` alone, not on git-tracked status, since this script runs against a + real checkout where untracked-but-not-ignored files are still valid dependencies. + + Raises if `path` exists on disk but every file under it is gitignored, rather than silently + producing an empty dependency. """ result = _run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "--", str(path)], cwd=context_root) files = sorted(Path(line) for line in result.stdout.splitlines() if line) @@ -179,10 +145,8 @@ def _dockerfile_logical_lines(dockerfile: Path) -> list[str]: def parse_copy_dependencies(dockerfile: Path) -> list[str]: """Local `COPY` source paths (relative to `dockerfile`'s own directory) that a base - Dockerfile depends on. Enforces the constraints listed in this module's docstring: - no `ADD`, no `RUN --mount`, and every `COPY` has exactly one source. - - `COPY --from=` lines are recognized and skipped (not a local repo path). + Dockerfile depends on. Enforces this module's Dockerfile constraints: no `ADD`, no + `RUN --mount`, and every `COPY` has exactly one source. """ dependencies: list[str] = [] for line in _dockerfile_logical_lines(dockerfile): @@ -219,27 +183,15 @@ def parse_copy_dependencies(dockerfile: Path) -> list[str]: def _dependency_paths(context_root: Path, dockerfile: Path) -> list[Path]: """Every non-gitignored file (relative to `context_root`) that `dockerfile` depends on, - sorted and deduplicated. `context_root` is `dockerfile`'s own directory, per this module's - Dockerfile constraints. This is the single place dependencies are resolved: both - `compute_hash` and `materialize_build_context` operate on exactly this list, so "the files - the hash was computed from" and "the files that get hardlinked" are the same set by - construction, not by two independent computations happening to agree. - - Each COPY source is resolved with `Path.glob()`, so a plain path and a glob pattern are - handled uniformly (a plain path just matches itself). A source matching zero files raises, - since that's almost certainly a typo; see this module's docstring ("Wildcard sources") for - why a pattern matching *fewer* files than intended is not, and cannot be, detected here. - - Every match must resolve to a path under `context_root` itself: per this module's Dockerfile - constraints, a base Dockerfile's `COPY` sources are only ever meant to reach within its own - directory (e.g. a `nodejs` base image has no business depending on a file under - `utils/build/docker/python/`), so a source escaping `context_root` is rejected even if it - stays inside the repository. - - Each match is then flattened to its constituent files via `_files_under`, which also filters - out gitignored content (e.g. a locally installed, never-committed `node_modules/` sitting - under a dependency directory): two COPY sources can overlap (a directory and one of its own - files listed separately), so the final list is deduplicated. + sorted and deduplicated. `context_root` is `dockerfile`'s own directory. + + Each COPY source is resolved with `Path.glob()` (a plain path just matches itself). A source + matching zero files raises (likely a typo); see this module's docstring ("Wildcard sources") + for why a pattern matching fewer files than intended can't be detected here. + + Every match must resolve to a path under `context_root` itself, per this module's Dockerfile + constraints (e.g. a `nodejs` base image can't depend on a file under + `utils/build/docker/python/`). """ files: set[Path] = set() for source in parse_copy_dependencies(dockerfile): @@ -262,13 +214,7 @@ def _dependency_paths(context_root: Path, dockerfile: Path) -> list[Path]: def compute_hash(build_dir: Path, bake_config: dict) -> str: """Content hash for a base image target: bake config (minus tags) + path and content of every file in `build_dir` (the isolated build context already materialized by - `materialize_build_context`, which includes the Dockerfile itself alongside its - dependencies). - - Hashing the materialized directory directly, rather than separately re-reading dependencies - from `context_root`, guarantees "the files the hash was computed from" and "the files the - build can actually see" are the exact same files by construction (same inodes, even), not - two independent reads that are merely supposed to agree. + `materialize_build_context`, including the Dockerfile itself). """ digest = hashlib.sha256() @@ -297,16 +243,8 @@ def materialize_build_context( """Hardlink (falling back to a copy) every file in `dependencies` (relative to `context_root`, already flattened and deduplicated by `_dependency_paths`), plus `dockerfile` itself, into a fresh `.base_image_build///` directory, mirroring each - file's path relative to `context_root`. - - Building from this directory instead of the real one is a safety net: it makes "the files - the hash was computed from" and "the files the build can actually see" the same set by - construction, so a dependency the parser failed to detect causes a build failure (missing - file) instead of silently succeeding against the full repository checkout. - - That guarantee doesn't extend to a wildcard COPY source matching fewer files than intended - (see this module's docstring, "Wildcard sources"): the build can't detect that on its own, - so the resulting file tree is printed here to make it easy to catch by eye instead. + file's path relative to `context_root`. Building from this directory instead of the real one + is the safety net described in this module's docstring. """ build_dir = BUILD_CONTEXT_ROOT / library / target shutil.rmtree(build_dir, ignore_errors=True) @@ -326,9 +264,9 @@ def materialize_build_context( def image_exists(tag: str) -> bool: - """Whether `tag` exists on the registry. `docker manifest inspect` exits non-zero both - when the tag genuinely doesn't exist and on unrelated failures (auth, network); print - the error either way so a real failure isn't silently mistaken for a missing tag. + """Whether `tag` exists on the registry. `docker manifest inspect` exits non-zero both for + a genuinely missing tag and for unrelated failures (auth, network); print stderr either way + so a real failure isn't mistaken for a missing tag. """ result = subprocess.run( ["docker", "manifest", "inspect", tag], @@ -370,18 +308,14 @@ def process_library(library: str, *, dry_run: bool) -> None: return for target, bake_config in _all_bake_configs(bake_file).items(): - # REPO_ROOT is only ever used here, to locate the bake target's own context directory - # (`docker buildx bake --print` may report it as either absolute or relative). Every - # dependency-resolution/hash/materialize step downstream operates purely in terms of - # context_root, never REPO_ROOT again. + # `docker buildx bake --print` may report `context` as absolute or relative. context_root = (REPO_ROOT / bake_config["context"]).resolve() dockerfile = context_root / bake_config["dockerfile"] dependencies = _dependency_paths(context_root, dockerfile) - # Materialize before hashing (hardlinking is essentially free, see this module's - # docstring), so the hash is always computed from the exact files the build will see, - # never from a separate read of context_root that is merely supposed to match. + # Materialize before hashing, so the hash is computed from the exact files the build + # will see. build_dir = materialize_build_context(library, target, context_root, dockerfile, dependencies) base_tag = bake_config["tags"][0]