diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e50b1d1255..b49270477c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,6 @@ jobs: _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }} _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }} _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }} - _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }} _enable_replay_scenarios: true _system_tests_dev_mode: ${{ matrix.version == 'dev' }} _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }} diff --git a/.github/workflows/compute-workflow-parameters.yml b/.github/workflows/compute-workflow-parameters.yml index f5ac934999a..0405493a000 100644 --- a/.github/workflows/compute-workflow-parameters.yml +++ b/.github/workflows/compute-workflow-parameters.yml @@ -52,11 +52,6 @@ on: default: '' required: false type: string - _build_weblog_base_images: - description: "Shall we build weblog base images" - default: false - required: false - type: boolean # Map the workflow outputs to job outputs outputs: @@ -142,7 +137,6 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ - --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --output $GITHUB_OUTPUT - name: log run: | @@ -156,7 +150,6 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ - --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --format json | jq - name: Extract library target branch diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 56564d9680b..1fc7d6e89f5 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -63,7 +63,7 @@ on: required: false type: boolean _build_weblog_base_image: - description: "Shall we build the weblog base image (only valid if weblog build is local)" + description: "Shall we wait for the weblog base image to be pushed by GitLab CI before building the weblog" default: false required: false type: boolean @@ -146,11 +146,9 @@ jobs: if: inputs._build_lambda_proxy_image run: ./build.sh -i lambda-proxy - - name: Build weblog base image + - name: Wait for weblog base image if: inputs._build_weblog_base_image - run: | - source venv/bin/activate - ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} + run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30 - name: Pull images uses: ./.github/actions/pull_images diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index a783c6f9ca1..9ea0bc6961a 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -44,11 +44,6 @@ on: default: 'custom' required: false type: string - _build_weblog_base_images: - description: "Shall we rebuild weblog base images" - default: false - required: false - type: boolean _build_buddies_images: description: "Shall we build buddies images" default: false @@ -154,7 +149,6 @@ jobs: binaries_artifact: ${{ inputs.binaries_artifact }} _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }} _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }} - _build_weblog_base_images: ${{ inputs._build_weblog_base_images }} parametric: needs: @@ -215,11 +209,9 @@ jobs: run: ls -la binaries/ - name: Export github token to a file run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt" - - name: Build weblog base images - if: matrix.weblog.build_base_image - run: | - source venv/bin/activate - ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} + - name: Wait for weblog base image + if: matrix.weblog.build_weblog_base_image + run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30 - name: Build weblog id: build run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt" diff --git a/.gitignore b/.gitignore index b8cc70f86c0..51e07b11d40 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,9 @@ version-check-output/ tmp/ +# Materialized build context for utils/scripts/build_base_images.py +/.base_image_build/ + # Go go.work* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7a9bb9f2ab..3a862f8dbd2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -474,3 +474,35 @@ mirror_images: - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' + +# ────────────────────────────────────────────── +# Rebuild weblog base images whose content changed. +# +# For every library with a `base_image_dependencies` section in +# utils/build/docker//weblog_metadata.yml, computes a content-hash tag per +# docker-bake.hcl target and pushes it if missing (see utils/scripts/build_base_images.py). +# Runs on every push, on every branch: it is idempotent (only pushes new tags, never +# overwrites existing ones), so it is safe to build ahead of merge. Since new tags never +# replace old ones, a weblog Dockerfile's FROM clause must be updated by hand to pick up +# a newly pushed base image (run the script with --dry-run to find the new tag). +# ────────────────────────────────────────────── + +build_base_images: + image: $CI_IMAGE + tags: + - docker-in-docker:amd64 + needs: + - job: build_ci_image + artifacts: false + optional: true + stage: e2e + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ln -sf /system-tests/venv venv + - source venv/bin/activate + script: + - python utils/scripts/build_base_images.py + rules: + - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' diff --git a/docs/CI/github-actions.md b/docs/CI/github-actions.md index 75a2fc61bce..2272e4f7787 100644 --- a/docs/CI/github-actions.md +++ b/docs/CI/github-actions.md @@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI | `_build_buddies_images` | Shall we build buddies images | boolean | false | false | | `_build_proxy_image` | Shall we build proxy image | boolean | false | false | | `_build_lambda_proxy_image` | Shall we build the lambda-proxy image | boolean | false | false | -| `_build_weblog_base_images` | Shall we build weblog base images | boolean | false | false | | `_enable_replay_scenarios` | Enable replay scenarios, should only be used in system-tests CI | boolean | false | false | diff --git a/docs/edit/update-docker-images.md b/docs/edit/update-docker-images.md index a2ae67d46e4..cc31659f7ba 100644 --- a/docs/edit/update-docker-images.md +++ b/docs/edit/update-docker-images.md @@ -1,11 +1,22 @@ Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/). -If you need to update them, you will need to follow those +For weblog base images (nodejs, python, php), the build and push are fully automated: -1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code) -2. create your PR, and add the relevant label to rebuild the image in the CI - * `build-python-base-images` for python weblogs - * `build-php-base-images` for PHP weblogs - * `build-nodejs-base-images` for Node.js weblogs - * `build-proxy-image` for proxy image -3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack) +1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every + branch. For each target in a library's `docker-bake.hcl`, it derives the target's dependencies from the + `COPY` instructions in its `.base.Dockerfile` (see `docs/understand/weblogs/weblog-metadata.md` + for the Dockerfile rules this relies on), computes a content hash of those dependencies, and, if a base + image tagged with that hash doesn't already exist on Docker Hub, builds and pushes it as + `-`. +2. If you changed a file referenced by a `.base.Dockerfile`'s `COPY` instructions, a new tag will + automatically be pushed by that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to + point to the new tag (you can find it by running `python utils/scripts/build_base_images.py --dry-run` + locally). +3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for + the tag referenced in the weblog's `FROM` line to become available before building the weblog, via + `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag + before (or shortly after) your PR's GitHub CI run starts. + +For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a +rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push +your image to hub.docker.com (`#apm-shared-testing` on slack). diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md index 2730d9e6875..c07f8b767b4 100644 --- a/docs/understand/weblogs/weblog-metadata.md +++ b/docs/understand/weblogs/weblog-metadata.md @@ -45,3 +45,40 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`. `WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges: 1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata). 2. Explicit overrides from `weblog_metadata.yml`. + +## Base image dependencies + +Base images (built by the `build_base_images` CI job, `utils/scripts/build_base_images.py`) are +declared in each library's `utils/build/docker//docker-bake.hcl`, one target per base +image. There is no separate dependency list to maintain: for each target, the job parses the +target's own `.base.Dockerfile` and treats every `COPY` source as a dependency. This works +because base Dockerfiles are required to follow a few rules that make that derivation +unambiguous: + +- No `ADD` — use `COPY` for everything (no glob sources, no whole-directory copies, no remote + URLs). +- Every `COPY` has exactly one source: `COPY [flags] `. +- The bake target's `context` is always the Dockerfile's own directory, so every `COPY` source + is a plain path relative to that directory. +- No `RUN --mount` — a bind/cache/secret mount reads from a path the script can't see, so it + would silently escape the derived dependency list. + +(`COPY --from=` is unaffected: it isn't a local repository path, so it's skipped.) + +The job computes a content hash from the resolved `docker-bake.hcl` target config, the target's +Dockerfile, and every git-tracked file under each derived dependency path, then pushes the base +image to Docker Hub tagged `-` if that tag doesn't already exist. It never +overwrites an existing tag, so weblog Dockerfiles that `FROM` a base image must have their tag +updated by hand after a new one is pushed (run the script with `--dry-run` to find the current +tag for each target). + +As a safety net, before building, every derived dependency is hardlinked (or copied, if +hardlinking isn't possible) into an isolated build context under `.base_image_build/`, and the +image is built from that directory instead of the real one. This way, if the Dockerfile +references a file the parser failed to recognize as a dependency, the build fails loudly +("file not found") instead of silently succeeding against the full checkout — which would leave +the tag's content hash stale without anyone noticing. + +GitHub Actions never builds these base images itself: `utils/scripts/wait_for_base_image.py` +polls Docker Hub for the tag currently referenced in the weblog's `FROM` line (with a timeout) +before building the weblog, since GitLab CI is the only pipeline that builds and pushes them. diff --git a/tests/test_the_test/test_build_base_images.py b/tests/test_the_test/test_build_base_images.py new file mode 100644 index 00000000000..7fa984431ec --- /dev/null +++ b/tests/test_the_test/test_build_base_images.py @@ -0,0 +1,487 @@ +from pathlib import Path +import subprocess +import tempfile +import textwrap + +import pytest + +from utils import scenarios +from utils.scripts import build_base_images +from utils.scripts.build_base_images import ( + _dependency_paths, + _files_under, + compute_hash, + materialize_build_context, + parse_copy_dependencies, +) + + +def _write_dockerfile(tmp_path: Path, content: str) -> Path: + dockerfile = tmp_path / "some.base.Dockerfile" + dockerfile.write_text(textwrap.dedent(content)) + return dockerfile + + +@scenarios.test_the_test +class Test_ParseCopyDependencies: + """Unit tests for utils.scripts.build_base_images.parse_copy_dependencies, which derives + a base image's dependencies from the `COPY` instructions of its Dockerfile. + """ + + def test_single_copy(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_multiple_copy_instructions(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express /usr/app + COPY express4/package.json ./ + COPY express4/bun.lock ./ + COPY nft-prune.mjs ./ + """, + ) + assert parse_copy_dependencies(dockerfile) == [ + "express", + "express4/package.json", + "express4/bun.lock", + "nft-prune.mjs", + ] + + def test_copy_with_flags_is_parsed(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY --chown=node:node app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_copy_from_external_image_is_skipped(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY --from=oven/bun:1.3.13-alpine /usr/local/bin/bun /usr/local/bin/bun + COPY app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_copy_from_build_stage_is_skipped(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM golang:1.22 AS build + RUN echo build + + FROM alpine + COPY --from=build /app/weblog /app/weblog + COPY app.sh . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.sh"] + + def test_line_continuation_is_joined(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY \\ + app.js \\ + . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_lowercase_instruction_is_parsed(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + copy app.js . + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_comments_and_blank_lines_are_ignored(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + + # This is a comment + COPY app.js . + + # docker build ... + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_run_without_mount_is_allowed(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js . + RUN node --version + """, + ) + assert parse_copy_dependencies(dockerfile) == ["app.js"] + + def test_no_copy_instructions_returns_empty_list(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + RUN node --version + """, + ) + assert parse_copy_dependencies(dockerfile) == [] + + def test_add_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + ADD app.js . + """, + ) + with pytest.raises(ValueError, match="ADD is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_add_with_glob_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + ADD binaries* /binaries/ + """, + ) + with pytest.raises(ValueError, match="ADD is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_run_mount_bind_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM golang:1.22 + RUN --mount=type=bind,source=.,target=/src \\ + go build ./... + """, + ) + with pytest.raises(ValueError, match="RUN --mount is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_run_mount_cache_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM python:3.11 + RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt + """, + ) + with pytest.raises(ValueError, match="RUN --mount is not allowed"): + parse_copy_dependencies(dockerfile) + + def test_multi_source_copy_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js dsm.js rasp.js ./ + """, + ) + with pytest.raises(ValueError, match="exactly one source"): + parse_copy_dependencies(dockerfile) + + def test_copy_with_no_destination_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js + """, + ) + with pytest.raises(ValueError, match="exactly one source"): + parse_copy_dependencies(dockerfile) + + +@scenarios.test_the_test +class Test_DependencyPaths: + """Unit tests for utils.scripts.build_base_images._dependency_paths, which resolves each + COPY source parsed by parse_copy_dependencies() against `context_root` (the Dockerfile's own + directory), expanding glob wildcards along the way (see the module docstring, "Wildcard + sources"), then flattens each match to its constituent non-gitignored files via + `_files_under` (see Test_FilesUnder). Every returned path is relative to `context_root`, + which is also the only boundary a COPY source is allowed to stay within (see the "escapes" + test below) — REPO_ROOT plays no part in this resolution, so these tests don't need to touch + it at all. + """ + + @pytest.fixture(autouse=True) + def _git_repo(self, tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + + def test_literal_source_resolves_to_itself(self, tmp_path: Path): + (tmp_path / "app.js").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY app.js . + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [Path("app.js")] + + def test_literal_directory_source_flattens_to_its_files(self, tmp_path: Path): + (tmp_path / "express").mkdir() + (tmp_path / "express" / "app.js").write_text("") + (tmp_path / "express" / "auth.js").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express /usr/app + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [ + Path("express/app.js"), + Path("express/auth.js"), + ] + + def test_wildcard_source_expands_to_every_match(self, tmp_path: Path): + (tmp_path / "package.json").write_text("") + (tmp_path / "package-lock.json").write_text("") + (tmp_path / "app.js").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY *.json . + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [ + Path("package-lock.json"), + Path("package.json"), + ] + + def test_overlapping_sources_are_deduplicated(self, tmp_path: Path): + # A directory dependency and one of its own files listed separately (as in + # `COPY express4-typescript /usr/app` + `COPY express4-typescript/package.json ./` in a + # real base Dockerfile) overlap: the file must only appear once in the final list. + (tmp_path / "express").mkdir() + (tmp_path / "express" / "app.js").write_text("") + (tmp_path / "express" / "package.json").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express /usr/app + COPY express/package.json ./ + """, + ) + assert _dependency_paths(tmp_path, dockerfile) == [ + Path("express/app.js"), + Path("express/package.json"), + ] + + def test_wildcard_source_matching_nothing_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY *.json . + """, + ) + with pytest.raises(ValueError, match="matched no files"): + _dependency_paths(tmp_path, dockerfile) + + def test_literal_source_matching_nothing_is_rejected(self, tmp_path: Path): + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY does-not-exist.txt . + """, + ) + with pytest.raises(ValueError, match="matched no files"): + _dependency_paths(tmp_path, dockerfile) + + def test_source_escaping_context_root_is_rejected(self, tmp_path: Path): + # A COPY source that reaches into a sibling library's directory (e.g. a nodejs base + # Dockerfile depending on something under utils/build/docker/python/) must be rejected, + # regardless of whether the resolved path stays inside the repository or not. + context_root = tmp_path / "nodejs" + sibling_dir = tmp_path / "python" + context_root.mkdir() + sibling_dir.mkdir() + (sibling_dir / "weblog_metadata.yml").write_text("") + + dockerfile = _write_dockerfile( + context_root, + """\ + FROM node:18-alpine + COPY ../python/weblog_metadata.yml . + """, + ) + with pytest.raises(ValueError, match="escapes the Dockerfile's context directory"): + _dependency_paths(context_root, dockerfile) + + +@scenarios.test_the_test +class Test_FilesUnder: + """Unit tests for utils.scripts.build_base_images._files_under, which lists the files under + a dependency path that should actually be hashed/hardlinked: every git-tracked file, plus + every untracked-but-not-gitignored file, but never anything gitignored (e.g. a locally + installed, never-committed node_modules/ sitting under a dependency directory). Filtering is + based purely on .gitignore rules, not on whether a file is committed or staged. + """ + + @pytest.fixture(autouse=True) + def _git_repo(self, tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + + def test_tracked_file_is_included(self, tmp_path: Path): + (tmp_path / "app.js").write_text("") + subprocess.run(["git", "add", "app.js"], cwd=tmp_path, check=True) + + assert _files_under(tmp_path, Path("app.js")) == [Path("app.js")] + + def test_untracked_but_not_gitignored_file_is_included(self, tmp_path: Path): + # Never `git add`-ed, and not covered by any .gitignore rule: this script must not + # require content to be committed, or even staged, to be picked up. + (tmp_path / "app.js").write_text("") + + assert _files_under(tmp_path, Path("app.js")) == [Path("app.js")] + + def test_gitignored_file_under_a_directory_dependency_is_excluded(self, tmp_path: Path): + (tmp_path / ".gitignore").write_text("node_modules/\n") + (tmp_path / "express").mkdir() + (tmp_path / "express" / "app.js").write_text("") + (tmp_path / "express" / "node_modules").mkdir() + (tmp_path / "express" / "node_modules" / "some_dep.js").write_text("") + subprocess.run(["git", "add", "express/app.js", ".gitignore"], cwd=tmp_path, check=True) + + assert _files_under(tmp_path, Path("express")) == [Path("express/app.js")] + + def test_dependency_entirely_gitignored_raises_with_an_explicit_message(self, tmp_path: Path): + (tmp_path / ".gitignore").write_text("secret.txt\n") + (tmp_path / "secret.txt").write_text("") + + with pytest.raises(ValueError, match="every file under it is gitignored"): + _files_under(tmp_path, Path("secret.txt")) + + def test_directory_dependency_entirely_gitignored_raises_with_an_explicit_message(self, tmp_path: Path): + (tmp_path / ".gitignore").write_text("ignored_dir/\n") + (tmp_path / "ignored_dir").mkdir() + (tmp_path / "ignored_dir" / "x.txt").write_text("") + + with pytest.raises(ValueError, match="every file under it is gitignored"): + _files_under(tmp_path, Path("ignored_dir")) + + def test_outside_a_git_repository_raises(self): + # This test needs a directory that is guaranteed not to be inside any git repository + # (unlike tmp_path, which _git_repo's autouse fixture always `git init`s), so it uses a + # fully independent temporary directory instead. + with tempfile.TemporaryDirectory() as outside_any_repo_str: + outside_any_repo = Path(outside_any_repo_str) + (outside_any_repo / "app.js").write_text("") + + with pytest.raises(subprocess.CalledProcessError): + _files_under(outside_any_repo, Path("app.js")) + + +@scenarios.test_the_test +class Test_MaterializeBuildContext: + """Unit tests for utils.scripts.build_base_images.materialize_build_context, which + hardlinks every dependency (plus the Dockerfile itself) into an isolated build directory. + """ + + @pytest.fixture(autouse=True) + def _git_repo(self, tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + + def test_overlapping_dependencies_are_deduplicated(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + # Regression test: a directory dependency and one of its own files listed separately as + # a second dependency (as in `COPY express4-typescript /usr/app` + + # `COPY express4-typescript/package.json ./` in a real base Dockerfile) overlap, so the + # same source file would otherwise be hardlinked into the same destination twice. + monkeypatch.setattr(build_base_images, "BUILD_CONTEXT_ROOT", tmp_path / "build") + + (tmp_path / "express4-typescript").mkdir() + (tmp_path / "express4-typescript" / "package.json").write_text("{}") + (tmp_path / "express4-typescript" / "app.ts").write_text("") + dockerfile = _write_dockerfile( + tmp_path, + """\ + FROM node:18-alpine + COPY express4-typescript /usr/app + COPY express4-typescript/package.json ./ + """, + ) + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + + dependencies = _dependency_paths(tmp_path, dockerfile) + build_dir = materialize_build_context("somelib", "sometarget", tmp_path, dockerfile, dependencies) + + materialized = sorted(p.relative_to(build_dir) for p in build_dir.rglob("*") if p.is_file()) + assert materialized == [ + Path("express4-typescript/app.ts"), + Path("express4-typescript/package.json"), + Path(dockerfile.name), + ] + + +@scenarios.test_the_test +class Test_ComputeHash: + """Unit tests for utils.scripts.build_base_images.compute_hash, which hashes the resolved + bake config (minus tags) together with the path and content of every file in an already + materialized build directory (see materialize_build_context) — not a separate read of the + dependency paths, so there is nothing git- or context_root-related to set up here. + """ + + def test_same_directory_and_config_produce_the_same_hash(self, tmp_path: Path): + (tmp_path / "app.js").write_text("console.log(1);") + bake_config = {"context": ".", "dockerfile": "x.base.Dockerfile", "tags": ["x"]} + + assert compute_hash(tmp_path, bake_config) == compute_hash(tmp_path, bake_config) + + def test_different_file_content_changes_the_hash(self, tmp_path_factory: pytest.TempPathFactory): + dir_a = tmp_path_factory.mktemp("a") + dir_b = tmp_path_factory.mktemp("b") + (dir_a / "app.js").write_text("console.log(1);") + (dir_b / "app.js").write_text("console.log(2);") + bake_config = {"tags": ["x"]} + + assert compute_hash(dir_a, bake_config) != compute_hash(dir_b, bake_config) + + def test_different_file_name_changes_the_hash(self, tmp_path_factory: pytest.TempPathFactory): + dir_a = tmp_path_factory.mktemp("a") + dir_b = tmp_path_factory.mktemp("b") + (dir_a / "app.js").write_text("same content") + (dir_b / "index.js").write_text("same content") + bake_config = {"tags": ["x"]} + + assert compute_hash(dir_a, bake_config) != compute_hash(dir_b, bake_config) + + def test_different_bake_config_changes_the_hash(self, tmp_path: Path): + (tmp_path / "app.js").write_text("console.log(1);") + + assert compute_hash(tmp_path, {"args": {"PHP_VERSION": "8.0"}, "tags": ["x"]}) != compute_hash( + tmp_path, {"args": {"PHP_VERSION": "8.1"}, "tags": ["x"]} + ) + + def test_tags_are_excluded_from_the_hash(self, tmp_path: Path): + (tmp_path / "app.js").write_text("console.log(1);") + + assert compute_hash(tmp_path, {"tags": ["x"]}) == compute_hash(tmp_path, {"tags": ["y"]}) diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index e2de933b7bd..4abd168d646 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -81,32 +81,27 @@ def test_weblog_build_mode_is_resolved_from_metadata(): @scenarios.test_the_test def test_nodejs_build_base_image(): scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]} - defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True) + defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "") assert defs["endtoend_defs"]["parallel_weblogs"] == [] jobs = {job["weblog"]: job for job in defs["endtoend_defs"]["parallel_jobs"]} - # express4 is build_mode=local and has a base Dockerfile → should build base image + # express4 has a base image tag → should wait for it assert jobs["express4"]["build_weblog_base_image"] is True - # openai-js is build_mode=none and has no base Dockerfile → should not build base image + # openai-js has no base image tag → should not wait for a base image assert jobs["openai-js@6.0.0"]["build_weblog_base_image"] is False @scenarios.test_the_test def test_python_build_base_image(): scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]} - defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True) - - # all python weblog has build_mode=prebuild. build_weblog_base_image - # only applies to build_mode=local weblogs → should not build base image inline - for job in defs["endtoend_defs"]["parallel_jobs"]: - assert job["build_weblog_base_image"] is False, job + defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "") - # all python weblog with build_mode=prebuild should rebuild base images in the build job + # all python weblogs with a base image tag should wait for it in the build job for job in defs["endtoend_defs"]["parallel_weblogs"]: - assert job["build_base_images"] is True, job + assert job["build_weblog_base_image"] is True, job @scenarios.test_the_test diff --git a/utils/_context/weblog_metadata.py b/utils/_context/weblog_metadata.py index 7d72d1a2bfb..f8881148f6c 100644 --- a/utils/_context/weblog_metadata.py +++ b/utils/_context/weblog_metadata.py @@ -33,20 +33,6 @@ def require_build(self) -> bool: """The run_end_to_end job builds the weblog locally (weblog_build_required).""" return self.build_mode != BuildMode.none - @property - def base_dockerfile(self) -> Path | None: - """Returns the path of the base image docker file if exists, else None""" - image_name = self.base_image_tag - - if image_name is None: - return None - - file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0] - assert file_prefix.endswith(".base") - - path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile") - return path if path.exists() else None - @property def base_image_tag(self) -> str | None: """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" diff --git a/utils/build/docker/nodejs/docker-bake.hcl b/utils/build/docker/nodejs/docker-bake.hcl index a8430a0ed51..5a46688c07b 100644 --- a/utils/build/docker/nodejs/docker-bake.hcl +++ b/utils/build/docker/nodejs/docker-bake.hcl @@ -1,4 +1,8 @@ # Docker Buildx bake file for Node.js base images +# +# `context` is always this directory: base image Dockerfiles only COPY files from +# here, so paths in the Dockerfile are relative to it (see build_base_images.py, +# which derives base_image_dependencies from these COPY instructions). group "default" { targets = [ @@ -11,40 +15,36 @@ group "default" { } target "_common" { + context = "." provenance = false } target "express4" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/express4.base.Dockerfile" - tags = ["datadog/system-tests:express4.base-v3"] + dockerfile = "express4.base.Dockerfile" + tags = ["datadog/system-tests:express4.base"] } target "express5" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/express5.base.Dockerfile" - tags = ["datadog/system-tests:express5.base-v3"] + dockerfile = "express5.base.Dockerfile" + tags = ["datadog/system-tests:express5.base"] } target "fastify" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/fastify.base.Dockerfile" - tags = ["datadog/system-tests:fastify.base-v3"] + dockerfile = "fastify.base.Dockerfile" + tags = ["datadog/system-tests:fastify.base"] } target "express4-typescript" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/express4-typescript.base.Dockerfile" - tags = ["datadog/system-tests:express4-typescript.base-v3"] + dockerfile = "express4-typescript.base.Dockerfile" + tags = ["datadog/system-tests:express4-typescript.base"] } target "nextjs" { inherits = ["_common"] - context = "." - dockerfile = "utils/build/docker/nodejs/nextjs.base.Dockerfile" - tags = ["datadog/system-tests:nextjs.base-v3"] + dockerfile = "nextjs.base.Dockerfile" + tags = ["datadog/system-tests:nextjs.base"] } diff --git a/utils/build/docker/nodejs/express4-typescript.base.Dockerfile b/utils/build/docker/nodejs/express4-typescript.base.Dockerfile index 2396c0ca98a..dbae3fde995 100644 --- a/utils/build/docker/nodejs/express4-typescript.base.Dockerfile +++ b/utils/build/docker/nodejs/express4-typescript.base.Dockerfile @@ -8,12 +8,13 @@ RUN node --version && npm --version && bun --version && curl --version WORKDIR /usr/app -COPY utils/build/docker/nodejs/express4-typescript /usr/app -COPY utils/build/docker/nodejs/express4-typescript/package.json utils/build/docker/nodejs/express4-typescript/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY express4-typescript /usr/app +COPY express4-typescript/package.json ./ +COPY express4-typescript/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs --keep-types app.ts node_modules/typescript/bin/tsc \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/express4-typescript.base.Dockerfile -t datadog/system-tests:express4-typescript.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/express4-typescript.base.Dockerfile -t datadog/system-tests:express4-typescript.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:express4-typescript.base-v3 diff --git a/utils/build/docker/nodejs/express4.base.Dockerfile b/utils/build/docker/nodejs/express4.base.Dockerfile index 8c24260b4e5..3d9003ecfad 100644 --- a/utils/build/docker/nodejs/express4.base.Dockerfile +++ b/utils/build/docker/nodejs/express4.base.Dockerfile @@ -10,12 +10,13 @@ WORKDIR /usr/app ENV NODE_ENV=production -COPY utils/build/docker/nodejs/express /usr/app -COPY utils/build/docker/nodejs/express4/package.json utils/build/docker/nodejs/express4/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY express /usr/app +COPY express4/package.json ./ +COPY express4/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs app.js \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/express4.base.Dockerfile -t datadog/system-tests:express4.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/express4.base.Dockerfile -t datadog/system-tests:express4.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:express4.base-v3 diff --git a/utils/build/docker/nodejs/express5.base.Dockerfile b/utils/build/docker/nodejs/express5.base.Dockerfile index 9eee044cbab..cd90f59d5d9 100644 --- a/utils/build/docker/nodejs/express5.base.Dockerfile +++ b/utils/build/docker/nodejs/express5.base.Dockerfile @@ -10,12 +10,13 @@ WORKDIR /usr/app ENV NODE_ENV=production -COPY utils/build/docker/nodejs/express /usr/app -COPY utils/build/docker/nodejs/express5/package.json utils/build/docker/nodejs/express5/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY express /usr/app +COPY express5/package.json ./ +COPY express5/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs app.js \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/express5.base.Dockerfile -t datadog/system-tests:express5.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/express5.base.Dockerfile -t datadog/system-tests:express5.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:express5.base-v3 diff --git a/utils/build/docker/nodejs/fastify.base.Dockerfile b/utils/build/docker/nodejs/fastify.base.Dockerfile index 095430dc098..479ca6f3aa0 100644 --- a/utils/build/docker/nodejs/fastify.base.Dockerfile +++ b/utils/build/docker/nodejs/fastify.base.Dockerfile @@ -10,19 +10,19 @@ WORKDIR /usr/app ENV NODE_ENV=production -COPY utils/build/docker/nodejs/fastify/app.js \ - utils/build/docker/nodejs/fastify/dsm.js \ - utils/build/docker/nodejs/fastify/rasp.js \ - ./ -COPY utils/build/docker/nodejs/fastify/debugger ./debugger -COPY utils/build/docker/nodejs/fastify/iast ./iast -COPY utils/build/docker/nodejs/fastify/integrations ./integrations -COPY utils/build/docker/nodejs/fastify/resources ./resources -COPY utils/build/docker/nodejs/fastify/package.json utils/build/docker/nodejs/fastify/bun.lock ./ -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY fastify/app.js ./ +COPY fastify/dsm.js ./ +COPY fastify/rasp.js ./ +COPY fastify/debugger ./debugger +COPY fastify/iast ./iast +COPY fastify/integrations ./integrations +COPY fastify/resources ./resources +COPY fastify/package.json ./ +COPY fastify/bun.lock ./ +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && node nft-prune.mjs app.js \ && rm -rf /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/fastify.base.Dockerfile -t datadog/system-tests:fastify.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/fastify.base.Dockerfile -t datadog/system-tests:fastify.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:fastify.base-v3 diff --git a/utils/build/docker/nodejs/nextjs.base.Dockerfile b/utils/build/docker/nodejs/nextjs.base.Dockerfile index 7aae00023a5..4a69ff707b0 100644 --- a/utils/build/docker/nodejs/nextjs.base.Dockerfile +++ b/utils/build/docker/nodejs/nextjs.base.Dockerfile @@ -8,9 +8,10 @@ RUN node --version && npm --version && bun --version && curl --version WORKDIR /usr/app -COPY utils/build/docker/nodejs/nextjs/package.json utils/build/docker/nodejs/nextjs/bun.lock ./ -COPY utils/build/docker/nodejs/nextjs /usr/app -COPY utils/build/docker/nodejs/nft-prune.mjs ./ +COPY nextjs/package.json ./ +COPY nextjs/bun.lock ./ +COPY nextjs /usr/app +COPY nft-prune.mjs ./ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ && bun run build \ && node nft-prune.mjs \ @@ -24,5 +25,5 @@ RUN bun install --frozen-lockfile --network-concurrency 8 --linker=hoisted \ node_modules/next/dist/compiled/terser \ && rm -rf .next/cache /root/.bun -# docker build --progress=plain -f utils/build/docker/nodejs/nextjs.base.Dockerfile -t datadog/system-tests:nextjs.base-v3 . +# docker build --progress=plain -f utils/build/docker/nodejs/nextjs.base.Dockerfile -t datadog/system-tests:nextjs.base-v3 utils/build/docker/nodejs # docker push datadog/system-tests:nextjs.base-v3 diff --git a/utils/build/docker/php/apache-mod.base.Dockerfile b/utils/build/docker/php/apache-mod.base.Dockerfile index 13af5872d0e..fb45008228c 100644 --- a/utils/build/docker/php/apache-mod.base.Dockerfile +++ b/utils/build/docker/php/apache-mod.base.Dockerfile @@ -10,8 +10,11 @@ ENV DD_TRACE_HEADER_TAGS=user-agent EXPOSE 7777/tcp -ADD binaries* /binaries/ -ADD utils/build/docker/php /tmp/php +# build.sh only reads from these three subdirectories (see apache-mod/build.sh); keep +# this list in sync with it, since it drives the base-image content hash. +COPY apache-mod /tmp/php/apache-mod +COPY weblogs /tmp/php/weblogs +COPY common /tmp/php/common RUN chmod +x /tmp/php/apache-mod/build.sh RUN /tmp/php/apache-mod/build.sh @@ -22,3 +25,6 @@ ENTRYPOINT [] RUN echo "#!/bin/bash\ndumb-init /entrypoint.sh" > app.sh RUN chmod +x app.sh CMD [ "./app.sh" ] + +# docker build --progress=plain -f utils/build/docker/php/apache-mod.base.Dockerfile -t datadog/system-tests:apache-mod-8.0.base utils/build/docker/php +# docker push datadog/system-tests:apache-mod-8.0.base diff --git a/utils/build/docker/php/docker-bake.hcl b/utils/build/docker/php/docker-bake.hcl index 880178b38b9..39557ee91f4 100644 --- a/utils/build/docker/php/docker-bake.hcl +++ b/utils/build/docker/php/docker-bake.hcl @@ -32,161 +32,181 @@ group "default" { ] } +target "_common" { + context = "." +} + target "apache-mod-7_0" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.0.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.0.base"] } target "apache-mod-7_1" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.1.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.1.base"] } target "apache-mod-7_2" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.2.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.2.base"] } target "apache-mod-7_3" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.3.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.3.base"] } target "apache-mod-7_4" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.4.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.4.base"] } target "apache-mod-8_0" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.0.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.0.base"] } target "apache-mod-8_1" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.1.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.1.base"] } target "apache-mod-8_2" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.2.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.2.base"] } target "apache-mod-7_0-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.0-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.0-zts.base"] } target "apache-mod-7_1-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.1-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.1-zts.base"] } target "apache-mod-7_2-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.2-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.2-zts.base"] } target "apache-mod-7_3-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.3-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.3-zts.base"] } target "apache-mod-7_4-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.4-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.4-zts.base"] } target "apache-mod-8_0-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.0-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.0-zts.base"] } target "apache-mod-8_1-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.1-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.1-zts.base"] } target "apache-mod-8_2-zts" { - dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" + inherits = ["_common"] + dockerfile = "apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.2-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.2-zts.base"] } target "php-fpm-7_0" { - context = "utils/build/docker/php/" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.0" } - tags = ["datadog/system-tests:php-fpm-7.0.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.0.base"] } target "php-fpm-7_1" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.1" } - tags = ["datadog/system-tests:php-fpm-7.1.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.1.base"] } target "php-fpm-7_2" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.2" } - tags = ["datadog/system-tests:php-fpm-7.2.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.2.base"] } target "php-fpm-7_3" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.3" } - tags = ["datadog/system-tests:php-fpm-7.3.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.3.base"] } target "php-fpm-7_4" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.4" } - tags = ["datadog/system-tests:php-fpm-7.4.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.4.base"] } target "php-fpm-8_0" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.0" } - tags = ["datadog/system-tests:php-fpm-8.0.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.0.base"] } target "php-fpm-8_1" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.1" } - tags = ["datadog/system-tests:php-fpm-8.1.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.1.base"] } target "php-fpm-8_2" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.2" } - tags = ["datadog/system-tests:php-fpm-8.2.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.2.base"] } target "php-fpm-8_5" { - context = "utils/build/docker/php" + inherits = ["_common"] dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.5" } - tags = ["datadog/system-tests:php-fpm-8.5.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.5.base"] } diff --git a/utils/build/docker/php/php-fpm.base.Dockerfile b/utils/build/docker/php/php-fpm.base.Dockerfile index ac981cc100f..2b79f6f912e 100644 --- a/utils/build/docker/php/php-fpm.base.Dockerfile +++ b/utils/build/docker/php/php-fpm.base.Dockerfile @@ -3,7 +3,12 @@ FROM ubuntu:24.04 ARG PHP_VERSION ENV PHP_VERSION=${PHP_VERSION} -ADD . /tmp/php +# build.sh only reads from these four subdirectories (see php-fpm/build.sh); keep this +# list in sync with it, since it drives the base-image content hash. +COPY apt-sources.d /tmp/php/apt-sources.d +COPY weblogs /tmp/php/weblogs +COPY php-fpm /tmp/php/php-fpm +COPY common /tmp/php/common RUN chmod +x /tmp/php/php-fpm/build.sh RUN /tmp/php/php-fpm/build.sh $PHP_VERSION @@ -13,3 +18,6 @@ ENV DD_TRACE_ENABLED=1 ENV DD_TRACE_GENERATE_ROOT_SPAN=1 ENV DD_TRACE_AGENT_FLUSH_AFTER_N_REQUESTS=0 ENV DD_TRACE_HEADER_TAGS=user-agent + +# docker build --progress=plain --build-arg PHP_VERSION=8.2 -f utils/build/docker/php/php-fpm.base.Dockerfile -t datadog/system-tests:php-fpm-8.2.base utils/build/docker/php +# docker push datadog/system-tests:php-fpm-8.2.base diff --git a/utils/build/docker/php/php-fpm/build.sh b/utils/build/docker/php/php-fpm/build.sh index c73bd0d6f3e..8a00683520c 100644 --- a/utils/build/docker/php/php-fpm/build.sh +++ b/utils/build/docker/php/php-fpm/build.sh @@ -31,7 +31,7 @@ nala update printf '#!/bin/sh\n\nexit 101\n' > /usr/sbin/policy-rc.d chmod +x /usr/sbin/policy-rc.d -nala install -y --no-install-recommends tzdata publicsuffix curl apache2 libapache2-mod-fcgid jq ca-certificates git unzip php$PHP_VERSION-fpm php$PHP_VERSION-curl apache2 php$PHP_VERSION-mysql php$PHP_VERSION-pgsql php$PHP_VERSION-xml php$PHP_VERSION-mongodb +nala install -y --no-install-recommends tzdata publicsuffix curl apache2 libapache2-mod-fcgid jq ca-certificates git unzip php$PHP_VERSION-fpm php$PHP_VERSION-curl apache2 php$PHP_VERSION-mysql php$PHP_VERSION-pgsql php$PHP_VERSION-xml php$PHP_VERSION-mongodb php$PHP_VERSION-mbstring rm -rf /usr/sbin/policy-rc.d rm -rf /var/lib/apt/lists/* diff --git a/utils/build/docker/php/weblog_metadata.yml b/utils/build/docker/php/weblog_metadata.yml new file mode 100644 index 00000000000..69bb692b872 --- /dev/null +++ b/utils/build/docker/php/weblog_metadata.yml @@ -0,0 +1 @@ +# docs/understand/weblogs/weblog-metadata.md diff --git a/utils/build/docker/python/django-poc.base.Dockerfile b/utils/build/docker/python/django-poc.base.Dockerfile index e2a360ce3aa..3c7a7936958 100644 --- a/utils/build/docker/python/django-poc.base.Dockerfile +++ b/utils/build/docker/python/django-poc.base.Dockerfile @@ -8,8 +8,8 @@ RUN python --version && curl --version # install python deps ENV PIP_ROOT_USER_ACTION=ignore -COPY utils/build/docker/python/django/requirements-django-poc.txt /tmp/django-requirements.txt +COPY django/requirements-django-poc.txt /tmp/django-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/django-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v11 . +# docker build --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v11 utils/build/docker/python # docker push datadog/system-tests:django-poc.base-v11 diff --git a/utils/build/docker/python/django-py3.13.base.Dockerfile b/utils/build/docker/python/django-py3.13.base.Dockerfile index 47a08691905..a077a7a06b9 100644 --- a/utils/build/docker/python/django-py3.13.base.Dockerfile +++ b/utils/build/docker/python/django-py3.13.base.Dockerfile @@ -7,8 +7,8 @@ RUN apt-get update && apt-get install -y curl RUN python --version && curl --version # install python deps ENV PIP_ROOT_USER_ACTION=ignore -COPY utils/build/docker/python/django/requirements-django-py3.13.txt /tmp/django-requirements.txt +COPY django/requirements-django-py3.13.txt /tmp/django-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/django-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/django-py3.13.base.Dockerfile -t datadog/system-tests:django-py3.13.base-v10 . +# docker build --progress=plain -f utils/build/docker/python/django-py3.13.base.Dockerfile -t datadog/system-tests:django-py3.13.base-v10 utils/build/docker/python # docker push datadog/system-tests:django-py3.13.base-v10 diff --git a/utils/build/docker/python/docker-bake.hcl b/utils/build/docker/python/docker-bake.hcl index 3e3f0e8af99..418d625e204 100644 --- a/utils/build/docker/python/docker-bake.hcl +++ b/utils/build/docker/python/docker-bake.hcl @@ -1,4 +1,8 @@ # Docker Buildx bake file for python base images +# +# `context` is always this directory: base image Dockerfiles only COPY files from +# here, so paths in the Dockerfile are relative to it (see build_base_images.py, +# which derives base_image_dependencies from these COPY instructions). group "default" { targets = [ @@ -12,44 +16,48 @@ group "default" { ] } +target "_common" { + context = "." +} + target "django-py3_13" { - context = "." - dockerfile = "utils/build/docker/python/django-py3.13.base.Dockerfile" - tags = ["datadog/system-tests:django-py3.13.base-v10"] + inherits = ["_common"] + dockerfile = "django-py3.13.base.Dockerfile" + tags = ["datadog/system-tests:django-py3.13.base"] } target "fastapi" { - context = "." - dockerfile = "utils/build/docker/python/fastapi.base.Dockerfile" - tags = ["datadog/system-tests:fastapi.base-v9"] + inherits = ["_common"] + dockerfile = "fastapi.base.Dockerfile" + tags = ["datadog/system-tests:fastapi.base"] } target "python3_12" { - context = "." - dockerfile = "utils/build/docker/python/python3.12.base.Dockerfile" - tags = ["datadog/system-tests:python3.12.base-v13"] + inherits = ["_common"] + dockerfile = "python3.12.base.Dockerfile" + tags = ["datadog/system-tests:python3.12.base"] } target "django-poc" { - context = "." - dockerfile = "utils/build/docker/python/django-poc.base.Dockerfile" - tags = ["datadog/system-tests:django-poc.base-v11"] + inherits = ["_common"] + dockerfile = "django-poc.base.Dockerfile" + tags = ["datadog/system-tests:django-poc.base"] } target "flask-poc" { - context = "." - dockerfile = "utils/build/docker/python/flask-poc.base.Dockerfile" - tags = ["datadog/system-tests:flask-poc.base-v14"] + inherits = ["_common"] + dockerfile = "flask-poc.base.Dockerfile" + tags = ["datadog/system-tests:flask-poc.base"] } target "uwsgi-poc" { - context = "." - dockerfile = "utils/build/docker/python/uwsgi-poc.base.Dockerfile" - tags = ["datadog/system-tests:uwsgi-poc.base-v10"] + inherits = ["_common"] + dockerfile = "uwsgi-poc.base.Dockerfile" + tags = ["datadog/system-tests:uwsgi-poc.base"] } target "tornado" { - context = "." - dockerfile = "utils/build/docker/python/tornado.base.Dockerfile" - tags = ["datadog/system-tests:tornado.base-v2"] + inherits = ["_common"] + dockerfile = "tornado.base.Dockerfile" + tags = ["datadog/system-tests:tornado.base"] } diff --git a/utils/build/docker/python/fastapi.base.Dockerfile b/utils/build/docker/python/fastapi.base.Dockerfile index 87701020efd..e676ee7ffba 100644 --- a/utils/build/docker/python/fastapi.base.Dockerfile +++ b/utils/build/docker/python/fastapi.base.Dockerfile @@ -7,11 +7,11 @@ RUN apt-get update && apt-get install -y curl RUN python --version && curl --version # install python deps -COPY utils/build/docker/python/fastapi/requirements-fastapi.txt /tmp/fastapi-requirements.txt +COPY fastapi/requirements-fastapi.txt /tmp/fastapi-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/fastapi-requirements.txt RUN mkdir app WORKDIR /app -# docker build --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v8 . +# docker build --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v8 utils/build/docker/python # docker push datadog/system-tests:fastapi.base-v8 diff --git a/utils/build/docker/python/flask-poc.base.Dockerfile b/utils/build/docker/python/flask-poc.base.Dockerfile index 58dfa951615..1af788d1d77 100644 --- a/utils/build/docker/python/flask-poc.base.Dockerfile +++ b/utils/build/docker/python/flask-poc.base.Dockerfile @@ -18,8 +18,8 @@ RUN apt-get update \ RUN apt update && apt install -y pkg-config default-libmysqlclient-dev pkg-config # install python deps (flask is pinned below because tracer does not support >=2.3.0) -COPY utils/build/docker/python/flask/requirements-flask-poc.txt /tmp/flask-requirements.txt +COPY flask/requirements-flask-poc.txt /tmp/flask-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/flask-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v14 . +# docker build --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v14 utils/build/docker/python # docker push datadog/system-tests:flask-poc.base-v14 diff --git a/utils/build/docker/python/python3.12.base.Dockerfile b/utils/build/docker/python/python3.12.base.Dockerfile index 6c6343ab632..6b72290dc6e 100644 --- a/utils/build/docker/python/python3.12.base.Dockerfile +++ b/utils/build/docker/python/python3.12.base.Dockerfile @@ -8,9 +8,9 @@ RUN python --version && curl --version # install python deps ENV PIP_ROOT_USER_ACTION=ignore -COPY utils/build/docker/python/django/requirements-python3.12.txt /tmp/django-requirements.txt +COPY django/requirements-python3.12.txt /tmp/django-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/django-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v13 . +# docker build --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v13 utils/build/docker/python # docker push datadog/system-tests:python3.12.base-v13 diff --git a/utils/build/docker/python/tornado.base.Dockerfile b/utils/build/docker/python/tornado.base.Dockerfile index 2af2fff6569..d62563746d4 100644 --- a/utils/build/docker/python/tornado.base.Dockerfile +++ b/utils/build/docker/python/tornado.base.Dockerfile @@ -7,11 +7,11 @@ RUN apt-get update && apt-get install -y curl RUN python --version && curl --version # install python deps -COPY utils/build/docker/python/tornado/requirements-tornado.txt /tmp/tornado-requirements.txt +COPY tornado/requirements-tornado.txt /tmp/tornado-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/tornado-requirements.txt RUN mkdir app WORKDIR /app -# docker build --progress=plain -f utils/build/docker/python/tornado.base.Dockerfile -t datadog/system-tests:tornado.base-v1 . +# docker build --progress=plain -f utils/build/docker/python/tornado.base.Dockerfile -t datadog/system-tests:tornado.base-v1 utils/build/docker/python # docker push datadog/system-tests:tornado.base-v1 diff --git a/utils/build/docker/python/uwsgi-poc.base.Dockerfile b/utils/build/docker/python/uwsgi-poc.base.Dockerfile index 9a3739ecc9b..16fbc9ff4df 100644 --- a/utils/build/docker/python/uwsgi-poc.base.Dockerfile +++ b/utils/build/docker/python/uwsgi-poc.base.Dockerfile @@ -11,8 +11,8 @@ RUN apt update && apt install -y pkg-config default-libmysqlclient-dev pkg-confi # install python deps # Tracer does not support flask 2.3.0 or higher, pin the flask version in the requirements file. -COPY utils/build/docker/python/flask/requirements-uwsgi-poc.txt /tmp/flask-requirements.txt +COPY flask/requirements-uwsgi-poc.txt /tmp/flask-requirements.txt RUN pip install --upgrade pip && pip install -r /tmp/flask-requirements.txt -# docker build --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v10 . +# docker build --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v10 utils/build/docker/python # docker push datadog/system-tests:uwsgi-poc.base-v10 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 2c596c89566..475fcf4e638 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -155,6 +155,9 @@ system_tests_build_pipeline: - job: mirror_images optional: true artifacts: false + - job: build_base_images + optional: true + artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$SYSTEM_TESTS_FORCE_EXECUTE" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$SYSTEM_TESTS_SKIP_EMPTY_SCENARIO" diff --git a/utils/scripts/build-base-image.py b/utils/scripts/build-base-image.py deleted file mode 100755 index f52d9c3ab8b..00000000000 --- a/utils/scripts/build-base-image.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import subprocess -import sys - -from utils._context.weblog_metadata import WeblogMetaData - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build a weblog base image") - parser.add_argument("library", help="Library name (e.g. nodejs, python)") - parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") - args = parser.parse_args() - - weblogs = WeblogMetaData.load(args.library) - names = [w.name for w in weblogs] - - if args.weblog not in names: - print(f"Error: weblog '{args.weblog}' not found for library '{args.library}'") - print(f"Available weblogs: {', '.join(sorted(names))}") - sys.exit(1) - - weblog = next(w for w in weblogs if w.name == args.weblog) - dockerfile = weblog.base_dockerfile - if dockerfile is None: - print(f"Error: no base Dockerfile found for {args.weblog}") - sys.exit(1) - - image_tag = weblog.base_image_tag - if image_tag is None: - print(f"Error: could not extract base image tag for {args.weblog}") - sys.exit(1) - - print(f"Building base image: {image_tag}") - subprocess.run(["docker", "build", "--progress=plain", "-f", str(dockerfile), "-t", image_tag, "."], check=True) - - print(f"\nDone. To push:\n docker push {image_tag}") - - -if __name__ == "__main__": - main() diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py new file mode 100644 index 00000000000..88f5a2fd2cd --- /dev/null +++ b/utils/scripts/build_base_images.py @@ -0,0 +1,353 @@ +"""Rebuild and push weblog base images whose content-hash tag is missing from Docker Hub. + +Usage (from a system-tests checkout, runner venv active): + + python utils/scripts/build_base_images.py + +For every library with a `utils/build/docker//docker-bake.hcl` file, and every target +in that file's "default" group: + + 1. Resolve the target's bake config via `docker buildx bake --print`. + 2. Parse the Dockerfile's `COPY` instructions for local, non-gitignored dependencies. + 3. Materialize those files, plus the Dockerfile, into an isolated build context (see "Safety + net" below). + 4. Hash the bake config (tags excluded) plus that materialized context. + 5. Append "-" to the bake file's base tag (e.g. "datadog/system-tests:express4.base"). + 6. Skip if that tag already exists on Docker Hub (`docker manifest inspect`); otherwise build + and push. + +Idempotent: never overwrites an existing tag, only creates new ones when dependencies change. + +Pass --dry-run to only print the computed tag and whether it already exists, without building +or pushing (useful to find the tag for a weblog Dockerfile's FROM clause after deps change). + +Base Dockerfile constraints +---------------------------- +So the dependency list above can be derived mechanically from the Dockerfile alone, every +`.base.Dockerfile` built by this script must follow these rules: + + - No `ADD`. Use `COPY` only (no whole-context copies, no remote URLs); wildcards are allowed + (see "Wildcard sources" below). + - Every `COPY` has exactly one source: `COPY [flags] `. + - The bake target's `context` is always the Dockerfile's own directory, so `COPY` sources are + plain paths relative to it (`COPY app.js .`, not `COPY utils/build/docker/nodejs/fastify/app.js .`). + - No `RUN --mount`: those mounts read paths this script cannot see, so they'd silently escape + the derived dependency list. + +`COPY --from=` is exempt from these rules and skipped when deriving dependencies +(not a local repository path). + +Wildcard sources +----------------- +A `COPY` source is resolved with `Path.glob()` against the Dockerfile's own directory (a plain +path just matches itself). A pattern matching zero files raises (likely a typo); a pattern +matching *fewer* files than intended cannot be detected and silently ships an incomplete image. +The materialized build context's file list is printed before every build to make that case easy +to catch by eye. + +Safety net: isolated build context +----------------------------------- +Every detected dependency is hardlinked (falling back to a copy, e.g. across filesystems) into +`.base_image_build///`, and the image is built from that directory instead of +the real one. This way, a Dockerfile reference the parser failed to detect as a dependency makes +the build fail loudly ("file not found") instead of silently succeeding against the full +repository checkout. The content hash is computed from this same materialized directory (see +`compute_hash`), not a separate read of the dependency paths, so the hashed files and the built +files are identical by construction. +""" + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# So `python utils/scripts/build_base_images.py` works regardless of the caller's cwd. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from utils.const import COMPONENT_GROUPS # noqa: E402 + +BUILD_CONTEXT_ROOT = REPO_ROOT / ".base_image_build" + +_SOURCE_AND_DEST_TOKEN_COUNT = 2 + + +def _bake_file(library: str) -> Path: + return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" + + +def _run(cmd: list[str], *, cwd: Path = REPO_ROOT) -> subprocess.CompletedProcess: + """Run `cmd`, printing stderr (and stdout, if any) before raising on failure.""" + result = subprocess.run(cmd, cwd=cwd, check=False, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: command failed: {' '.join(cmd)}") + if result.stdout: + print(result.stdout) + if result.stderr: + print(result.stderr) + result.check_returncode() + return result + + +def _all_bake_configs(bake_file: Path) -> dict[str, dict]: + """Resolved bake config (context, dockerfile, args, tags) for every target in the bake + file's "default" group. + """ + result = _run(["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), "default"]) + return json.loads(result.stdout)["target"] + + +def _files_under(context_root: Path, path: Path) -> list[Path]: + """Every non-gitignored file under `path` (a file or directory, relative to `context_root`), + itself relative to `context_root`, sorted. + + Filters on `.gitignore` alone, not on git-tracked status, since this script runs against a + real checkout where untracked-but-not-ignored files are still valid dependencies. + + Raises if `path` exists on disk but every file under it is gitignored, rather than silently + producing an empty dependency. + """ + result = _run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "--", str(path)], cwd=context_root) + files = sorted(Path(line) for line in result.stdout.splitlines() if line) + if not files: + raise ValueError( + f"{context_root / path}: matched by a COPY source, but every file under it is " + "gitignored (or it is not inside a git repository at all)" + ) + return files + + +def _dockerfile_logical_lines(dockerfile: Path) -> list[str]: + """Dockerfile lines with backslash-continuations joined into one logical line each, and + full-line comments/blank lines dropped. + """ + logical_lines: list[str] = [] + buffer = "" + for raw_line in dockerfile.read_text().splitlines(): + line = raw_line.rstrip() + stripped = line.strip() + if not buffer and (not stripped or stripped.startswith("#")): + continue + if line.endswith("\\"): + buffer += line[:-1] + " " + continue + buffer += line + logical_lines.append(buffer.strip()) + buffer = "" + if buffer: # pragma: no cover - malformed Dockerfile ending on a continuation + logical_lines.append(buffer.strip()) + return logical_lines + + +def parse_copy_dependencies(dockerfile: Path) -> list[str]: + """Local `COPY` source paths (relative to `dockerfile`'s own directory) that a base + Dockerfile depends on. Enforces this module's Dockerfile constraints: no `ADD`, no + `RUN --mount`, and every `COPY` has exactly one source. + """ + dependencies: list[str] = [] + for line in _dockerfile_logical_lines(dockerfile): + instruction, _, rest = line.partition(" ") + instruction = instruction.upper() + + if instruction == "ADD": + raise ValueError(f"{dockerfile}: ADD is not allowed in a base Dockerfile, use COPY instead: {line!r}") + + if instruction == "RUN" and "--mount" in rest: + raise ValueError(f"{dockerfile}: RUN --mount is not allowed in a base Dockerfile: {line!r}") + + if instruction != "COPY": + continue + + tokens = rest.split() + flags = [t for t in tokens if t.startswith("--")] + paths = [t for t in tokens if not t.startswith("--")] + + if any(flag.startswith("--from=") for flag in flags): + continue # copies from another build stage or an external image, not a local path + + if len(paths) != _SOURCE_AND_DEST_TOKEN_COUNT: + raise ValueError( + f"{dockerfile}: COPY must be of the form 'COPY [flags] ' " + f"(exactly one source), got: {line!r}" + ) + + source, _dest = paths + dependencies.append(source) + + return dependencies + + +def _dependency_paths(context_root: Path, dockerfile: Path) -> list[Path]: + """Every non-gitignored file (relative to `context_root`) that `dockerfile` depends on, + sorted and deduplicated. `context_root` is `dockerfile`'s own directory. + + Each COPY source is resolved with `Path.glob()` (a plain path just matches itself). A source + matching zero files raises (likely a typo); see this module's docstring ("Wildcard sources") + for why a pattern matching fewer files than intended can't be detected here. + + Every match must resolve to a path under `context_root` itself, per this module's Dockerfile + constraints (e.g. a `nodejs` base image can't depend on a file under + `utils/build/docker/python/`). + """ + files: set[Path] = set() + for source in parse_copy_dependencies(dockerfile): + matches = sorted(context_root.glob(source)) + if not matches: + raise ValueError(f"{dockerfile}: COPY source {source!r} matched no files") + + for match in matches: + resolved = match.resolve() + try: + relative = resolved.relative_to(context_root) + except ValueError: + raise ValueError( + f"{dockerfile}: COPY source {source!r} escapes the Dockerfile's context directory ({context_root})" + ) from None + files.update(_files_under(context_root, relative)) + return sorted(files) + + +def compute_hash(build_dir: Path, bake_config: dict) -> str: + """Content hash for a base image target: bake config (minus tags) + path and content of + every file in `build_dir` (the isolated build context already materialized by + `materialize_build_context`, including the Dockerfile itself). + """ + digest = hashlib.sha256() + + config_without_tags = {k: v for k, v in bake_config.items() if k != "tags"} + digest.update(json.dumps(config_without_tags, sort_keys=True).encode()) + + for file in sorted(p for p in build_dir.rglob("*") if p.is_file()): + digest.update(str(file.relative_to(build_dir)).encode()) + digest.update(file.read_bytes()) + + return digest.hexdigest()[:12] + + +def _link_or_copy(source: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, dest) + except OSError: + # e.g. source and dest are on different filesystems + shutil.copy2(source, dest) + + +def materialize_build_context( + library: str, target: str, context_root: Path, dockerfile: Path, dependencies: list[Path] +) -> Path: + """Hardlink (falling back to a copy) every file in `dependencies` (relative to + `context_root`, already flattened and deduplicated by `_dependency_paths`), plus `dockerfile` + itself, into a fresh `.base_image_build///` directory, mirroring each + file's path relative to `context_root`. Building from this directory instead of the real one + is the safety net described in this module's docstring. + """ + build_dir = BUILD_CONTEXT_ROOT / library / target + shutil.rmtree(build_dir, ignore_errors=True) + build_dir.mkdir(parents=True) + + for file in dependencies: + _link_or_copy(context_root / file, build_dir / file) + + _link_or_copy(dockerfile, build_dir / dockerfile.relative_to(context_root)) + + print(f"Build context for {library}/{target} ({build_dir}):") + for file in sorted(build_dir.rglob("*")): + if file.is_file(): + print(f" {file.relative_to(build_dir)}") + + return build_dir + + +def image_exists(tag: str) -> bool: + """Whether `tag` exists on the registry. `docker manifest inspect` exits non-zero both for + a genuinely missing tag and for unrelated failures (auth, network); print stderr either way + so a real failure isn't mistaken for a missing tag. + """ + result = subprocess.run( + ["docker", "manifest", "inspect", tag], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0 and result.stderr: + print(result.stderr.strip()) + return result.returncode == 0 + + +def build_and_push(bake_file: Path, target: str, tag: str, build_dir: Path, dockerfile_name: str) -> None: + print(f"Building and pushing {tag}") + _run( + [ + "docker", + "buildx", + "bake", + "--push", + "--progress=plain", + "--set", + f"{target}.tags={tag}", + "--set", + f"{target}.context={build_dir}", + "--set", + f"{target}.dockerfile={dockerfile_name}", + "-f", + str(bake_file), + target, + ] + ) + + +def process_library(library: str, *, dry_run: bool) -> None: + bake_file = _bake_file(library) + if not bake_file.exists(): + return + + for target, bake_config in _all_bake_configs(bake_file).items(): + # `docker buildx bake --print` may report `context` as absolute or relative. + context_root = (REPO_ROOT / bake_config["context"]).resolve() + dockerfile = context_root / bake_config["dockerfile"] + + dependencies = _dependency_paths(context_root, dockerfile) + + # Materialize before hashing, so the hash is computed from the exact files the build + # will see. + build_dir = materialize_build_context(library, target, context_root, dockerfile, dependencies) + + base_tag = bake_config["tags"][0] + content_hash = compute_hash(build_dir, bake_config) + tag = f"{base_tag}-{content_hash}" + + if dry_run: + state = "exists" if image_exists(tag) else "missing" + print(f"{library}/{target}: {tag} ({state})") + continue + + if image_exists(tag): + print(f"{tag} already exists, skipping") + continue + + build_and_push(bake_file, target, tag, build_dir, bake_config["dockerfile"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Rebuild and push weblog base images with a content-hash tag") + parser.add_argument("--library", help="Only process this library (default: all libraries)") + parser.add_argument( + "--dry-run", + action="store_true", + help="Only print the computed tag and whether it exists on Docker Hub; never build or push", + ) + args = parser.parse_args() + + libraries = [args.library] if args.library else sorted(COMPONENT_GROUPS.all) + for library in libraries: + process_library(library, dry_run=args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 4cdc6100b4e..ba5cb662714 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -206,8 +206,6 @@ def __init__( weblog_instance: int, scenarios_times: dict[str, float], build_time: float, - *, - build_base_images: bool, ): self.library = library self.weblog = weblog @@ -224,9 +222,6 @@ def __init__( # split mechanism self.build_time = build_time - self.build_base_images = build_base_images - """ Shall the end-to-end scenario rebuild the weblog base image for fully baked weblog """ - def serialize(self) -> dict: return { "runs_on": "ubuntu-latest", @@ -237,9 +232,10 @@ def serialize(self) -> dict: "scenarios": sorted(self.scenarios), "expected_job_time": self.expected_job_time + self.build_time, "binaries_artifact": self.weblog.artifact_name, + # only local weblogs build their base image inline; prebuild weblogs wait for it + # once in the dedicated build_end_to_end job (see _get_weblog_build_job). "build_weblog_base_image": self.weblog.build_mode == BuildMode.local - and self.build_base_images - and self.weblog.base_dockerfile is not None, + and self.weblog.base_image_tag is not None, } @property @@ -275,7 +271,6 @@ def split_for_parallel_execution(self, desired_execution_time: float) -> list["J weblog_instance=i + 1, scenarios_times={scenario: self._scenarios_times[scenario] for scenario in scenarios}, build_time=self.build_time, - build_base_images=self.build_base_images, ) ) @@ -314,8 +309,6 @@ def get_endtoend_definitions( maximum_parallel_jobs: int, unique_id: str, binaries_artifact: str, - *, - build_base_images: bool = False, ) -> dict: scenarios = scenario_map.get("endtoend", []) @@ -353,7 +346,6 @@ def get_endtoend_definitions( weblog_instance=1, scenarios_times=scenarios_times, build_time=_get_build_time(library, weblog, time_stats["build"]), - build_base_images=build_base_images, ) ) @@ -372,20 +364,18 @@ def get_endtoend_definitions( "endtoend_defs": { "parallel_enable": len(jobs) > 0, "parallel_weblogs": [ - _get_weblog_build_job(weblog, build_base_images=build_base_images) - for weblog in weblogs - if weblog.build_mode == BuildMode.prebuild + _get_weblog_build_job(weblog) for weblog in weblogs if weblog.build_mode == BuildMode.prebuild ], "parallel_jobs": [job.serialize() for job in jobs], } } -def _get_weblog_build_job(weblog: Weblog, *, build_base_images: bool) -> dict: +def _get_weblog_build_job(weblog: Weblog) -> dict: return { "name": weblog.name, "artifact_name": weblog.artifact_name, - "build_base_images": build_base_images and weblog.base_dockerfile is not None, + "build_weblog_base_image": weblog.base_image_tag is not None, } diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index c004f0621db..07836492c02 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,7 +43,6 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, - build_weblog_base_images: bool = False, ): # this data struture is a dict where: # the key is the workflow identifier @@ -88,7 +87,6 @@ def __init__( maximum_parallel_jobs=256, unique_id=self.unique_id, binaries_artifact=self.binaries_artifact, - build_base_images=build_weblog_base_images, ) self.data["parametric"] = { @@ -288,12 +286,6 @@ def _get_workflow_map( "--system-tests-dev-mode", type=str, help="true if running in system-tests CI, with the dev mode", default="" ) parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) - parser.add_argument( - "--build-weblog-base-images", - type=str, - help="Rebuild weblog base images", - default="", - ) args = parser.parse_args() @@ -313,5 +305,4 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, - build_weblog_base_images=args.build_weblog_base_images == "true", ).export(export_format=args.format, output=args.output) diff --git a/utils/scripts/wait_for_base_image.py b/utils/scripts/wait_for_base_image.py new file mode 100755 index 00000000000..6aeb658451e --- /dev/null +++ b/utils/scripts/wait_for_base_image.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Wait for a weblog's base image to be available on Docker Hub. + +Weblog base images (e.g. `datadog/system-tests:express4.base-`) are built and +pushed by a dedicated GitLab CI job (see `utils/scripts/build_base_images.py`), not by +GitHub Actions. There is no direct dependency mechanism between the two CI systems, so +this script simply polls `docker manifest inspect` for the tag declared in the weblog's +Dockerfile until it appears, or a timeout is reached. + +It never builds or pushes anything: updating the tag in the weblog's Dockerfile after a +base image dependency changes remains the contributor's responsibility (see +docs/edit/update-docker-images.md). + +Deliberately depends only on the standard library (no `utils` package, no venv) so it +can run as a plain CI step before the runner virtualenv is built. +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + + +def _base_image_tag(library: str, weblog: str) -> str | None: + """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" + dockerfile = Path(f"utils/build/docker/{library}/{weblog}.Dockerfile") + if not dockerfile.exists(): + print(f"Error: no Dockerfile found for weblog '{weblog}' in library '{library}'") + sys.exit(1) + + for line in dockerfile.read_text().splitlines(): + if line.startswith("FROM "): + image_name = line.split()[1] + return image_name if image_name.startswith("datadog/system-tests:") else None + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description="Wait for a weblog's base image to exist on Docker Hub") + parser.add_argument("library", help="Library name (e.g. nodejs, python)") + parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") + parser.add_argument("--timeout", type=int, default=900, help="Max time to wait, in seconds (default: 900)") + parser.add_argument("--poll-interval", type=int, default=30, help="Time between polls, in seconds (default: 30)") + args = parser.parse_args() + + image_tag = _base_image_tag(args.library, args.weblog) + + if image_tag is None: + print(f"{args.weblog} does not use a system-tests base image, nothing to wait for") + return + + print(f"Waiting for {image_tag} to be available on Docker Hub (timeout: {args.timeout}s)") + + deadline = time.monotonic() + args.timeout + while True: + result = subprocess.run( + ["docker", "manifest", "inspect", image_tag], + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + print(f"{image_tag} is available") + return + + if time.monotonic() >= deadline: + print(f"Error: timed out waiting for {image_tag}") + if result.stderr: + print(result.stderr.strip()) + sys.exit(1) + + print(f"{image_tag} not found yet, retrying in {args.poll_interval}s...") + time.sleep(args.poll_interval) + + +if __name__ == "__main__": + main()