Base image build process

.github/workflows/ci.yml

@@ -86,7 +86,6 @@ jobs:
       _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }}
       _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
-      _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }}
       _enable_replay_scenarios: true
       _system_tests_dev_mode: ${{ matrix.version == 'dev' }}
       _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }}

.github/workflows/compute-workflow-parameters.yml

@@ -52,11 +52,6 @@ on:
         default: ''
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we build weblog base images"
-        default: false
-        required: false
-        type: boolean
 
     # Map the workflow outputs to job outputs
     outputs:
@@ -142,7 +137,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --output $GITHUB_OUTPUT
     - name: log
       run: |
@@ -156,7 +150,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --format json | jq
 
     - name: Extract library target branch

.github/workflows/run-end-to-end.yml

@@ -63,7 +63,7 @@ on:
         required: false
         type: boolean
       _build_weblog_base_image:
-        description: "Shall we build the weblog base image (only valid if weblog build is local)"
+        description: "Shall we wait for the weblog base image to be pushed by GitLab CI before building the weblog"
         default: false
         required: false
         type: boolean
@@ -146,11 +146,9 @@ jobs:
       if: inputs._build_lambda_proxy_image
       run: ./build.sh -i lambda-proxy
 
-    - name: Build weblog base image
+    - name: Wait for weblog base image
       if: inputs._build_weblog_base_image
-      run: |
-        source venv/bin/activate
-        ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }}
+      run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30
 
     - name: Pull images
       uses: ./.github/actions/pull_images

.github/workflows/system-tests.yml

@@ -44,11 +44,6 @@ on:
         default: 'custom'
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we rebuild weblog base images"
-        default: false
-        required: false
-        type: boolean
       _build_buddies_images:
         description: "Shall we build buddies images"
         default: false
@@ -154,7 +149,6 @@ jobs:
       binaries_artifact: ${{ inputs.binaries_artifact }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}
       _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }}
-      _build_weblog_base_images: ${{ inputs._build_weblog_base_images }}
 
   parametric:
     needs:
@@ -215,11 +209,9 @@ jobs:
         run: ls -la binaries/
       - name: Export github token to a file
         run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt"
-      - name: Build weblog base images
-        if: matrix.weblog.build_base_image
-        run: |
-          source venv/bin/activate
-          ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }}
+      - name: Wait for weblog base image
+        if: matrix.weblog.build_weblog_base_image
+        run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30
       - name: Build weblog
         id: build
         run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt"

.gitignore

@@ -74,6 +74,9 @@ version-check-output/
 
 tmp/
 
+# Materialized build context for utils/scripts/build_base_images.py
+/.base_image_build/
+
 # Go
 go.work*
 

.gitlab-ci.yml

@@ -474,3 +474,35 @@ mirror_images:
     - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror
   rules:
     - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null'
+
+# ──────────────────────────────────────────────
+# Rebuild weblog base images whose content changed.
+#
+# For every library with a `base_image_dependencies` section in
+# utils/build/docker/<library>/weblog_metadata.yml, computes a content-hash tag per
+# docker-bake.hcl target and pushes it if missing (see utils/scripts/build_base_images.py).
+# Runs on every push, on every branch: it is idempotent (only pushes new tags, never
+# overwrites existing ones), so it is safe to build ahead of merge. Since new tags never
+# replace old ones, a weblog Dockerfile's FROM clause must be updated by hand to pick up
+# a newly pushed base image (run the script with --dry-run to find the new tag).
+# ──────────────────────────────────────────────
+
+build_base_images:
+  image: $CI_IMAGE
+  tags:
+    - docker-in-docker:amd64
+  needs:
+    - job: build_ci_image
+      artifacts: false
+      optional: true
+  stage: e2e
+  before_script:
+    - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text)
+    - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text)
+    - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin
+    - ln -sf /system-tests/venv venv
+    - source venv/bin/activate
+  script:
+    - python utils/scripts/build_base_images.py
+  rules:
+    - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null'

docs/CI/github-actions.md

@@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI
 | `_build_buddies_images`               | Shall we build buddies images                                        | boolean | false    | false      |
 | `_build_proxy_image`                  | Shall we build proxy image                                           | boolean | false    | false      |
 | `_build_lambda_proxy_image`           | Shall we build the lambda-proxy image                                | boolean | false    | false      |
-| `_build_weblog_base_images`           | Shall we build weblog base images                                    | boolean | false    | false      |
 | `_enable_replay_scenarios`            | Enable replay scenarios, should only be used in system-tests CI      | boolean | false    | false      |

docs/edit/update-docker-images.md

@@ -1,11 +1,22 @@
 Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/).
 
-If you need to update them, you will need to follow those
+For weblog base images (nodejs, python, php), the build and push are fully automated:
 
-1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code)
-2. create your PR, and add the relevant label to rebuild the image in the CI
-  * `build-python-base-images` for python weblogs
-  * `build-php-base-images` for PHP weblogs
-  * `build-nodejs-base-images` for Node.js weblogs
-  * `build-proxy-image` for proxy image
-3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack)
+1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every
+   branch. For each target in a library's `docker-bake.hcl`, it derives the target's dependencies from the
+   `COPY` instructions in its `<name>.base.Dockerfile` (see `docs/understand/weblogs/weblog-metadata.md`
+   for the Dockerfile rules this relies on), computes a content hash of those dependencies, and, if a base
+   image tagged with that hash doesn't already exist on Docker Hub, builds and pushes it as
+   `<base tag>-<hash>`.
+2. If you changed a file referenced by a `.base.Dockerfile`'s `COPY` instructions, a new tag will
+   automatically be pushed by that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to
+   point to the new tag (you can find it by running `python utils/scripts/build_base_images.py --dry-run`
+   locally).
+3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for
+   the tag referenced in the weblog's `FROM` line to become available before building the weblog, via
+   `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag
+   before (or shortly after) your PR's GitHub CI run starts.
+
+For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a
+rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push
+your image to hub.docker.com (`#apm-shared-testing` on slack).

docs/understand/weblogs/weblog-metadata.md

@@ -45,3 +45,40 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`.
 `WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges:
 1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata).
 2. Explicit overrides from `weblog_metadata.yml`.
+
+## Base image dependencies
+
+Base images (built by the `build_base_images` CI job, `utils/scripts/build_base_images.py`) are
+declared in each library's `utils/build/docker/<library>/docker-bake.hcl`, one target per base
+image. There is no separate dependency list to maintain: for each target, the job parses the
+target's own `<name>.base.Dockerfile` and treats every `COPY` source as a dependency. This works
+because base Dockerfiles are required to follow a few rules that make that derivation
+unambiguous:
+
+- No `ADD` — use `COPY` for everything (no glob sources, no whole-directory copies, no remote
+  URLs).
+- Every `COPY` has exactly one source: `COPY [flags] <source> <dest>`.
+- The bake target's `context` is always the Dockerfile's own directory, so every `COPY` source
+  is a plain path relative to that directory.
+- No `RUN --mount` — a bind/cache/secret mount reads from a path the script can't see, so it
+  would silently escape the derived dependency list.
+
+(`COPY --from=<stage-or-image>` is unaffected: it isn't a local repository path, so it's skipped.)
+
+The job computes a content hash from the resolved `docker-bake.hcl` target config, the target's
+Dockerfile, and every git-tracked file under each derived dependency path, then pushes the base
+image to Docker Hub tagged `<base-tag>-<hash12>` if that tag doesn't already exist. It never
+overwrites an existing tag, so weblog Dockerfiles that `FROM` a base image must have their tag
+updated by hand after a new one is pushed (run the script with `--dry-run` to find the current
+tag for each target).
+
+As a safety net, before building, every derived dependency is hardlinked (or copied, if
+hardlinking isn't possible) into an isolated build context under `.base_image_build/`, and the
+image is built from that directory instead of the real one. This way, if the Dockerfile
+references a file the parser failed to recognize as a dependency, the build fails loudly
+("file not found") instead of silently succeeding against the full checkout — which would leave
+the tag's content hash stale without anyone noticing.
+
+GitHub Actions never builds these base images itself: `utils/scripts/wait_for_base_image.py`
+polls Docker Hub for the tag currently referenced in the weblog's `FROM` line (with a timeout)
+before building the weblog, since GitLab CI is the only pipeline that builds and pushes them.