From c4cfe5407985971a27c153692b85b16ceecc698c Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:35:19 +0900 Subject: [PATCH 1/2] Build MinIO and mc images from source for S3 compat CI --- .github/workflows/e2e-test.yml | 9 +++++++++ docker/minio/Dockerfile.mc | 15 +++++++++++++++ docker/minio/Dockerfile.minio | 15 +++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 docker/minio/Dockerfile.mc create mode 100644 docker/minio/Dockerfile.minio diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 8897a8a..381ca92 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -608,6 +608,15 @@ jobs: -f docker/elasticsearch/Dockerfile.amd64 docker/elasticsearch echo "ELASTICSEARCH_IMAGE=elasticsearch-kuromoji:amd64" >> "$GITHUB_ENV" + - name: Build MinIO images from source + if: env.MINIO_ENABLED == 'true' + working-directory: e2e-tests + run: | + docker build -t minio-source:local -f docker/minio/Dockerfile.minio docker/minio + docker build -t minio-mc-source:local -f docker/minio/Dockerfile.mc docker/minio + echo "MINIO_IMAGE=minio-source:local" >> "$GITHUB_ENV" + echo "MINIO_MC_IMAGE=minio-mc-source:local" >> "$GITHUB_ENV" + - name: Install Docker Compose run: | sudo curl -L "https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose diff --git a/docker/minio/Dockerfile.mc b/docker/minio/Dockerfile.mc new file mode 100644 index 0000000..00d1c7e --- /dev/null +++ b/docker/minio/Dockerfile.mc @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +# minio/mc is no longer published on Docker Hub; build the last release from source. +FROM golang:1.24-alpine AS build +ARG MC_RELEASE_TAG=RELEASE.2025-08-13T08-35-41Z +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MC_RELEASE_TAG}" https://github.com/minio/mc.git /src +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags "$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go "${MC_RELEASE_TAG#RELEASE.}")" \ + -o /out/mc . + +FROM alpine:3.22 +RUN apk add --no-cache ca-certificates +COPY --from=build /out/mc /usr/bin/mc +ENTRYPOINT ["/usr/bin/mc"] diff --git a/docker/minio/Dockerfile.minio b/docker/minio/Dockerfile.minio new file mode 100644 index 0000000..7a0a702 --- /dev/null +++ b/docker/minio/Dockerfile.minio @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +# minio/minio is no longer published on Docker Hub; build the last release from source. +FROM golang:1.24-alpine AS build +ARG MINIO_RELEASE_TAG=RELEASE.2025-10-15T17-29-55Z +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MINIO_RELEASE_TAG}" https://github.com/minio/minio.git /src +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags "$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go "${MINIO_RELEASE_TAG#RELEASE.}")" \ + -o /out/minio . + +FROM alpine:3.22 +RUN apk add --no-cache ca-certificates curl +COPY --from=build /out/minio /usr/bin/minio +ENTRYPOINT ["/usr/bin/minio"] From d94056779a47b6d65180a2917d18d72fc74a8b93 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:01:45 +0900 Subject: [PATCH 2/2] Build shared images once and distribute them to matrix jobs as an artifact --- .github/workflows/e2e-test.yml | 134 ++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 381ca92..b3c2bf2 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -105,7 +105,78 @@ on: - 'false' jobs: + build-images: + runs-on: ubuntu-latest + outputs: + wb_custom: ${{ steps.wb_params.outputs.branch != '' }} + steps: + - name: Checkout test repository + uses: actions/checkout@v4 + with: + path: e2e-tests + + - name: Build Elasticsearch image with Kuromoji (amd64) + working-directory: e2e-tests + run: | + docker build -t elasticsearch-kuromoji:amd64 \ + -f docker/elasticsearch/Dockerfile.amd64 docker/elasticsearch + mkdir -p ../prebuilt-images + docker save -o ../prebuilt-images/elasticsearch-kuromoji.tar elasticsearch-kuromoji:amd64 + + - name: Build MinIO images from source + working-directory: e2e-tests + run: | + docker build -t minio-source:local -f docker/minio/Dockerfile.minio docker/minio + docker build -t minio-mc-source:local -f docker/minio/Dockerfile.mc docker/minio + docker save -o ../prebuilt-images/minio.tar minio-source:local minio-mc-source:local + + - name: Parse WaterButler parameters + id: wb_params + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + printf '%s\n' "$PR_BODY" > /tmp/pr_body.txt + echo "repository=$(grep -oP '^-\s*WB_REPOSITORY:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true)" >> $GITHUB_OUTPUT + echo "branch=$(grep -oP '^-\s*WB_BRANCH:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true)" >> $GITHUB_OUTPUT + else + echo "repository=${{ github.event.inputs.wb_repository }}" >> $GITHUB_OUTPUT + echo "branch=${{ github.event.inputs.wb_branch }}" >> $GITHUB_OUTPUT + fi + + - name: Build custom WaterButler image + if: steps.wb_params.outputs.branch != '' + run: | + WB_REPO="${{ steps.wb_params.outputs.repository || 'RCOSDP/RDM-waterbutler' }}" + WB_BRANCH="${{ steps.wb_params.outputs.branch }}" + WB_BASE_BRANCH="develop" + + echo "Building custom WaterButler image from ${WB_REPO}@${WB_BRANCH}" + + # Clone base branch + git clone --depth=50 -b "$WB_BASE_BRANCH" "https://github.com/${WB_REPO}.git" RDM-waterbutler + cd RDM-waterbutler + + # Merge custom branch + git fetch origin "$WB_BRANCH" + git config user.name "github-actions[bot]" + git config user.email "RDM-e2e-test-nb@example.com" + git merge --no-ff FETCH_HEAD -m "Merge ${WB_BRANCH} for E2E tests" + + # Build Docker image + docker build -t rdm-waterbutler:custom . + cd .. + docker save -o prebuilt-images/rdm-waterbutler-custom.tar rdm-waterbutler:custom + + - name: Upload prebuilt images + uses: actions/upload-artifact@v4 + with: + name: prebuilt-images + path: prebuilt-images/ + retention-days: 7 + e2e-test: + needs: build-images runs-on: ubuntu-latest timeout-minutes: 120 strategy: @@ -601,21 +672,28 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build Elasticsearch image with Kuromoji (amd64) - working-directory: e2e-tests + - name: Download prebuilt images + uses: actions/download-artifact@v4 + with: + name: prebuilt-images + path: prebuilt-images + + - name: Load prebuilt images run: | - docker build -t elasticsearch-kuromoji:amd64 \ - -f docker/elasticsearch/Dockerfile.amd64 docker/elasticsearch + docker load -i prebuilt-images/elasticsearch-kuromoji.tar echo "ELASTICSEARCH_IMAGE=elasticsearch-kuromoji:amd64" >> "$GITHUB_ENV" - - name: Build MinIO images from source - if: env.MINIO_ENABLED == 'true' - working-directory: e2e-tests - run: | - docker build -t minio-source:local -f docker/minio/Dockerfile.minio docker/minio - docker build -t minio-mc-source:local -f docker/minio/Dockerfile.mc docker/minio - echo "MINIO_IMAGE=minio-source:local" >> "$GITHUB_ENV" - echo "MINIO_MC_IMAGE=minio-mc-source:local" >> "$GITHUB_ENV" + if [ "${MINIO_ENABLED}" = "true" ]; then + docker load -i prebuilt-images/minio.tar + echo "MINIO_IMAGE=minio-source:local" >> "$GITHUB_ENV" + echo "MINIO_MC_IMAGE=minio-mc-source:local" >> "$GITHUB_ENV" + fi + + if [ "${{ needs.build-images.outputs.wb_custom }}" = "true" ]; then + docker load -i prebuilt-images/rdm-waterbutler-custom.tar + echo "WB_IMAGE=rdm-waterbutler:custom" >> "$GITHUB_ENV" + fi + rm -rf prebuilt-images - name: Install Docker Compose run: | @@ -784,9 +862,6 @@ jobs: cat > /tmp/pr_body.txt << 'EOF' ${{ github.event.pull_request.body }} EOF - echo "WB_CUSTOM_REPO=$(grep -oP '^-\s*WB_REPOSITORY:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true)" >> $GITHUB_ENV - echo "WB_CUSTOM_BRANCH=$(grep -oP '^-\s*WB_BRANCH:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true)" >> $GITHUB_ENV - # Wiki parameters from PR body WIKI_ENABLED_OVERRIDE=$(grep -oP '^-\s*WIKI_ENABLED:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' | tr '[:upper:]' '[:lower:]' || true) WIKI_REPO_OVERRIDE=$(grep -oP '^-\s*WIKI_REPOSITORY:\s*\K.*$' /tmp/pr_body.txt | head -1 | tr -d ' \r' || true) @@ -805,41 +880,12 @@ jobs: echo "Y_WEBSOCKET_CUSTOM_REPO=${Y_WEBSOCKET_REPO_OVERRIDE}" >> $GITHUB_ENV echo "Y_WEBSOCKET_CUSTOM_BRANCH=${Y_WEBSOCKET_BRANCH_OVERRIDE}" >> $GITHUB_ENV else - echo "WB_CUSTOM_REPO=${{ github.event.inputs.wb_repository }}" >> $GITHUB_ENV - echo "WB_CUSTOM_BRANCH=${{ github.event.inputs.wb_branch }}" >> $GITHUB_ENV echo "WIKI_CUSTOM_REPO=${{ github.event.inputs.wiki_repository }}" >> $GITHUB_ENV echo "WIKI_CUSTOM_BRANCH=${{ github.event.inputs.wiki_branch }}" >> $GITHUB_ENV echo "Y_WEBSOCKET_CUSTOM_REPO=${{ github.event.inputs.y_websocket_repository }}" >> $GITHUB_ENV echo "Y_WEBSOCKET_CUSTOM_BRANCH=${{ github.event.inputs.y_websocket_branch }}" >> $GITHUB_ENV fi - - name: Build custom WaterButler image - if: env.WB_CUSTOM_BRANCH != '' - run: | - WB_REPO="${WB_CUSTOM_REPO:-RCOSDP/RDM-waterbutler}" - WB_BRANCH="${WB_CUSTOM_BRANCH}" - WB_BASE_BRANCH="develop" - - echo "Building custom WaterButler image from ${WB_REPO}@${WB_BRANCH}" - - # Clone base branch - git clone --depth=50 -b "$WB_BASE_BRANCH" "https://github.com/${WB_REPO}.git" RDM-waterbutler - cd RDM-waterbutler - - # Merge custom branch - git fetch origin "$WB_BRANCH" - git config user.name "github-actions[bot]" - git config user.email "RDM-e2e-test-nb@example.com" - git merge --no-ff FETCH_HEAD -m "Merge ${WB_BRANCH} for E2E tests" - - # Build Docker image - docker build -t rdm-waterbutler:custom . - cd .. - - # Override WB_IMAGE for subsequent steps - echo "WB_IMAGE=rdm-waterbutler:custom" >> $GITHUB_ENV - echo "Custom WaterButler image built successfully" - - name: Build Wiki bundle and OSF image if: env.WIKI_ENABLED == 'true' run: |