[WIP] Python・依存ライブラリの更新とコンテナイメージのマルチステージ化

.dockerignore

@@ -2,3 +2,4 @@
 *.pyc
 **/*.pyc
 Dockerfile
+*venv*

.github/scripts/ci/render-trivy-report.py

@@ -0,0 +1,128 @@
+import argparse
+import json
+from collections import Counter
+
+
+SEVERITIES = ('CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'UNKNOWN')
+SEVERITY_ORDER = {severity: index for index, severity in enumerate(SEVERITIES)}
+COMMENT_MARKER = '<!-- trivy-image-scan -->'
+
+
+def markdown_escape(value):
+    return str(value or '-').replace('\n', ' ').replace('|', '\\|')
+
+
+def get_vulnerabilities(report):
+    return [
+        vulnerability
+        for result in report.get('Results', [])
+        for vulnerability in result.get('Vulnerabilities') or []
+    ]
+
+
+def render_report(report, image, commit, run_url):
+    vulnerabilities = get_vulnerabilities(report)
+    counts = Counter(
+        vulnerability.get('Severity', 'UNKNOWN').upper()
+        for vulnerability in vulnerabilities
+    )
+    fixable_counts = Counter(
+        vulnerability.get('Severity', 'UNKNOWN').upper()
+        for vulnerability in vulnerabilities
+        if vulnerability.get('FixedVersion')
+    )
+
+    lines = [
+        COMMENT_MARKER,
+        '## Container image vulnerability scan',
+        '',
+        f'- Image: `{markdown_escape(image)}`',
+        f'- Commit: [`{commit[:12]}`]({run_url.rsplit("/actions/runs/", 1)[0]}/commit/{commit})',
+        f'- Workflow run: [details]({run_url})',
+        f'- Findings: **{len(vulnerabilities)}** '
+        f'({sum(fixable_counts.values())} with a fix available)',
+        '',
+        '| Severity | Total | Fix available |',
+        '| --- | ---: | ---: |',
+    ]
+    for severity in SEVERITIES:
+        lines.append(
+            f'| {severity} | {counts[severity]} | {fixable_counts[severity]} |',
+        )
+
+    important = sorted(
+        (
+            vulnerability
+            for vulnerability in vulnerabilities
+            if vulnerability.get('Severity', 'UNKNOWN').upper() in {'CRITICAL', 'HIGH'}
+        ),
+        key=lambda vulnerability: (
+            SEVERITY_ORDER.get(
+                vulnerability.get('Severity', 'UNKNOWN').upper(),
+                len(SEVERITIES),
+            ),
+            vulnerability.get('VulnerabilityID', ''),
+            vulnerability.get('PkgName', ''),
+        ),
+    )
+    lines.extend([
+        '',
+        '### High-priority findings',
+        '',
+    ])
+    if important:
+        lines.extend([
+            '| Severity | Vulnerability | Package | Installed | Fixed |',
+            '| --- | --- | --- | --- | --- |',
+        ])
+        for vulnerability in important[:20]:
+            lines.append(
+                '| {severity} | {identifier} | {package} | {installed} | {fixed} |'.format(
+                    severity=markdown_escape(vulnerability.get('Severity')),
+                    identifier=markdown_escape(vulnerability.get('VulnerabilityID')),
+                    package=markdown_escape(vulnerability.get('PkgName')),
+                    installed=markdown_escape(vulnerability.get('InstalledVersion')),
+                    fixed=markdown_escape(vulnerability.get('FixedVersion')),
+                ),
+            )
+        if len(important) > 20:
+            lines.extend([
+                '',
+                f'_Showing 20 of {len(important)} HIGH/CRITICAL findings._',
+            ])
+    else:
+        lines.append('No HIGH or CRITICAL findings were detected.')
+
+    lines.extend([
+        '',
+        '> This scan is currently reporting-only and establishes the vulnerability baseline.',
+        '> The complete JSON and SARIF reports are attached to the workflow run.',
+        '',
+    ])
+    return '\n'.join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input', required=True)
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--image', required=True)
+    parser.add_argument('--commit', required=True)
+    parser.add_argument('--run-url', required=True)
+    args = parser.parse_args()
+
+    with open(args.input, encoding='utf-8') as report_file:
+        report = json.load(report_file)
+
+    rendered = render_report(
+        report,
+        image=args.image,
+        commit=args.commit,
+        run_url=args.run_url,
+    )
+    with open(args.output, 'w', encoding='utf-8') as output_file:
+        output_file.write(rendered)
+
+
+if __name__ == '__main__':
+    main()

.github/scripts/ci/run-tests.sh

@@ -12,16 +12,15 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
 cd "$REPO_ROOT"
 
 read -r -d '' container_script <<'BASH' || true
-export WHEELHOUSE=$HOME/wheelhouse
-pip install --upgrade pip
-pip install setuptools==37.0.0
-pip install wheel==0.26.0
-pip install invoke==0.13.0
-invoke wheelhouse --develop
-invoke install --develop
-invoke test
+set -euo pipefail
+apt-get update
+apt-get install -y --no-install-recommends git build-essential
+python -m ensurepip --default-pip
+pip install poetry==2.5.1
+poetry install --without=docs --with=dev
+poetry run invoke test
 BASH
 
-docker run --rm -t \
+docker run --rm \
     -e TEST_BUILD="$TEST_BUILD" \
-    ${MFR_TEST_IMAGE} bash -lc "$container_script"
+    "${MFR_TEST_IMAGE}" bash -lc "$container_script"

.github/workflows/test-build.yml

@@ -0,0 +1,75 @@
+name: mfr_ci_testing
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-24.04
+    env:
+      GHA_DISTRO: ubuntu-24.04
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    strategy:
+      matrix:
+        python-version: [3.13]
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache Build Requirements
+        id: pip-cache-step
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install poetry==2.5.1 setuptools==80.10.2
+          poetry install --no-root --without=docs --with=dev
+
+  runtests:
+    name: Run unit tests
+    needs: build
+    runs-on: ubuntu-24.04
+    env:
+      GHA_DISTRO: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: [3.13]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
+
+      - name: Install test dependencies
+        run: |
+          python -m pip install poetry==2.5.1
+          python -m pip install setuptools==80.10.2
+          poetry install --without=docs --with=dev
+
+      - name: Run flake8
+        run: poetry run flake8 .
+
+      - name: Run unit tests
+        run: |
+          poetry run pytest --cov-report term-missing --cov mfr tests
+
+      - name: Upload coverage to Coveralls
+        run: poetry run coveralls --service=github
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/unit-tests.yml

@@ -38,6 +38,117 @@ jobs:
         id: meta
         run: echo "image=rdm-mfr-test:${{ github.sha }}" >> "$GITHUB_OUTPUT"
 
+  scan-image:
+    runs-on: ubuntu-latest
+    needs: build-image
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: rdm-mfr-image
+          path: /tmp
+
+      - name: Scan Docker image with Trivy
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          scan-type: image
+          input: /tmp/rdm-mfr-image.tar
+          scanners: vuln
+          vuln-type: os,library
+          format: json
+          output: trivy-results.json
+          exit-code: '0'
+          list-all-pkgs: 'true'
+
+      - name: Convert Trivy report to SARIF
+        env:
+          TRIVY_SCANNERS: ''
+        run: |
+          trivy convert \
+            --format sarif \
+            --output trivy-results.sarif \
+            trivy-results.json
+
+      - name: Render vulnerability summary
+        env:
+          IMAGE: ${{ needs.build-image.outputs.image }}
+          COMMIT: ${{ github.sha }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          python3 .github/scripts/ci/render-trivy-report.py \
+            --input trivy-results.json \
+            --output trivy-report.md \
+            --image "$IMAGE" \
+            --commit "$COMMIT" \
+            --run-url "$RUN_URL"
+          tee -a "$GITHUB_STEP_SUMMARY" < trivy-report.md
+
+      - name: Upload vulnerability reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: trivy-image-scan
+          path: |
+            trivy-results.json
+            trivy-results.sarif
+            trivy-report.md
+          retention-days: 14
+
+      - name: Upload SARIF to GitHub Code Scanning
+        if: >-
+          github.event_name != 'pull_request' ||
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: github/codeql-action/upload-sarif@bce182f857edf1feab116e9795a3393d21977282 # v4
+        with:
+          sarif_file: trivy-results.sarif
+          category: trivy-container-image
+
+  comment-image-scan:
+    runs-on: ubuntu-latest
+    needs: scan-image
+    if: >-
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Download vulnerability summary
+        uses: actions/download-artifact@v4
+        with:
+          name: trivy-image-scan
+
+      - name: Create or update PR comment
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          set -euo pipefail
+          comments_endpoint="repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments"
+          comment_id="$({
+            gh api --paginate "$comments_endpoint" --jq \
+              '.[] | select(.user.login == "github-actions[bot]" and (.body | contains("<!-- trivy-image-scan -->"))) | .id'
+          } | sed -n '1p')"
+          if [[ -n "$comment_id" ]]; then
+            gh api \
+              --method PATCH \
+              "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" \
+              -F body=@trivy-report.md
+          else
+            gh api \
+              --method POST \
+              "$comments_endpoint" \
+              -F body=@trivy-report.md
+          fi
+
   tests:
     runs-on: ubuntu-latest
     needs: build-image

.mailmap

@@ -14,7 +14,6 @@ Eric Bower <neurosnap@gmail.com>
 Erin Braswell <erin.braswell@gmail.com> erinspace <erin.braswell@gmail.com>
 Fitz Elliott <fitz@cos.io> Fitz Elliott <felliott@fiskur.org>
 Haoyu Chen <hc6as@virginia.edu> haoyuchen1992 <hc6as@virginia.edu>
-Jeffrey Spies <jspies@gmail.com> Jeffrey Spies <github@jeffspies.com>
 John Tordoff <john@cos.io> Johnetordoff <Johnetordoff@users.noreply.github.com>
 Jonathon Love <jon@thon.cc>
 Joshua Carp <jm.carp@gmail.com>