diff --git a/build.sh b/build.sh index 38f1f7d4..d6fbb909 100755 --- a/build.sh +++ b/build.sh @@ -451,6 +451,111 @@ resolve_targets() { echo "${resolved[@]}" } +# Given a canonical project URL (or suffix), find all images whose +# sources.txt references it. Prints matching image targets, one per line. +# Used by the content provider to implement `s2i_ci_images: auto`. +# Args: [stream] +auto_detect() { + local needle="$1" + local stream="${2:-}" + local matches=() + + if [[ -z "${needle}" ]]; then + echo "ERROR: auto-detect requires a project URL or suffix" >&2 + return 1 + fi + + for sources_file in "${CONTAINERS_DIR}"/*/sources.txt \ + "${CONTAINERS_DIR}"/*/*/sources.txt; do + [[ -f "${sources_file}" ]] || continue + + while IFS=' ' read -r entry_stream name url _branch _hash; do + [[ -z "${entry_stream}" || "${entry_stream}" == \#* ]] && continue + [[ "${name}" == "upper-constraints" ]] && continue + [[ -n "${stream}" && "${entry_stream}" != "${stream}" ]] && continue + + # Match the needle against the URL's path (strip scheme + host, + # strip trailing .git). Supports full URLs and short forms like + # "openstack/tempest". + local url_path="${url#*://}" # remove scheme + url_path="${url_path#*/}" # remove hostname + url_path="${url_path%.git}" # remove .git suffix + if [[ "${url_path}" == "${needle}" || + "${url}" == "${needle}" || + "${url%.git}" == "${needle}" ]]; then + local rel="${sources_file#"${CONTAINERS_DIR}"/}" + rel="${rel%/sources.txt}" + local project="${rel%%/*}" + # Resolve to actual image targets under this project + for img in $(discover_images); do + if [[ "${img}" == "${project}/"* ]] || [[ "${img}" == "${project}" ]]; then + local already=0 + for m in "${matches[@]+"${matches[@]}"}"; do + [[ "${m}" == "${img}" ]] && already=1 && break + done + [[ ${already} -eq 0 ]] && matches+=("${img}") + fi + done + fi + done < "${sources_file}" + done + + if [[ ${#matches[@]} -eq 0 ]]; then + echo "ERROR: no images reference '${needle}'" >&2 + return 1 + fi + + printf '%s\n' "${matches[@]}" +} + +# List source dependencies for an image target. +# Outputs pipe-delimited records: name|canonical_project|url|dest_dir +# Used by Ansible playbooks to stage Zuul sources without re-parsing +# sources.txt themselves (build.sh is the single source of truth). +# Args: [stream] +list_sources() { + local target="$1" + local stream="${2:-${STREAM:-}}" + local project="${target%%/*}" + + if [[ -z "${target}" ]]; then + echo "ERROR: list-sources requires an image target" >&2 + return 1 + fi + + local sources_files=() + if [[ "${target}" == "base" ]]; then + [[ -f "${CONTAINERS_DIR}/base/sources.txt" ]] && \ + sources_files+=("${CONTAINERS_DIR}/base/sources.txt") + else + [[ -f "${CONTAINERS_DIR}/${project}/sources.txt" ]] && \ + sources_files+=("${CONTAINERS_DIR}/${project}/sources.txt") + [[ -f "${CONTAINERS_DIR}/${target}/sources.txt" ]] && \ + sources_files+=("${CONTAINERS_DIR}/${target}/sources.txt") + fi + + for sources_file in "${sources_files[@]}"; do + while IFS=' ' read -r entry_stream name url _branch _hash; do + [[ -z "${entry_stream}" || "${entry_stream}" == \#* ]] && continue + [[ "${name}" == "upper-constraints" ]] && continue + [[ -n "${stream}" && "${entry_stream}" != "${stream}" ]] && continue + + local url_path="${url#*://}" + url_path="${url_path#*/}" + url_path="${url_path%.git}" + + local dest_dir + if [[ "${target}" == "base" ]]; then + dest_dir="${CONTAINERS_DIR}/base/src/${name}" + else + dest_dir="${CONTAINERS_DIR}/${project}/src/${name}" + fi + + echo "${name}|${url_path}|${url}|${dest_dir}" + done < "${sources_file}" + done +} + # Clone a repo at a branch tip (or tag) and store the resolved commit hash # in _CLONE_RESULT. Must NOT be called via command substitution ($(...)) # because _AUTO_CLONED assignments would be lost in the subshell. @@ -1329,8 +1434,22 @@ case "${ACTION}" in list) list_images ;; + auto-detect) + if [[ ${#TARGETS[@]} -eq 0 || "${TARGETS[0]}" == "all" ]]; then + echo "Usage: $0 auto-detect [stream]" >&2 + exit 1 + fi + auto_detect "${TARGETS[0]}" "${TARGETS[1]:-}" + ;; + list-sources) + if [[ ${#TARGETS[@]} -eq 0 || "${TARGETS[0]}" == "all" ]]; then + echo "Usage: STREAM= $0 list-sources " >&2 + exit 1 + fi + list_sources "${TARGETS[0]}" "${TARGETS[1]:-}" + ;; *) - echo "Usage: STREAM= $0 {build|build-parallel|push|update-sources|update-lockfiles|install-deps|list} [target ...]" + echo "Usage: STREAM= $0 {build|build-parallel|push|update-sources|update-lockfiles|install-deps|list|auto-detect|list-sources} [target ...]" echo "" echo "Images (discovered from containers/):" for dir_name in $(discover_images); do diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 909bcfa0..36ce06d6 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -438,3 +438,67 @@ for a commit shares the same `master-` tag and consistent OS packages. STREAM=master ./build.sh update-sources STREAM=master ./build.sh build ``` + +## Speculative builds (Zuul integration) + +When a patch is submitted against an upstream OpenStack project (e.g., +`openstack/tempest`), the CI pipeline can automatically build fresh +container images incorporating that patch. This is called a **speculative +build**. + +### Auto-detection + +The content provider job accepts `s2i_ci_images: auto` to automatically +determine which images need rebuilding based on the triggering project: + +```yaml +# In a Zuul job definition +vars: + s2i_ci_images: auto +``` + +Under the hood, auto-detection: + +1. Inspects `zuul.items` to find the projects in the speculative change + queue. +2. Runs `build.sh auto-detect [stream]` for each, + which scans `sources.txt` files to find which images reference that + project. +3. Replaces `s2i_ci_images` with the de-duplicated list of affected + image targets. + +You can test auto-detection locally: + +```bash +# Which images would be rebuilt for a tempest patch? +PARALLEL=1 ./build.sh auto-detect openstack/tempest master + +# Full URL form also works +PARALLEL=1 ./build.sh auto-detect https://opendev.org/openstack/neutron.git +``` + +### Source staging + +After auto-detection resolves the image list, the pipeline stages Zuul's +source checkouts into the container build contexts. This replaces the +pinned source with the patched version so the built image includes the +speculative change. + +The staging playbook (`shared/stage-zuul-sources.yaml`) delegates all +`sources.txt` parsing to `build.sh list-sources`, keeping build.sh as the +single source of truth for the source manifest format. + +### Querying source dependencies + +`build.sh list-sources` prints pipe-delimited records for all source +dependencies of an image target: + +```bash +PARALLEL=1 ./build.sh list-sources tempest/tempest master +# Output: name|canonical_project|url|dest_dir +# tempest|openstack/tempest|https://opendev.org/openstack/tempest.git|/.../containers/tempest/src/tempest +# barbican-tempest-plugin|openstack/barbican-tempest-plugin|https://... +``` + +This is used by the Ansible playbooks but is also useful for debugging +which upstream repos feed into a given image. diff --git a/playbooks/container-ci/shared/resolve-auto-images.yaml b/playbooks/container-ci/shared/resolve-auto-images.yaml new file mode 100644 index 00000000..b8456950 --- /dev/null +++ b/playbooks/container-ci/shared/resolve-auto-images.yaml @@ -0,0 +1,72 @@ +--- +# Resolve `s2i_ci_images: auto` by scanning sources.txt files to find +# which images are affected by the projects in the Zuul workspace. +# +# Called from shared/run.yaml when s2i_ci_images == "auto". +# After this task file completes, s2i_ci_images is replaced with the +# de-duplicated list of detected image targets. +# +# Required variables (set by run.yaml before including this): +# s2i_ci_container_repo - path to the s2i-openstack-containers checkout +# s2i_ci_container_project - canonical name of the container project +# s2i_ci_stream - stream name (e.g. "master") + +- name: Collect changed projects from Zuul queue + ansible.builtin.set_fact: + s2i_ci_changed_projects: >- + {{ (zuul.items | default([])) + | selectattr('project', 'defined') + | map(attribute='project') + | map(attribute='canonical_name', default='') + | select('ne', '') + | reject('equalto', s2i_ci_container_project) + | unique + | list }} + +- name: Fall back to primary project when no queue items + ansible.builtin.set_fact: + s2i_ci_changed_projects: >- + {{ [zuul.project.canonical_name] + | reject('equalto', s2i_ci_container_project) + | list }} + when: s2i_ci_changed_projects | length == 0 + +- name: Run auto-detect for each changed project + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - auto-detect + - "{{ item }}" + - "{{ s2i_ci_stream }}" + args: + chdir: "{{ s2i_ci_container_repo }}" + loop: "{{ s2i_ci_changed_projects }}" + register: s2i_ci_auto_detect_results + changed_when: false + failed_when: false + +- name: Collect auto-detected images + ansible.builtin.set_fact: + s2i_ci_images: >- + {{ s2i_ci_auto_detect_results.results + | selectattr('rc', 'equalto', 0) + | map(attribute='stdout_lines') + | flatten + | unique + | list }} + +- name: Fail if no images were detected + ansible.builtin.assert: + that: + - s2i_ci_images | length > 0 + fail_msg: >- + Auto-detection found no images for projects: + {{ s2i_ci_changed_projects | join(', ') }}. + Verify that sources.txt references these projects. + +- name: Report auto-detected images + ansible.builtin.debug: + msg: >- + Auto-detected {{ s2i_ci_images | length }} image(s) for + {{ s2i_ci_changed_projects | join(', ') }}: + {{ s2i_ci_images | join(', ') }} diff --git a/playbooks/container-ci/shared/run.yaml b/playbooks/container-ci/shared/run.yaml new file mode 100644 index 00000000..303ae2d2 --- /dev/null +++ b/playbooks/container-ci/shared/run.yaml @@ -0,0 +1,350 @@ +--- +- name: Build and publish selected container images + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_workspace_root: "{{ zuul_user_dir | default(ansible_user_dir) }}" + + tasks: + - name: Default optional selective provider inputs + ansible.builtin.set_fact: + s2i_ci_namespace: "{{ s2i_ci_namespace | default('openstack') }}" + s2i_ci_tag: >- + {{ s2i_ci_tag | default( + (zuul.change | default('build') | string) ~ '-' ~ + (zuul.patchset | default('latest') | string) + ) }} + s2i_ci_stream: "{{ s2i_ci_stream | default('master') }}" + s2i_ci_parallel: >- + {{ s2i_ci_parallel | + default(ansible_facts.processor_vcpus | default(1)) }} + s2i_ci_images: "{{ s2i_ci_images | default('all') }}" + s2i_ci_image_mappings: "{{ s2i_ci_image_mappings | default({}) }}" + + - name: Validate the selected container project + ansible.builtin.assert: + that: + - s2i_ci_container_project in zuul.projects + fail_msg: The selected container project is not in the Zuul workspace + + - name: Resolve the inventory-described container repository + ansible.builtin.set_fact: + s2i_ci_container_repo: >- + {{ s2i_ci_workspace_root }}/{{ + zuul.projects[s2i_ci_container_project].src_dir }} + + - name: Inspect the container build entry point on the builder + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/build.sh" + follow: true + register: s2i_ci_build_entry_point + + - name: Canonicalize the builder workspace root + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_workspace_root }}"] + register: s2i_ci_workspace_root_realpath + changed_when: false + failed_when: false + + - name: Canonicalize the container repository on the builder + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_container_repo }}"] + register: s2i_ci_container_repo_realpath + changed_when: false + failed_when: false + + - name: Resolve auto-detected images when s2i_ci_images is 'auto' + when: s2i_ci_images == 'auto' + ansible.builtin.include_tasks: resolve-auto-images.yaml + + - name: Validate selective provider inputs + ansible.builtin.assert: + that: + - s2i_ci_build_entry_point.stat.isreg | default(false) + - s2i_ci_workspace_root_realpath.rc == 0 + - s2i_ci_container_repo_realpath.rc == 0 + - s2i_ci_container_repo_realpath.stdout.startswith( + s2i_ci_workspace_root_realpath.stdout + '/') + - >- + s2i_ci_images == 'all' or + (s2i_ci_images is sequence and s2i_ci_images is not string and + s2i_ci_images | length > 0 and + s2i_ci_images | select('string') | list | length == + s2i_ci_images | length and + s2i_ci_images | + reject('match', '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$') | + list | length == 0 and + s2i_ci_images | unique | list | length == s2i_ci_images | length) + - s2i_ci_image_mappings is mapping + - s2i_ci_tag is string + - "',' not in s2i_ci_tag" + fail_msg: Selective provider inputs are invalid + + - name: Record the requested target expression + ansible.builtin.set_fact: + s2i_ci_target_expression: >- + {{ 'all' if s2i_ci_images == 'all' + else ((['base'] + s2i_ci_images) | unique | list | join(',')) }} + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Resolve selected image targets + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - resolve + - "{{ s2i_ci_target_expression }}" + args: + chdir: "{{ s2i_ci_container_repo }}" + register: s2i_ci_resolved_images + changed_when: false + + - name: Record normalized selected targets + ansible.builtin.set_fact: + s2i_ci_selected_images: "{{ s2i_ci_resolved_images.stdout_lines }}" + + - name: Validate optional mapping overrides + ansible.builtin.include_tasks: validate-mapping-overrides.yaml + + - name: Load tracked image mappings + ansible.builtin.include_tasks: load-image-mappings.yaml + + - name: Load and validate selected image metadata + ansible.builtin.include_tasks: image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: validate-deployment-keys.yaml + + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + no_log: true + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + no_log: true + + - name: Validate normalized registry connection + ansible.builtin.assert: + that: + - s2i_ci_registry is mapping + - s2i_ci_registry.endpoint is string + - s2i_ci_registry.endpoint | length > 0 + - s2i_ci_registry.public_host is string + - s2i_ci_registry.public_host | length > 0 + - s2i_ci_registry.public_port | int > 0 + - s2i_ci_registry.auth_file | default('') is string + - s2i_ci_registry.cert_dir | default('') is string + + - name: Resolve intended exact image references before mutation + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_intended_refs + changed_when: false + + - name: Record intended exact image references + ansible.builtin.set_fact: + s2i_ci_intended_images: >- + {{ s2i_ci_intended_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Validate intended exact image references + ansible.builtin.assert: + that: + - s2i_ci_intended_images | length == s2i_ci_selected_images | length + - s2i_ci_intended_images | unique | list | length == + s2i_ci_intended_images | length + - s2i_ci_intended_images | + select('match', '^' + s2i_ci_registry.endpoint | regex_escape + + '/' + s2i_ci_namespace + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_intended_images | length + fail_msg: Intended image references are incomplete or unsafe + + - name: Persist intended references for partial-failure cleanup + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/intended-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_registry.endpoint, + 'namespace': s2i_ci_namespace, + 'images': s2i_ci_intended_images, + 'selected_images': s2i_ci_selected_images, + 'target': s2i_ci_target_expression + } | to_nice_json }} + + - name: Stage Zuul source checkouts for speculative builds + ansible.builtin.include_tasks: stage-zuul-sources.yaml + + - name: Build selected images + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - build-parallel + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + PARALLEL: "{{ s2i_ci_parallel | string }}" + BUILD_LOGS_DIR: "{{ s2i_ci_log_dir }}" + register: s2i_ci_build + changed_when: s2i_ci_build.rc == 0 + + - name: Save build command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/build.log" + mode: "0644" + content: | + {{ s2i_ci_build.stdout }} + {{ s2i_ci_build.stderr }} + + - name: Push selected images to the buildset registry + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - push + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + REGISTRY_AUTH_FILE: "{{ s2i_ci_registry.auth_file | default('') }}" + REGISTRY_CERT_DIR: "{{ s2i_ci_registry.cert_dir | default('') }}" + register: s2i_ci_push + changed_when: s2i_ci_push.rc == 0 + + - name: Save push command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/push.log" + mode: "0644" + content: | + {{ s2i_ci_push.stdout }} + {{ s2i_ci_push.stderr }} + + - name: Resolve successfully published image references + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_refs + changed_when: false + + - name: Record exact builder-local published image references + ansible.builtin.set_fact: + s2i_ci_local_published_images: >- + {{ s2i_ci_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Require intended and published references to match + ansible.builtin.assert: + that: + - s2i_ci_local_published_images == s2i_ci_intended_images + fail_msg: Published references differ from the pre-recorded intent + + - name: Record the public buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_public_registry_endpoint: >- + {{ s2i_ci_registry.public_host }}:{{ s2i_ci_registry.public_port }} + + - name: Convert published references to the public buildset endpoint + ansible.builtin.set_fact: + s2i_ci_published_images: >- + {{ s2i_ci_local_published_images | + map('regex_replace', + '^' + (s2i_ci_registry.endpoint | regex_escape), + s2i_ci_public_registry_endpoint) | list }} + + - name: Validate public exact published references + ansible.builtin.assert: + that: + - s2i_ci_published_images | length == + s2i_ci_local_published_images | length + - s2i_ci_published_images | + select('match', '^' + + (s2i_ci_public_registry_endpoint | regex_escape) + '/') | + list | length == s2i_ci_published_images | length + - s2i_ci_published_images | + select('search', s2i_ci_registry.endpoint | regex_escape) | + list | length == 0 + fail_msg: Public references must not expose the builder-local alias + + - name: Pull published images back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry.auth_file] + if s2i_ci_registry.auth_file | default('') | length > 0 else []) + + (['--cert-dir', s2i_ci_registry.cert_dir] + if s2i_ci_registry.cert_dir | default('') | length > 0 else []) + + [item] }} + loop: "{{ s2i_ci_local_published_images }}" + changed_when: true + + - name: Initialize exact reference expansion + ansible.builtin.set_fact: + s2i_ci_target_references: {} + s2i_ci_local_target_references: {} + s2i_ci_custom_container_images: {} + + - name: Map selected targets to exact references + ansible.builtin.include_tasks: published-image-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + index_var: s2i_ci_image_index + label: "{{ s2i_ci_image }}" + + - name: Expand exact references into deployment keys + ansible.builtin.include_tasks: deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Save published image metadata + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/published-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_public_registry_endpoint, + 'namespace': s2i_ci_namespace, + 'tag': s2i_ci_tag, + 'target': s2i_ci_target_expression, + 'stream': s2i_ci_stream, + 'selected_images': s2i_ci_selected_images, + 'images': s2i_ci_published_images, + 'image_mappings': s2i_ci_effective_image_mappings, + 'mapping_sources': s2i_ci_mapping_sources, + 'custom_container_images': + s2i_ci_custom_container_images + } | to_nice_json }} diff --git a/playbooks/container-ci/shared/stage-zuul-source-sync.yaml b/playbooks/container-ci/shared/stage-zuul-source-sync.yaml new file mode 100644 index 00000000..5d6b62d8 --- /dev/null +++ b/playbooks/container-ci/shared/stage-zuul-source-sync.yaml @@ -0,0 +1,43 @@ +--- +# Sync a single Zuul source checkout into its container build context. +# +# s2i_ci_src is a list: [name, canonical_project, url, dest_dir] +# from `build.sh list-sources` output. + +- name: "{{ s2i_ci_src[0] }} | Resolve Zuul checkout path" + ansible.builtin.set_fact: + s2i_ci_sync_from: >- + {{ s2i_ci_workspace_root }}/{{ + zuul.projects[s2i_ci_src[1]].src_dir }} + s2i_ci_sync_to: "{{ s2i_ci_src[3] }}" + +- name: "{{ s2i_ci_src[0] }} | Verify Zuul checkout exists" + ansible.builtin.stat: + path: "{{ s2i_ci_sync_from }}/.git" + register: s2i_ci_sync_check + +- name: "{{ s2i_ci_src[0] }} | Sync into build context" + when: s2i_ci_sync_check.stat.isdir | default(false) + block: + - name: "{{ s2i_ci_src[0] }} | Ensure destination directory" + ansible.builtin.file: + path: "{{ s2i_ci_sync_to }}" + state: directory + mode: "0755" + + - name: "{{ s2i_ci_src[0] }} | Copy source" + ansible.builtin.command: + argv: + - rsync + - --archive + - --delete + - --exclude=.tox + - --exclude=.venv + - --exclude=__pycache__ + - "{{ s2i_ci_sync_from }}/" + - "{{ s2i_ci_sync_to }}/" + changed_when: true + + - name: "{{ s2i_ci_src[0] }} | Log staged source" + ansible.builtin.debug: + msg: "Staged {{ s2i_ci_src[1] }} -> {{ s2i_ci_sync_to }}" diff --git a/playbooks/container-ci/shared/stage-zuul-sources.yaml b/playbooks/container-ci/shared/stage-zuul-sources.yaml new file mode 100644 index 00000000..66d5ff2b --- /dev/null +++ b/playbooks/container-ci/shared/stage-zuul-sources.yaml @@ -0,0 +1,56 @@ +--- +# Stage Zuul-checked-out source repositories into container build contexts. +# +# For speculative builds, Zuul checks out patched source repos into the +# workspace. This task file copies those checkouts into the container +# src/ directories so build.sh uses the patched code instead of cloning +# from pinned hashes. +# +# It delegates all sources.txt parsing to `build.sh list-sources`, keeping +# build.sh as the single source of truth for the source manifest format. +# +# Required facts (set by shared/run.yaml before including this): +# s2i_ci_container_repo - path to the s2i-openstack-containers checkout +# s2i_ci_stream - stream name (e.g. "master") +# s2i_ci_workspace_root - Zuul workspace root +# s2i_ci_resolved_images - register from `build.sh resolve` + +- name: Collect Zuul project canonical names + ansible.builtin.set_fact: + s2i_ci_zuul_project_names: >- + {{ zuul.projects | default({}) | list }} + +- name: Query source dependencies for each image target + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - list-sources + - "{{ item }}" + - "{{ s2i_ci_stream }}" + args: + chdir: "{{ s2i_ci_container_repo }}" + loop: "{{ s2i_ci_resolved_images.stdout.split() }}" + register: s2i_ci_source_queries + changed_when: false + failed_when: false + +- name: Build unified source placement list + ansible.builtin.set_fact: + s2i_ci_source_placements: >- + {{ s2i_ci_source_queries.results + | selectattr('stdout_lines', 'defined') + | map(attribute='stdout_lines') + | flatten + | unique + | list }} + +- name: Stage each Zuul-available source into its build context + ansible.builtin.include_tasks: stage-zuul-source-sync.yaml + loop: >- + {{ s2i_ci_source_placements + | map('split', '|') + | selectattr('1', 'in', s2i_ci_zuul_project_names) + | list }} + loop_control: + loop_var: s2i_ci_src + label: "{{ s2i_ci_src[0] }} ({{ s2i_ci_src[1] }})" diff --git a/tests/test_auto_detect.py b/tests/test_auto_detect.py new file mode 100644 index 00000000..43e8d43f --- /dev/null +++ b/tests/test_auto_detect.py @@ -0,0 +1,227 @@ +"""Tests for build.sh auto-detect and list-sources commands.""" + +import os +import pathlib +import subprocess +import unittest + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +BUILD_SH = REPO_ROOT / "build.sh" +CONTAINERS_DIR = REPO_ROOT / "containers" + + +def _run_build_sh(command: str, *args: str) -> subprocess.CompletedProcess: + env = {**os.environ, "PARALLEL": "1"} + return subprocess.run( + ["bash", str(BUILD_SH), command, *args], + capture_output=True, + text=True, + cwd=str(REPO_ROOT), + env=env, + timeout=30, + ) + + +def _run_auto_detect(*args: str) -> subprocess.CompletedProcess: + return _run_build_sh("auto-detect", *args) + + +def _sources_txt_projects() -> dict[str, list[str]]: + """Parse all sources.txt to build a map of canonical_project -> images.""" + result: dict[str, set[str]] = {} + for sources_file in sorted(CONTAINERS_DIR.rglob("sources.txt")): + rel = sources_file.relative_to(CONTAINERS_DIR) + project_name = str(rel).split("/")[0] + + for line in sources_file.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + fields = line.split() + if len(fields) < 4: + continue + _stream, name, url, *_ = fields + if name == "upper-constraints": + continue + url_path = url.split("://", 1)[-1] if "://" in url else url + url_path = url_path.split("/", 1)[-1] if "/" in url_path else "" + url_path = url_path.removesuffix(".git") + if not url_path: + continue + result.setdefault(url_path, set()) + # Find actual image targets under this project + for containerfile in CONTAINERS_DIR.glob( + f"{project_name}/*/Containerfile" + ): + image = ( + f"{project_name}/{containerfile.parent.name}" + ) + result[url_path].add(image) + if project_name == "base" and ( + CONTAINERS_DIR / "base" / "Containerfile" + ).exists(): + result[url_path].add("base") + return {k: sorted(v) for k, v in result.items()} + + +class TestAutoDetect(unittest.TestCase): + def test_exact_project_match(self): + proc = _run_auto_detect("openstack/tempest") + self.assertEqual(proc.returncode, 0) + images = proc.stdout.strip().splitlines() + self.assertIn("tempest/tempest", images) + + def test_no_false_positives_for_substring(self): + """openstack/watcher should NOT match openstack/watcher-tempest-plugin.""" + proc = _run_auto_detect("openstack/watcher") + self.assertEqual(proc.returncode, 0) + images = proc.stdout.strip().splitlines() + self.assertNotIn("tempest/tempest", images) + self.assertIn("watcher/watcher-base", images) + + def test_full_url_form(self): + proc = _run_auto_detect( + "https://opendev.org/openstack/tempest.git" + ) + self.assertEqual(proc.returncode, 0) + images = proc.stdout.strip().splitlines() + self.assertIn("tempest/tempest", images) + + def test_unknown_project_fails(self): + proc = _run_auto_detect("openstack/nonexistent-project") + self.assertNotEqual(proc.returncode, 0) + self.assertIn("no images reference", proc.stderr) + + def test_empty_arg_fails(self): + proc = _run_auto_detect("") + self.assertNotEqual(proc.returncode, 0) + + def test_no_arg_fails(self): + env = {**os.environ, "PARALLEL": "1"} + proc = subprocess.run( + ["bash", str(BUILD_SH), "auto-detect"], + capture_output=True, + text=True, + cwd=str(REPO_ROOT), + env=env, + timeout=30, + ) + self.assertNotEqual(proc.returncode, 0) + + def test_multi_image_project(self): + """Projects like neutron should return multiple images.""" + proc = _run_auto_detect("openstack/neutron") + self.assertEqual(proc.returncode, 0) + images = proc.stdout.strip().splitlines() + self.assertGreater(len(images), 1) + for img in images: + self.assertTrue( + img.startswith("neutron/"), + f"Expected neutron/ prefix, got: {img}", + ) + + def test_sub_image_sources(self): + """networking-baremetal is in sub-image sources.txt files.""" + proc = _run_auto_detect("openstack/networking-baremetal") + self.assertEqual(proc.returncode, 0) + images = proc.stdout.strip().splitlines() + self.assertIn("neutron/ironic-neutron-agent", images) + self.assertIn("neutron/neutron-server", images) + + def test_consistency_with_sources_txt(self): + """Every project in sources.txt should be detectable.""" + project_map = _sources_txt_projects() + for project, expected_images in project_map.items(): + with self.subTest(project=project): + proc = _run_auto_detect(project) + self.assertEqual( + proc.returncode, + 0, + f"auto-detect failed for {project}: {proc.stderr}", + ) + detected = sorted(proc.stdout.strip().splitlines()) + self.assertEqual( + detected, + expected_images, + f"Mismatch for {project}", + ) + + +class TestListSources(unittest.TestCase): + """Tests for build.sh list-sources command.""" + + def test_returns_pipe_delimited_records(self): + proc = _run_build_sh("list-sources", "watcher/watcher-base", "master") + self.assertEqual(proc.returncode, 0, proc.stderr) + lines = proc.stdout.strip().splitlines() + self.assertGreater(len(lines), 0) + for line in lines: + fields = line.split("|") + self.assertEqual( + len(fields), + 4, + f"Expected 4 pipe-delimited fields, got: {line}", + ) + name, canonical, url, dest = fields + self.assertTrue(name, "name field is empty") + self.assertTrue(canonical, "canonical_project field is empty") + self.assertTrue(url.startswith("https://"), f"unexpected url: {url}") + self.assertIn("/src/", dest, f"dest should contain /src/: {dest}") + + def test_excludes_upper_constraints(self): + proc = _run_build_sh("list-sources", "watcher/watcher-base", "master") + self.assertEqual(proc.returncode, 0, proc.stderr) + for line in proc.stdout.strip().splitlines(): + name = line.split("|")[0] + self.assertNotEqual(name, "upper-constraints") + + def test_no_target_fails(self): + proc = _run_build_sh("list-sources") + self.assertNotEqual(proc.returncode, 0) + + def test_tempest_includes_plugins(self): + proc = _run_build_sh("list-sources", "tempest/tempest", "master") + self.assertEqual(proc.returncode, 0, proc.stderr) + names = [line.split("|")[0] for line in proc.stdout.strip().splitlines()] + self.assertIn("tempest", names) + plugin_count = sum(1 for n in names if "tempest-plugin" in n) + self.assertGreater( + plugin_count, 0, "tempest/tempest should include tempest plugins" + ) + + def test_sub_image_sources_merge(self): + """list-sources for a sub-image should include project-level sources.""" + proc = _run_build_sh( + "list-sources", "neutron/ironic-neutron-agent", "master" + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + names = [line.split("|")[0] for line in proc.stdout.strip().splitlines()] + self.assertIn("neutron", names, "project-level source should be included") + self.assertIn( + "networking-baremetal", + names, + "sub-image source should be included", + ) + + def test_dest_paths_point_to_project_src(self): + proc = _run_build_sh("list-sources", "watcher/watcher-base", "master") + self.assertEqual(proc.returncode, 0, proc.stderr) + for line in proc.stdout.strip().splitlines(): + dest = line.split("|")[3] + self.assertIn( + "/containers/watcher/src/", + dest, + f"dest should be under project's src/: {dest}", + ) + + def test_stream_filter(self): + """Passing a non-existent stream should return nothing.""" + proc = _run_build_sh( + "list-sources", "watcher/watcher-base", "nonexistent-stream" + ) + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stdout.strip(), "") + + +if __name__ == "__main__": + unittest.main()