diff --git a/.github/actions/setup-tox/action.yml b/.github/actions/setup-tox/action.yml index bef55cb0..007939b0 100644 --- a/.github/actions/setup-tox/action.yml +++ b/.github/actions/setup-tox/action.yml @@ -1,13 +1,13 @@ name: Setup Python and tox -description: Install Python and tox (repository must already be checked out) +description: Install tox with the repository's canonical Python 3.12 runs: using: composite steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: - python-version: "3.x" + python-version: "3.12" - name: Install tox shell: bash - run: pip install tox + run: python3 -m pip install tox diff --git a/.github/workflows/test-update-sources.yml b/.github/workflows/test-update-sources.yml index 1e5c6275..75532ff9 100644 --- a/.github/workflows/test-update-sources.yml +++ b/.github/workflows/test-update-sources.yml @@ -1,26 +1,43 @@ -name: Tests (update-sources) +name: Tests (source reproducibility) on: pull_request: paths: + - .github/actions/setup-tox/action.yml + - .github/workflows/test-update-sources.yml - build.sh - containers/** + - tox.ini - '!**/OWNERS' - '!**/OWNERS_ALIASES' push: paths: + - .github/actions/setup-tox/action.yml + - .github/workflows/test-update-sources.yml - build.sh - containers/** + - tox.ini - '!**/OWNERS' - '!**/OWNERS_ALIASES' workflow_dispatch: jobs: - update-sources: + update-lockfiles: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-tox - - name: Verify update-sources runs successfully - run: tox -e update-sources + - name: Regenerate from committed source pins + run: tox -e update-lockfiles + + - name: Require reproducible generated files + run: git diff --exit-code -- containers/ + + - name: Preserve frozen source references + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: frozen-source-refs + path: .tmp/source-maintenance/frozen-source-refs.master.tsv + if-no-files-found: ignore diff --git a/build.sh b/build.sh index 9b6f4bab..f7461cc6 100755 --- a/build.sh +++ b/build.sh @@ -83,6 +83,8 @@ # PIP_NO_BINARY If set, passed as --build-arg to buildah so Containerfiles # can set ENV PIP_NO_BINARY. Use ":all:" to force pip to # build all packages from source instead of using wheels. +# REGISTRY_AUTH_FILE Authentication file passed explicitly to buildah push. +# REGISTRY_CERT_DIR TLS certificate directory passed explicitly to buildah push. set -euo pipefail @@ -102,7 +104,11 @@ UPSTREAM_CONSTRAINTS="upper-constraints.txt" DEFAULT_STREAM="${DEFAULT_STREAM:-master}" SKIP_HASH_UPDATE="${SKIP_HASH_UPDATE:-}" PIP_NO_BINARY="${PIP_NO_BINARY:-}" +REGISTRY_AUTH_FILE="${REGISTRY_AUTH_FILE:-}" +REGISTRY_CERT_DIR="${REGISTRY_CERT_DIR:-}" PARALLEL="${PARALLEL:-$(nproc)}" +SOURCE_REFS_STREAM="${STREAM//\//%2F}" +SOURCE_REFS_MANIFEST="${REPO_ROOT}/.tmp/source-maintenance/frozen-source-refs.${SOURCE_REFS_STREAM}.tsv" # Discover all buildable images from the directory structure. discover_images() { @@ -174,14 +180,18 @@ image_tag_args() { echo "${args}" } -# Track which repos were auto-cloned so we can clean up +# Track temporary source and frozen-ref repositories so they can be cleaned. declare -A _AUTO_CLONED=() +declare -A _FROZEN_REPOSITORIES=() -# Remove auto-cloned sources on exit cleanup_auto() { - for src_dir in "${!_AUTO_CLONED[@]}"; do - echo "--- Removing auto-cloned source: ${src_dir} ---" - rm -rf "${src_dir}" + local path + for path in "${!_AUTO_CLONED[@]}"; do + echo "--- Removing auto-cloned source: ${path} ---" + rm -rf "${path}" + done + for path in "${_FROZEN_REPOSITORIES[@]}"; do + rm -rf "${path}" done } trap cleanup_auto EXIT @@ -370,10 +380,32 @@ push_image() { name="$(image_name "${dir_name}")" IFS=',' read -ra tags <<< "${TAG}" + local registry_args=() + [[ -n "${REGISTRY_AUTH_FILE}" ]] && \ + registry_args+=(--authfile "${REGISTRY_AUTH_FILE}") + [[ -n "${REGISTRY_CERT_DIR}" ]] && \ + registry_args+=(--cert-dir "${REGISTRY_CERT_DIR}") + for t in "${tags[@]}"; do local full_tag="${REGISTRY}/${NAMESPACE}/${name}:${t}" echo "=== Pushing ${full_tag} ===" - buildah push "${full_tag}" + buildah push "${registry_args[@]}" "${full_tag}" + done +} + +# Print the exact image references produced for a target. +target_refs() { + local dir_name + local name + local tag + local -a tags + resolve_targets_array "$@" || return 1 + for dir_name in "${_RESOLVED_TARGETS[@]}"; do + name="$(image_name "${dir_name}")" + IFS=',' read -ra tags <<< "${TAG}" + for tag in "${tags[@]}"; do + echo "${REGISTRY}/${NAMESPACE}/${name}:${tag}" + done done } @@ -397,145 +429,363 @@ list_images() { fi } -# Resolve which images to process (accepts one or more targets) +# Resolve one or more image expressions in their requested order. A +# comma-separated expression is an explicit ordered union and includes base +# when it selects a service image. Separate positional targets preserve the +# upstream multi-target behavior. resolve_targets() { local all_images all_images=($(discover_images)) - local resolved=() + local -a resolved=() + local expression item image item_images + declare -A seen=() - for target in "$@"; do - if [[ "${target}" == "all" ]]; then + for expression in "$@"; do + if [[ "${expression}" == "all" ]]; then echo "${all_images[@]}" return fi - local found=0 + if [[ "${expression}" == *,* ]]; then + local -a requested + local selected_service=0 + IFS=',' read -ra requested <<< "${expression}" + for item in "${requested[@]}"; do + if [[ -z "${item}" || "${item}" != "${item//[[:space:]]/}" ]]; then + echo "ERROR: Invalid empty or whitespace-containing target '${item}'" >&2 + return 1 + fi + if ! item_images=$(resolve_targets "${item}"); then + return 1 + fi + for image in ${item_images}; do + [[ "${image}" != "base" ]] && selected_service=1 + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi + done + done + if [[ ${selected_service} -eq 1 && -z "${seen[base]:-}" ]]; then + resolved=("base" "${resolved[@]}") + seen[base]=1 + fi + continue + fi - # Exact match - for dir_name in "${all_images[@]}"; do - if [[ "${dir_name}" == "${target}" ]]; then - resolved+=("${target}") + local found=0 + for image in "${all_images[@]}"; do + if [[ "${image}" == "${expression}" ]]; then + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi found=1 break fi done [[ ${found} -eq 1 ]] && continue - # Project prefix match - for dir_name in "${all_images[@]}"; do - if [[ "${dir_name}" == "${target}/"* ]]; then - resolved+=("${dir_name}") + for image in "${all_images[@]}"; do + if [[ "${image}" == "${expression}/"* ]]; then + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi found=1 fi done [[ ${found} -eq 1 ]] && continue - echo "ERROR: Unknown image or project '${target}'" >&2 + echo "ERROR: Unknown image or project '${expression}'" >&2 echo "Available images:" >&2 - for dir_name in "${all_images[@]}"; do - echo " ${dir_name}" >&2 + for image in "${all_images[@]}"; do + echo " ${image}" >&2 done return 1 done + if [[ ${#resolved[@]} -eq 0 ]]; then + echo "ERROR: Explicit target selection is empty" >&2 + return 1 + fi echo "${resolved[@]}" } -# Clone a repo at a branch tip (or tag) and store the resolved commit hash -# in _CLONE_RESULT. Must NOT be called via command substitution ($(...)) -# because _AUTO_CLONED assignments would be lost in the subshell. -# If the destination already exists, use it as-is (same policy as clone_at_hash). -# Args: -clone_at_branch() { - local dest="$1" - local url="$2" - local branch="$3" +# Resolve an expression without losing failures in command substitutions. +declare -a _RESOLVED_TARGETS=() +resolve_targets_array() { + local output + if ! output=$(resolve_targets "$@"); then + return 1 + fi + _RESOLVED_TARGETS=(${output}) +} - if [[ -d "${dest}" ]]; then - echo "--- Using existing source: ${dest} ---" - else - mkdir -p "$(dirname "${dest}")" - echo "--- Cloning ${url} (${branch}) into ${dest} ---" - if ! git clone --branch "${branch}" "${url}" "${dest}" 2>/dev/null; then - git clone "${url}" "${dest}" 2>/dev/null - git -C "${dest}" checkout "${branch}" +# Collect selected source manifests in deterministic target order. The parallel +# arrays describe the manifest, source checkout directory, and project context. +declare -a _SOURCE_FILES=() +declare -a _SOURCE_DIRS=() +declare -a _SOURCE_PROJECT_DIRS=() +collect_source_scopes() { + local -a targets + local img project sources_file + declare -A seen=() + + resolve_targets_array "$@" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") + _SOURCE_FILES=() + _SOURCE_DIRS=() + _SOURCE_PROJECT_DIRS=() + + for img in "${targets[@]}"; do + project="$(project_name "${img}")" + if [[ -z "${project}" ]]; then + [[ "${img}" == "base" ]] || continue + sources_file="${CONTAINERS_DIR}/base/sources.txt" + if [[ -f "${sources_file}" && -z "${seen[${sources_file}]:-}" ]]; then + seen["${sources_file}"]=1 + _SOURCE_FILES+=("${sources_file}") + _SOURCE_DIRS+=("${CONTAINERS_DIR}/base/src") + _SOURCE_PROJECT_DIRS+=("${CONTAINERS_DIR}/base") + fi + continue + fi + + sources_file="${CONTAINERS_DIR}/${project}/sources.txt" + if [[ -f "${sources_file}" && -z "${seen[${sources_file}]:-}" ]]; then + seen["${sources_file}"]=1 + _SOURCE_FILES+=("${sources_file}") + _SOURCE_DIRS+=("${CONTAINERS_DIR}/${project}/src") + _SOURCE_PROJECT_DIRS+=("${CONTAINERS_DIR}/${project}") fi - _AUTO_CLONED["${dest}"]=1 + + sources_file="${CONTAINERS_DIR}/${img}/sources.txt" + if [[ -f "${sources_file}" && -z "${seen[${sources_file}]:-}" ]]; then + seen["${sources_file}"]=1 + _SOURCE_FILES+=("${sources_file}") + _SOURCE_DIRS+=("${CONTAINERS_DIR}/${img}/src") + _SOURCE_PROJECT_DIRS+=("${CONTAINERS_DIR}/${project}") + fi + done +} + +source_record_key() { + printf '%s\x1f%s\x1f%s\x1f%s' "$1" "$2" "$3" "$4" +} + +source_ref_key() { + printf '%s\x1f%s' "$1" "$2" +} + +declare -A _FROZEN_COMMITS=() +declare -A _FROZEN_AUTHORITIES=() +declare -A _FROZEN_REF_COMMITS=() + +# Fetch one declared ref into a temporary bare repository and retain its exact +# commit for the rest of this process. This prevents a moving branch from +# changing inputs after preflight. +freeze_remote_ref() { + local url="$1" + local ref="$2" + local ref_key repository + ref_key="$(source_ref_key "${url}" "${ref}")" + if [[ -n "${_FROZEN_REF_COMMITS[${ref_key}]:-}" ]]; then + _FREEZE_RESULT="${_FROZEN_REF_COMMITS[${ref_key}]}" + return fi - _CLONE_RESULT=$(git -C "${dest}" rev-parse HEAD) + repository=$(mktemp -d) + git -C "${repository}" init --quiet --bare + if ! git -C "${repository}" fetch --quiet "${url}" "${ref}"; then + echo "ERROR: Could not freeze ref '${ref}' for ${url}" >&2 + rm -rf "${repository}" + return 1 + fi + if ! _FREEZE_RESULT=$(git -C "${repository}" rev-parse --verify 'FETCH_HEAD^{commit}'); then + echo "ERROR: Ref '${ref}' for ${url} does not resolve to a commit" >&2 + rm -rf "${repository}" + return 1 + fi + _FROZEN_REF_COMMITS["${ref_key}"]="${_FREEZE_RESULT}" + _FROZEN_REPOSITORIES["${ref_key}"]="${repository}" +} + +validate_stream_name() { + local stream="$1" + local component + local -a components + if [[ ! "${stream}" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] || \ + [[ "${stream}" == */ || "${stream}" == *//* ]]; then + echo "ERROR: Unsafe stream name '${stream}'" >&2 + return 1 + fi + IFS='/' read -ra components <<< "${stream}" + for component in "${components[@]}"; do + if [[ "${component}" == "." || "${component}" == ".." ]]; then + echo "ERROR: Unsafe stream name '${stream}'" >&2 + return 1 + fi + done } -# Update pinned hashes in a single sources.txt file for the given stream. -# Clones source repos at the branch tip to resolve hashes, and extracts -# upper-constraints.txt when encountered. -# Args: +# Resolve and record every selected source before the first tracked mutation. +freeze_source_refs() { + local stream="${!#}" + local targets_args=("${@:1:$#-1}") + local manifest_dir manifest_tmp + local index line entry_stream name url branch pinned_hash extra + local sources_file src_dir relative_file record_key authority frozen + + validate_stream_name "${stream}" || return 1 + collect_source_scopes "${targets_args[@]}" || return 1 + _FROZEN_COMMITS=() + _FROZEN_AUTHORITIES=() + _FROZEN_REF_COMMITS=() + _FROZEN_REPOSITORIES=() + + manifest_dir="$(dirname "${SOURCE_REFS_MANIFEST}")" + if [[ "${manifest_dir}" != "${REPO_ROOT}/.tmp/source-maintenance" ]]; then + echo "ERROR: Frozen source manifest escaped repository temporary state" >&2 + return 1 + fi + mkdir -p "${manifest_dir}" + rm -f "${SOURCE_REFS_MANIFEST}" + manifest_tmp=$(mktemp "${manifest_dir}/.frozen-source-refs.XXXXXX") + printf 'source_file\tstream\tname\turl\tdeclared_ref\tcommitted_pin\tfrozen_commit\tauthority\n' > "${manifest_tmp}" + + for index in "${!_SOURCE_FILES[@]}"; do + sources_file="${_SOURCE_FILES[${index}]}" + src_dir="${_SOURCE_DIRS[${index}]}" + relative_file="${sources_file#"${REPO_ROOT}/"}" + while IFS= read -r line; do + [[ -z "${line}" || "${line}" == \#* ]] && continue + read -r entry_stream name url branch pinned_hash extra <<< "${line}" + [[ "${entry_stream}" == "${stream}" ]] || continue + if [[ -z "${name}" || -z "${url}" || -z "${branch}" || -z "${pinned_hash}" || -n "${extra:-}" ]]; then + echo "ERROR: Malformed source record in ${relative_file}: ${line}" >&2 + rm -f "${manifest_tmp}" + return 1 + fi + + record_key="$(source_record_key "${sources_file}" "${name}" "${url}" "${branch}")" + if [[ "${name}" != "upper-constraints" && -d "${src_dir}/${name}" ]]; then + local checkout_root + checkout_root=$(git -C "${src_dir}/${name}" rev-parse --show-toplevel 2>/dev/null || true) + if [[ -z "${checkout_root}" || "$(realpath -e "${checkout_root}")" != "$(realpath -e "${src_dir}/${name}")" ]] || \ + ! frozen=$(git -C "${src_dir}/${name}" rev-parse --verify 'HEAD^{commit}' 2>/dev/null); then + echo "ERROR: Pre-existing source is not a Git checkout: ${src_dir}/${name}" >&2 + rm -f "${manifest_tmp}" + return 1 + fi + authority="pre-existing-checkout" + else + if [[ -n "${SKIP_HASH_UPDATE}" ]]; then + freeze_remote_ref "${url}" "${pinned_hash}" || { + rm -f "${manifest_tmp}" + return 1 + } + authority="committed-pin" + else + freeze_remote_ref "${url}" "${branch}" || { + rm -f "${manifest_tmp}" + return 1 + } + authority="declared-ref" + fi + frozen="${_FREEZE_RESULT}" + fi + + _FROZEN_COMMITS["${record_key}"]="${frozen}" + _FROZEN_AUTHORITIES["${record_key}"]="${authority}" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${relative_file}" "${entry_stream}" "${name}" "${url}" \ + "${branch}" "${pinned_hash}" "${frozen}" "${authority}" \ + >> "${manifest_tmp}" + done < "${sources_file}" + done + + mv "${manifest_tmp}" "${SOURCE_REFS_MANIFEST}" + echo "--- Frozen source references: ${SOURCE_REFS_MANIFEST} ---" +} + +# Materialize one source declaration from its frozen preflight repository and +# update its maintained pin only in advancement mode. update_sources_file() { local sources_file="$1" local stream="$2" local src_dir="$3" local project_dir="$4" - - if [[ ! -f "${sources_file}" ]]; then - return - fi - - local tmp_file - tmp_file=$(mktemp) + local tmp_file line entry_stream name url branch pinned_hash extra + local record_key ref_key frozen authority repository output_tmp local updated=0 + tmp_file=$(mktemp) while IFS= read -r line; do - # Preserve comments and blank lines if [[ -z "${line}" || "${line}" == \#* ]]; then echo "${line}" >> "${tmp_file}" continue fi - read -r entry_stream name url branch pinned_hash <<< "${line}" - - # Only update entries for the requested stream + read -r entry_stream name url branch pinned_hash extra <<< "${line}" if [[ "${entry_stream}" != "${stream}" ]]; then echo "${line}" >> "${tmp_file}" continue fi - local new_hash - if [[ "${name}" == "upper-constraints" ]]; then - # Clone without checkout, resolve hash from branch, extract file - local uc_tmp - uc_tmp=$(mktemp -d) - git clone --no-checkout "${url}" "${uc_tmp}" 2>/dev/null - new_hash=$(git -C "${uc_tmp}" rev-parse --verify "origin/${branch}" 2>/dev/null \ - || git -C "${uc_tmp}" rev-parse --verify "${branch}" 2>/dev/null) - git -C "${uc_tmp}" checkout "${new_hash}" -- upper-constraints.txt - cp "${uc_tmp}/upper-constraints.txt" "${project_dir}/${UPSTREAM_CONSTRAINTS}.${stream}" - rm -rf "${uc_tmp}" - elif [[ -d "${src_dir}/${name}" ]]; then - # Pre-existing checkout — use it for pip-compile but don't update the hash + record_key="$(source_record_key "${sources_file}" "${name}" "${url}" "${branch}")" + frozen="${_FROZEN_COMMITS[${record_key}]:-}" + authority="${_FROZEN_AUTHORITIES[${record_key}]:-}" + if [[ -z "${frozen}" || -z "${authority}" ]]; then + echo "ERROR: Missing frozen source record for ${name} in ${sources_file}" >&2 + rm -f "${tmp_file}" + return 1 + fi + + if [[ "${authority}" == "pre-existing-checkout" ]]; then echo " ${name}: skipped (pre-existing checkout at ${src_dir}/${name})" - new_hash="${pinned_hash}" else - clone_at_branch "${src_dir}/${name}" "${url}" "${branch}" - new_hash="${_CLONE_RESULT}" - fi + if [[ "${authority}" == "committed-pin" ]]; then + ref_key="$(source_ref_key "${url}" "${pinned_hash}")" + else + ref_key="$(source_ref_key "${url}" "${branch}")" + fi + repository="${_FROZEN_REPOSITORIES[${ref_key}]:-}" + if [[ -z "${repository}" || ! -d "${repository}" ]]; then + echo "ERROR: Missing frozen repository for ${name}" >&2 + rm -f "${tmp_file}" + return 1 + fi - if [[ -z "${new_hash}" ]]; then - echo "ERROR: Could not resolve ref '${branch}' for ${url}" >&2 - rm "${tmp_file}" - return 1 + if [[ "${name}" == "upper-constraints" ]]; then + output_tmp=$(mktemp "${project_dir}/.${UPSTREAM_CONSTRAINTS}.${stream}.XXXXXX") + if ! git -C "${repository}" show "${frozen}:upper-constraints.txt" > "${output_tmp}"; then + rm -f "${output_tmp}" "${tmp_file}" + return 1 + fi + mv "${output_tmp}" "${project_dir}/${UPSTREAM_CONSTRAINTS}.${stream}" + else + mkdir -p "${src_dir}" + echo "--- Materializing ${url} at ${frozen} into ${src_dir}/${name} ---" + git -C "${src_dir}" init --quiet "${name}" + _AUTO_CLONED["${src_dir}/${name}"]=1 + git -C "${src_dir}/${name}" fetch --quiet "${repository}" "${frozen}" + git -C "${src_dir}/${name}" checkout --quiet --detach FETCH_HEAD + fi fi - if [[ "${new_hash}" != "${pinned_hash}" ]]; then - echo " ${name}: ${pinned_hash:-} → ${new_hash} (${branch})" + if [[ -z "${SKIP_HASH_UPDATE}" && "${authority}" != "pre-existing-checkout" && "${frozen}" != "${pinned_hash}" ]]; then + echo " ${name}: ${pinned_hash} → ${frozen} (${branch})" + pinned_hash="${frozen}" updated=1 fi - echo "${entry_stream} ${name} ${url} ${branch} ${new_hash}" >> "${tmp_file}" + echo "${entry_stream} ${name} ${url} ${branch} ${pinned_hash}" >> "${tmp_file}" done < "${sources_file}" if [[ ${updated} -eq 1 ]]; then install -m 644 "${tmp_file}" "${sources_file}" else - echo " (no changes)" + echo " (no pin changes)" fi rm "${tmp_file}" } @@ -618,6 +868,24 @@ filter_lockfile_rpm_packages() { rm "${tmp}" } +# Remove generator-runtime and package-index details from a lock file. +normalize_generated_lock() { + local lockfile="$1" + local tmp + tmp=$(mktemp) + + awk ' + !seen_package && /^--(index-url|extra-index-url|trusted-host)[[:space:]]/ { + next + } + !seen_package && (/^#/ || /^$/) { next } + !/^#/ { seen_package = 1 } + { print } + ' "${lockfile}" > "${tmp}" + install -m 644 "${tmp}" "${lockfile}" + rm "${tmp}" +} + # Generate a single requirements.lock for a project by running pip-compile # against requirements.txt from all source packages (project + all images) # plus pythondeps.txt and pythonbuilddeps.txt from every image, @@ -676,11 +944,11 @@ generate_requirements_lock() { echo "--- Generating ${project_dir}/${lock_file} ---" (cd "${project_dir}" && \ - pip-compile --allow-unsafe --strip-extras \ + pip-compile --allow-unsafe --no-annotate --strip-extras \ -c "${UPSTREAM_CONSTRAINTS}.${stream}" \ -o "${lock_file}" \ "${input_files[@]}" && \ - sed -i "/# This file is autogenerated/{N;d;}" "${lock_file}") + normalize_generated_lock "${lock_file}") local rpm_pkgs rpm_pkgs=$(collect_rpm_python_packages "${project_dir}") @@ -738,10 +1006,11 @@ generate_buildrequirements_lock() { echo "--- Generating ${project_dir}/${build_lock_file} ---" (cd "${project_dir}" && \ - pybuild-deps compile --no-annotate \ + pybuild-deps compile \ + --no-annotate \ -o "${build_lock_file}" \ "${CONSTRAINTS_FILE}.${stream}" && \ - sed -i "/# This file is autogenerated/{N;d;}" "${build_lock_file}") + normalize_generated_lock "${build_lock_file}") } # Generate buildrequirements.lock for each project in the target scope. @@ -901,71 +1170,30 @@ ensure_sources_for_targets() { done } -# Update sources.txt files for targets in scope. -# Clones source repos at branch tips to resolve hashes, fetches -# upper-constraints.txt, and updates pinned hashes in sources.txt. +# Materialize all preflight-frozen sources and update maintained pins only when +# SKIP_HASH_UPDATE is unset. update_sources() { local stream="${!#}" - local targets_args=("${@:1:$#-1}") + local index sources_file if [[ -z "${stream}" ]]; then echo "ERROR: STREAM is required for update-sources." >&2 echo " Example: STREAM=master ./build.sh update-sources watcher" >&2 return 1 fi + if [[ ${#_SOURCE_FILES[@]} -eq 0 ]]; then + echo "ERROR: Source preflight did not collect any manifests" >&2 + return 1 + fi - local targets - targets=($(resolve_targets "${targets_args[@]}")) - - declare -A projects_seen - - for img in "${targets[@]}"; do - local project - project="$(project_name "${img}")" - - # Base container: flat layout, sources.txt directly in containers/base/ - if [[ -z "${project}" ]]; then - if [[ "${img}" == "base" ]] && [[ -z "${projects_seen[base]:-}" ]]; then - projects_seen["base"]=1 - local base_sources="${CONTAINERS_DIR}/base/sources.txt" - if [[ -f "${base_sources}" ]]; then - echo "--- Updating ${base_sources} (stream: ${stream}) ---" - if ! update_sources_file "${base_sources}" "${stream}" \ - "${CONTAINERS_DIR}/base/src" \ - "${CONTAINERS_DIR}/base"; then - echo "ERROR: Failed to update ${base_sources}" >&2 - return 1 - fi - fi - fi - continue - fi - - # Project-level sources.txt (only process once per project) - if [[ -z "${projects_seen[$project]:-}" ]]; then - projects_seen["${project}"]=1 - local project_sources="${CONTAINERS_DIR}/${project}/sources.txt" - if [[ -f "${project_sources}" ]]; then - echo "--- Updating ${project_sources} (stream: ${stream}) ---" - if ! update_sources_file "${project_sources}" "${stream}" \ - "${CONTAINERS_DIR}/${project}/src" \ - "${CONTAINERS_DIR}/${project}"; then - echo "ERROR: Failed to update ${project_sources}" >&2 - return 1 - fi - fi - fi - - # Image-level sources.txt - local image_sources="${CONTAINERS_DIR}/${img}/sources.txt" - if [[ -f "${image_sources}" ]]; then - echo "--- Updating ${image_sources} (stream: ${stream}) ---" - if ! update_sources_file "${image_sources}" "${stream}" \ - "${CONTAINERS_DIR}/${img}/src" \ - "${CONTAINERS_DIR}/${project}"; then - echo "ERROR: Failed to update ${image_sources}" >&2 - return 1 - fi + for index in "${!_SOURCE_FILES[@]}"; do + sources_file="${_SOURCE_FILES[${index}]}" + echo "--- Updating ${sources_file} (stream: ${stream}) ---" + if ! update_sources_file "${sources_file}" "${stream}" \ + "${_SOURCE_DIRS[${index}]}" \ + "${_SOURCE_PROJECT_DIRS[${index}]}"; then + echo "ERROR: Failed to update ${sources_file}" >&2 + return 1 fi done } @@ -987,11 +1215,24 @@ case "${ACTION}" in ;; build-parallel) _bp_targets=($(resolve_targets "${TARGETS[@]}")) + if [[ ! "${PARALLEL}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: PARALLEL must be a positive integer" >&2 + exit 1 + fi + if [[ -n "${BUILD_LOGS_DIR:-}" ]]; then + _bp_logdir="${BUILD_LOGS_DIR}" + mkdir -p "${_bp_logdir}" + else + _bp_logdir=$(mktemp -d) + fi # Build base first (all service images depend on it) for _bp_img in "${_bp_targets[@]}"; do [[ -n "$(project_name "${_bp_img}")" ]] && continue - build_image "${_bp_img}" + set -o pipefail + build_image "${_bp_img}" 2>&1 | + sed -u "s|^|[${_bp_img}] |" | + tee "${_bp_logdir}/${_bp_img//\//_}.log" done # Pre-clone sources so parallel builds don't race on the same directories @@ -1001,12 +1242,6 @@ case "${ACTION}" in done # Build service images in parallel (max PARALLEL at a time) - if [[ -n "${BUILD_LOGS_DIR:-}" ]]; then - _bp_logdir="${BUILD_LOGS_DIR}" - mkdir -p "${_bp_logdir}" - else - _bp_logdir=$(mktemp -d) - fi _bp_service_imgs=() for _bp_img in "${_bp_targets[@]}"; do [[ -z "$(project_name "${_bp_img}")" ]] && continue @@ -1020,47 +1255,50 @@ case "${ACTION}" in _bp_running=0 for _bp_img in "${_bp_service_imgs[@]}"; do - # Wait for a slot if at the limit while [[ ${_bp_running} -ge ${PARALLEL} ]]; do - if ! wait -n; then + _bp_finished="" + if wait -n -p _bp_finished "${!_bp_pids[@]}"; then + unset '_bp_pids['"${_bp_finished}"']' + ((_bp_running--)) || true + else _bp_fail=1 + [[ -n "${_bp_finished}" ]] && unset '_bp_pids['"${_bp_finished}"']' break 2 fi - ((_bp_running--)) || true done _bp_log="${_bp_logdir}/${_bp_img//\//_}.log" - build_image "${_bp_img}" > "${_bp_log}" 2>&1 & + ( + set -o pipefail + build_image "${_bp_img}" 2>&1 | + sed -u "s|^|[${_bp_img}] |" | + tee "${_bp_log}" + ) & _bp_pids[$!]="${_bp_img}" ((_bp_running++)) || true done - # Wait for remaining builds if [[ ${_bp_fail} -eq 0 ]]; then while [[ ${_bp_running} -gt 0 ]]; do - if ! wait -n; then + _bp_finished="" + if wait -n -p _bp_finished "${!_bp_pids[@]}"; then + unset '_bp_pids['"${_bp_finished}"']' + ((_bp_running--)) || true + else _bp_fail=1 + [[ -n "${_bp_finished}" ]] && unset '_bp_pids['"${_bp_finished}"']' break fi - ((_bp_running--)) || true done fi - # Show logs for all builds - for _bp_log in "${_bp_logdir}"/*.log; do - _bp_name=$(basename "${_bp_log}" .log) - echo "=== ${_bp_name} ===" - cat "${_bp_log}" - echo "" - done - if [[ ${_bp_fail} -eq 1 ]]; then - echo "ERROR: A build failed, killing remaining builds" >&2 + echo "ERROR: A build failed; stopping remaining builds" >&2 for _bp_pid in "${!_bp_pids[@]}"; do kill "${_bp_pid}" 2>/dev/null || true done wait 2>/dev/null || true - [[ -z "${BUILD_LOGS_DIR:-}" ]] && rm -rf "${_bp_logdir}" + echo "Build logs are available in ${_bp_logdir}" >&2 exit 1 fi @@ -1082,13 +1320,19 @@ case "${ACTION}" in push_image "${img}" done ;; + refs) + target_refs "${TARGETS[@]}" + ;; + resolve) + resolve_targets_array "${TARGETS[@]}" + printf '%s\n' "${_RESOLVED_TARGETS[@]}" + ;; update-sources) if [[ -n "${SKIP_HASH_UPDATE}" ]]; then echo "=== Skipping hash update (SKIP_HASH_UPDATE is set) ===" - ensure_sources_for_targets "${TARGETS[@]}" "${STREAM}" - else - update_sources "${TARGETS[@]}" "${STREAM}" fi + freeze_source_refs "${TARGETS[@]}" "${STREAM}" + update_sources "${TARGETS[@]}" "${STREAM}" # Generate lockfiles and metadata echo "" @@ -1151,7 +1395,7 @@ case "${ACTION}" in list_images ;; *) - echo "Usage: STREAM= $0 {build|build-parallel|push|update-sources|install-deps|list} [target ...]" + echo "Usage: STREAM= $0 {build|build-parallel|push|refs|resolve|update-sources|install-deps|list} [target ...]" echo "" echo "Images (discovered from containers/):" for dir_name in $(discover_images); do @@ -1175,6 +1419,8 @@ case "${ACTION}" in echo " BUILD_LOGS_DIR Persist build-parallel logs to this directory" echo " SKIP_HASH_UPDATE Skip updating pinned hashes; regenerate locks only" echo " PIP_NO_BINARY Pass PIP_NO_BINARY to container build (e.g., ':all:')" + echo " REGISTRY_AUTH_FILE Registry authentication file for pushes" + echo " REGISTRY_CERT_DIR Registry TLS certificate directory for pushes" echo "" echo "Source directories: containers//src//" echo "Overrides: containers//src/overrides//" diff --git a/containers/base/buildrequirements.lock.master b/containers/base/buildrequirements.lock.master index bde466c0..9cd81cad 100644 --- a/containers/base/buildrequirements.lock.master +++ b/containers/base/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# flit-core==4.0.2 packaging==26.3 setuptools-scm==10.2.1 diff --git a/containers/base/requirements.lock.master b/containers/base/requirements.lock.master index ea0d8ab6..f94e40f5 100644 --- a/containers/base/requirements.lock.master +++ b/containers/base/requirements.lock.master @@ -1,22 +1,8 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras pythondeps.txt -# crudini==0.9.6 - # via -r pythondeps.txt dumb-init==1.2.5.post1 - # via -r pythondeps.txt iniparse==0.5 - # via crudini pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r pythondeps.txt six==1.17.0 - # via - # -c upper-constraints.txt.master - # iniparse # The following packages are considered to be unsafe in a requirements file: setuptools==82.0.1 - # via pbr diff --git a/containers/cyborg/buildrequirements.lock.master b/containers/cyborg/buildrequirements.lock.master index eee1c155..23f26bc2 100644 --- a/containers/cyborg/buildrequirements.lock.master +++ b/containers/cyborg/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# calver==2025.10.20 coherent-licensed==0.5.2 cython==3.2.9 diff --git a/containers/cyborg/requirements.lock.master b/containers/cyborg/requirements.lock.master index 87747144..1bc56468 100644 --- a/containers/cyborg/requirements.lock.master +++ b/containers/cyborg/requirements.lock.master @@ -1,590 +1,107 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras cyborg-agent/pythonbuilddeps.txt cyborg-agent/pythondeps.txt cyborg/pythonbuilddeps.txt cyborg/pythondeps.txt src/cyborg/requirements.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # oslo-messaging certifi==2026.6.17 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography - # oslo-privsep charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cotyledon==2.2.0 - # via - # -c upper-constraints.txt.master - # oslo-service debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # futurist - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-privsep - # oslo-service - # python-keystoneclient decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # oslo-service greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # eventlet - # oslo-service - # sqlalchemy idna==3.18 - # via - # -c upper-constraints.txt.master - # requests importlib-metadata==9.0.0 - # via - # -c upper-constraints.txt.master - # wsme iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk - # oslo-utils jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # openstacksdk - # warlock jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # warlock jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # openstacksdk - # python-glanceclient - # python-keystoneclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic - # pecan markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako microversion-parse==2.1.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-privsep - # oslo-serialization netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # oslo-config - # oslo-utils - # oslo-versionedobjects - # wsme openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt os-resource-classes==1.1.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r cyborg-agent/pythondeps.txt - # -r cyborg/pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-service oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # pycadf - # python-keystoneclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy - # oslo-versionedobjects oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r cyborg-agent/pythondeps.txt - # -r cyborg/pythondeps.txt - # -r src/cyborg/requirements.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-service - # oslo-upgradecheck - # oslo-utils - # oslo-versionedobjects - # python-glanceclient - # python-keystoneclient oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-messaging - # oslo-metrics - # oslo-privsep - # oslo-service - # oslo-versionedobjects oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-versionedobjects oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-upgradecheck oslo-privsep==3.12.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-policy - # pycadf - # python-keystoneclient oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-messaging - # oslo-service oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # python-glanceclient - # python-keystoneclient oslo-versionedobjects==3.11.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-utils - # python-keystoneclient - # wheel paste==3.10.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r cyborg-agent/pythonbuilddeps.txt - # -r cyborg/pythonbuilddeps.txt - # -r src/cyborg/requirements.txt - # keystonemiddleware - # openstacksdk - # os-resource-classes - # os-service-types - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-utils - # python-glanceclient - # python-keystoneclient pecan==1.8.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # oslo-upgradecheck - # python-glanceclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # openstacksdk - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pyopenssl==24.2.1 - # via - # -c upper-constraints.txt.master - # python-glanceclient pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # oslo-utils python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # oslo-log python-glanceclient==4.12.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache pytz==2026.2 - # via - # -c upper-constraints.txt.master - # wsme pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # keystonemiddleware - # oslo-config - # oslo-policy - # python-glanceclient - # python-keystoneclient rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config routes==2.5.1 - # via - # -c upper-constraints.txt.master - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing setproctitle==1.3.7 - # via - # -c upper-constraints.txt.master - # cotyledon simplegeneric==0.8.1 - # via - # -c upper-constraints.txt.master - # wsme six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # alembic - # oslo-db statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt - # dogpile-cache - # keystoneauth1 - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-keystoneclient testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # cotyledon - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu warlock==2.1.0 - # via - # -c upper-constraints.txt.master - # python-glanceclient wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable webob==1.8.10 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # microversion-parse - # oslo-messaging - # oslo-middleware - # oslo-service - # pecan - # wsme wheel==0.47.0 - # via - # -r cyborg-agent/pythonbuilddeps.txt - # -r cyborg/pythonbuilddeps.txt wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector - # python-glanceclient wsme==0.12.1 - # via - # -c upper-constraints.txt.master - # -r src/cyborg/requirements.txt yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service zipp==4.1.0 - # via - # -c upper-constraints.txt.master - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/glance/buildrequirements.lock.master b/containers/glance/buildrequirements.lock.master index a4625a84..87851e32 100644 --- a/containers/glance/buildrequirements.lock.master +++ b/containers/glance/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# build==1.5.0 calver==2025.10.20 coherent-licensed==0.5.2 @@ -26,7 +22,7 @@ tomlkit==0.15.1 trove-classifiers==2026.6.1.19 typing-extensions==4.15.0 vcs-versioning==2.2.4 -wheel==0.47.0 +wheel==0.48.0 # The following packages are considered to be unsafe in a requirements file: setuptools==82.0.1 diff --git a/containers/glance/requirements.lock.master b/containers/glance/requirements.lock.master index 2fa70252..7f0f33c4 100644 --- a/containers/glance/requirements.lock.master +++ b/containers/glance/requirements.lock.master @@ -1,736 +1,126 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras glance-api/pythonbuilddeps.txt glance-api/pythondeps.txt src/glance/requirements.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing automaton==3.4.0 - # via - # -c upper-constraints.txt.master - # taskflow autopage==0.6.0 - # via - # -c upper-constraints.txt.master - # cliff bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # taskflow castellan==5.8.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # cursive certifi==2026.7.22 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography - # oslo-privsep charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cliff==4.15.0 - # via - # -c upper-constraints.txt.master - # python-barbicanclient cmd2==4.0.0 - # via - # -c upper-constraints.txt.master - # cliff cursive==0.3.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # futurist - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-privsep - # oslo-rootwrap - # oslo-service - # python-keystoneclient - # taskflow decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk defusedxml==0.7.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency - # taskflow futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # glance-store - # oslo-messaging - # taskflow glance-store==5.6.0 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythondeps.txt - # -r src/glance/requirements.txt greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # eventlet - # oslo-service - # sqlalchemy httplib2==0.32.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt idna==3.18 - # via - # -c upper-constraints.txt.master - # requests importlib-metadata==9.0.0 - # via - # -c upper-constraints.txt.master - # wsme iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk - # oslo-utils jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # oslo-reports jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # glance-store - # taskflow jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # glance-store - # keystonemiddleware - # openstacksdk - # oslo-limit - # python-barbicanclient - # python-cinderclient - # python-keystoneclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic markdown-it-py==4.2.0 - # via - # -c upper-constraints.txt.master - # rich markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako mdurl==0.1.2 - # via - # -c upper-constraints.txt.master - # markdown-it-py msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-privsep - # oslo-serialization netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # oslo-config - # oslo-utils - # osprofiler - # wsme networkx==3.6.1 - # via - # -c upper-constraints.txt.master - # taskflow openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # oslo-limit os-brick==7.1.0 - # via - # -c upper-constraints.txt.master - # glance-store os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # glance-store - # os-brick - # oslo-service - # osprofiler oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # glance-store - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-limit - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # osprofiler - # pycadf - # python-keystoneclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # keystonemiddleware - # os-brick - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythondeps.txt - # -r src/glance/requirements.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cursive - # glance-store - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-limit - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient oslo-limit==2.12.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cursive - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-limit - # oslo-messaging - # oslo-metrics - # oslo-privsep - # oslo-service oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-upgradecheck oslo-privsep==3.12.0 - # via - # -c upper-constraints.txt.master - # glance-store - # os-brick oslo-reports==3.9.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-rootwrap==7.10.0 - # via - # -c upper-constraints.txt.master - # glance-store oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # cursive - # glance-store - # keystonemiddleware - # os-brick - # oslo-log - # oslo-messaging - # oslo-policy - # oslo-reports - # osprofiler - # pycadf - # python-barbicanclient - # python-keystoneclient - # taskflow oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # os-brick - # oslo-messaging oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cursive - # glance-store - # keystonemiddleware - # os-brick - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # taskflow osprofiler==4.4.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-utils - # python-keystoneclient paste==3.10.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r glance-api/pythonbuilddeps.txt - # -r src/glance/requirements.txt - # castellan - # cursive - # keystonemiddleware - # openstacksdk - # os-brick - # os-service-types - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-reports - # oslo-rootwrap - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # taskflow platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # automaton - # cliff - # oslo-upgradecheck - # osprofiler - # python-cinderclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics prompt-toolkit==3.0.52 - # via - # -c upper-constraints.txt.master - # cmd2 psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # openstacksdk - # os-brick - # oslo-reports - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pydot==4.0.1 - # via - # -c upper-constraints.txt.master - # taskflow pygments==2.20.0 - # via - # -c upper-constraints.txt.master - # rich pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # httplib2 - # oslo-utils - # pydot pyperclip==1.11.0 - # via - # -c upper-constraints.txt.master - # cmd2 python-barbicanclient==7.5.0 - # via - # -c upper-constraints.txt.master - # castellan python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-cinderclient==9.9.0 - # via - # -c upper-constraints.txt.master - # glance-store python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # oslo-log python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # glance-store - # keystonemiddleware python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache python-swiftclient==4.10.0 - # via - # -c upper-constraints.txt.master - # glance-store pytz==2026.2 - # via - # -c upper-constraints.txt.master - # wsme pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # cliff - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # castellan - # glance-store - # keystoneauth1 - # keystonemiddleware - # os-brick - # oslo-config - # oslo-policy - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-swiftclient retrying==1.4.2 - # via - # -c upper-constraints.txt.master - # glance-store rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config rich==15.0.0 - # via - # -c upper-constraints.txt.master - # cmd2 - # rich-argparse rich-argparse==1.8.0 - # via - # -c upper-constraints.txt.master - # cmd2 routes==2.5.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing simplegeneric==0.8.1 - # via - # -c upper-constraints.txt.master - # wsme six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # alembic - # oslo-db statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # castellan - # cliff - # dogpile-cache - # glance-store - # keystoneauth1 - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-cinderclient - # python-keystoneclient - # taskflow taskflow==6.3.0 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt tenacity==9.1.4 - # via - # -c upper-constraints.txt.master - # os-brick - # taskflow testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # automaton - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable - # prompt-toolkit webob==1.8.10 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt - # keystonemiddleware - # oslo-messaging - # oslo-middleware - # oslo-service - # osprofiler - # wsme wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector wsme==0.12.1 - # via - # -c upper-constraints.txt.master - # -r src/glance/requirements.txt yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service zipp==4.1.0 - # via - # -c upper-constraints.txt.master - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/manila/buildrequirements.lock.master b/containers/manila/buildrequirements.lock.master index c9246d98..db159a4b 100644 --- a/containers/manila/buildrequirements.lock.master +++ b/containers/manila/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# build==1.5.0 calver==2025.10.20 cffi==2.0.0 @@ -27,7 +23,7 @@ tomlkit==0.15.1 trove-classifiers==2026.6.1.19 typing-extensions==4.15.0 vcs-versioning==2.2.4 -wheel==0.47.0 +wheel==0.48.0 # The following packages are considered to be unsafe in a requirements file: setuptools==82.0.1 diff --git a/containers/manila/requirements.lock.master b/containers/manila/requirements.lock.master index c6a6a0d7..57fe0dfa 100644 --- a/containers/manila/requirements.lock.master +++ b/containers/manila/requirements.lock.master @@ -1,715 +1,122 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras pythonbuilddeps.txt pythondeps.txt src/manila/requirements.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing autopage==0.6.0 - # via - # -c upper-constraints.txt.master - # cliff bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # paramiko cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-messaging castellan==5.8.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt certifi==2026.7.22 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography - # oslo-privsep - # pynacl charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cliff==4.15.0 - # via - # -c upper-constraints.txt.master - # osc-lib - # python-barbicanclient - # python-neutronclient cmd2==4.0.0 - # via - # -c upper-constraints.txt.master - # cliff cotyledon==2.2.0 - # via - # -c upper-constraints.txt.master - # oslo-service debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # futurist - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-privsep - # oslo-rootwrap - # oslo-service - # python-keystoneclient - # python-neutronclient decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk defusedxml==0.7.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency - # tooz futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # oslo-service - # tooz greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # eventlet - # oslo-service - # sqlalchemy idna==3.18 - # via - # -c upper-constraints.txt.master - # requests invoke==3.0.3 - # via - # -c upper-constraints.txt.master - # paramiko iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk - # oslo-utils - # python-novaclient jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # oslo-reports jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # openstacksdk - # osc-lib - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging lxml==6.1.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic markdown-it-py==4.2.0 - # via - # -c upper-constraints.txt.master - # rich markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako mdurl==0.1.2 - # via - # -c upper-constraints.txt.master - # markdown-it-py msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-privsep - # oslo-serialization - # tooz netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-config - # oslo-utils - # osprofiler - # python-neutronclient openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # osc-lib - # python-neutronclient os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk osc-lib==4.7.0 - # via - # -c upper-constraints.txt.master - # python-neutronclient oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service - # osprofiler oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # osprofiler - # pycadf - # python-keystoneclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r pythondeps.txt - # -r src/manila/requirements.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # oslo-cache - # oslo-messaging - # oslo-metrics - # oslo-privsep - # oslo-service - # python-neutronclient oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-upgradecheck oslo-privsep==3.12.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-reports==3.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-rootwrap==7.10.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-policy - # oslo-reports - # osprofiler - # pycadf - # python-barbicanclient - # python-keystoneclient - # python-neutronclient - # python-novaclient - # tooz oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-messaging oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-privsep - # oslo-reports - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient - # tooz osprofiler==4.4.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # kombu - # oslo-utils - # python-keystoneclient paramiko==4.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt paste==3.10.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r pythonbuilddeps.txt - # -r src/manila/requirements.txt - # castellan - # keystonemiddleware - # openstacksdk - # os-service-types - # osc-lib - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-reports - # oslo-rootwrap - # oslo-utils - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient - # python-novaclient platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # cliff - # oslo-upgradecheck - # osprofiler - # python-cinderclient - # python-novaclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics prompt-toolkit==3.0.52 - # via - # -c upper-constraints.txt.master - # cmd2 psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-reports - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pygments==2.20.0 - # via - # -c upper-constraints.txt.master - # rich pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pynacl==1.6.2 - # via - # -c upper-constraints.txt.master - # paramiko pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-utils pyperclip==1.11.0 - # via - # -c upper-constraints.txt.master - # cmd2 python-barbicanclient==7.5.0 - # via - # -c upper-constraints.txt.master - # castellan python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-cinderclient==9.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # oslo-log python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # python-neutronclient python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache python-neutronclient==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt python-novaclient==18.13.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # cliff - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # keystoneauth1 - # keystonemiddleware - # osc-lib - # oslo-config - # oslo-policy - # osprofiler - # python-barbicanclient - # python-cinderclient - # python-keystoneclient - # python-neutronclient rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config rich==15.0.0 - # via - # -c upper-constraints.txt.master - # cmd2 - # rich-argparse rich-argparse==1.8.0 - # via - # -c upper-constraints.txt.master - # cmd2 routes==2.5.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing setproctitle==1.3.7 - # via - # -c upper-constraints.txt.master - # cotyledon six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # alembic - # oslo-db - # sqlalchemy-utils sqlalchemy-utils==0.42.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # castellan - # cliff - # dogpile-cache - # keystoneauth1 - # osc-lib - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-cinderclient - # python-keystoneclient - # python-novaclient - # tooz tenacity==9.1.4 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # tooz testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db tooz==9.0.1 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # cotyledon - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu voluptuous==0.16.0 - # via - # -c upper-constraints.txt.master - # tooz wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable - # prompt-toolkit webob==1.8.10 - # via - # -c upper-constraints.txt.master - # -r src/manila/requirements.txt - # keystonemiddleware - # oslo-messaging - # oslo-middleware - # oslo-service - # osprofiler wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/watcher/buildrequirements.lock.master b/containers/watcher/buildrequirements.lock.master index f9077f6e..dafe28fa 100644 --- a/containers/watcher/buildrequirements.lock.master +++ b/containers/watcher/buildrequirements.lock.master @@ -1,7 +1,3 @@ -# -# -# pybuild-deps compile --no-annotate --output-file=buildrequirements.lock.master requirements.lock.master -# build==1.5.0 calver==2025.10.20 coherent-licensed==0.5.2 diff --git a/containers/watcher/requirements.lock.master b/containers/watcher/requirements.lock.master index 7cfa9716..3c7e735e 100644 --- a/containers/watcher/requirements.lock.master +++ b/containers/watcher/requirements.lock.master @@ -1,739 +1,130 @@ -# -# -# pip-compile --allow-unsafe --constraint=upper-constraints.txt.master --output-file=requirements.lock.master --strip-extras src/watcher/requirements.txt watcher-base/pythonbuilddeps.txt watcher-base/pythondeps.txt -# alembic==1.18.5 - # via - # -c upper-constraints.txt.master - # oslo-db amqp==5.3.1 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-messaging apscheduler==3.11.2 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt attrs==26.1.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing automaton==3.4.0 - # via - # -c upper-constraints.txt.master - # taskflow autopage==0.6.0 - # via - # -c upper-constraints.txt.master - # cliff bcrypt==5.0.0 - # via - # -c upper-constraints.txt.master - # oslo-middleware cachetools==7.1.4 - # via - # -c upper-constraints.txt.master - # oslo-messaging - # taskflow certifi==2026.6.17 - # via requests cffi==2.0.0 - # via - # -c upper-constraints.txt.master - # cryptography charset-normalizer==3.4.7 - # via - # -c upper-constraints.txt.master - # requests cliff==4.15.0 - # via - # -c upper-constraints.txt.master - # gnocchiclient - # osc-lib - # python-ironicclient - # python-observabilityclient - # python-openstackclient cmd2==4.0.0 - # via - # -c upper-constraints.txt.master - # cliff cotyledon==2.2.0 - # via - # -c upper-constraints.txt.master - # oslo-service croniter==6.2.2 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt debtcollector==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # futurist - # gnocchiclient - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-service - # python-keystoneclient - # python-manilaclient - # taskflow decorator==5.3.1 - # via - # -c upper-constraints.txt.master - # dogpile-cache - # openstacksdk dnspython==2.8.0 - # via - # -c upper-constraints.txt.master - # eventlet dogpile-cache==1.5.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-cache - # python-ironicclient eventlet==0.41.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-service fasteners==0.20 - # via - # -c upper-constraints.txt.master - # oslo-concurrency - # taskflow futurist==3.4.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # gnocchiclient - # oslo-messaging - # oslo-service - # taskflow gnocchiclient==7.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt greenlet==3.5.3 - # via - # -c upper-constraints.txt.master - # eventlet - # oslo-service - # sqlalchemy idna==3.18 - # via - # -c upper-constraints.txt.master - # requests importlib-metadata==9.0.0 - # via - # -c upper-constraints.txt.master - # wsme iso8601==2.1.0 - # via - # -c upper-constraints.txt.master - # gnocchiclient - # keystoneauth1 - # openstacksdk - # oslo-utils - # python-openstackclient jinja2==3.1.6 - # via - # -c upper-constraints.txt.master - # oslo-middleware - # oslo-reports jmespath==1.1.0 - # via - # -c upper-constraints.txt.master - # openstacksdk jsonpatch==1.33 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # openstacksdk jsonpointer==3.1.1 - # via - # -c upper-constraints.txt.master - # jsonpatch jsonschema==4.26.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # python-ironicclient - # taskflow jsonschema-specifications==2025.9.1 - # via - # -c upper-constraints.txt.master - # jsonschema keystoneauth1==5.15.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # gnocchiclient - # keystonemiddleware - # openstacksdk - # osc-lib - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # python-observabilityclient keystonemiddleware==13.0.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt kombu==5.6.2 - # via - # -c upper-constraints.txt.master - # oslo-messaging lxml==6.1.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt mako==1.3.12 - # via - # -c upper-constraints.txt.master - # alembic - # pecan markdown-it-py==4.2.0 - # via - # -c upper-constraints.txt.master - # rich markupsafe==3.0.3 - # via - # -c upper-constraints.txt.master - # jinja2 - # mako mdurl==0.1.2 - # via - # -c upper-constraints.txt.master - # markdown-it-py microversion-parse==2.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt msgpack==1.2.1 - # via - # -c upper-constraints.txt.master - # oslo-serialization netaddr==1.3.0 - # via - # -c upper-constraints.txt.master - # oslo-config - # oslo-utils - # oslo-versionedobjects - # wsme networkx==3.6.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # taskflow openstacksdk==4.18.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # osc-lib - # python-ironicclient - # python-openstackclient os-resource-classes==1.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt os-service-types==1.8.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # openstacksdk osc-lib==4.7.0 - # via - # -c upper-constraints.txt.master - # python-ironicclient - # python-manilaclient - # python-observabilityclient - # python-openstackclient oslo-cache==4.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # -r watcher-base/pythondeps.txt - # keystonemiddleware oslo-concurrency==7.6.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-service oslo-config==10.6.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # pycadf - # python-keystoneclient - # python-manilaclient oslo-context==6.5.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-middleware - # oslo-policy - # oslo-versionedobjects oslo-db==18.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # -r watcher-base/pythondeps.txt oslo-i18n==6.9.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-config - # oslo-db - # oslo-log - # oslo-middleware - # oslo-policy - # oslo-reports - # oslo-service - # oslo-upgradecheck - # oslo-utils - # oslo-versionedobjects - # python-keystoneclient - # python-openstackclient oslo-log==8.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-cache - # oslo-messaging - # oslo-metrics - # oslo-service - # oslo-versionedobjects - # python-manilaclient oslo-messaging==18.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-versionedobjects oslo-metrics==0.16.0 - # via - # -c upper-constraints.txt.master - # oslo-messaging oslo-middleware==8.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-messaging oslo-policy==6.0.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-upgradecheck oslo-reports==3.9.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt oslo-serialization==5.11.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # oslo-log - # oslo-messaging - # oslo-policy - # oslo-reports - # pycadf - # python-keystoneclient - # python-manilaclient - # taskflow oslo-service==4.8.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-messaging - # oslo-service oslo-upgradecheck==2.8.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt oslo-utils==10.1.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # osc-lib - # oslo-cache - # oslo-concurrency - # oslo-db - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-policy - # oslo-reports - # oslo-serialization - # oslo-service - # oslo-upgradecheck - # oslo-versionedobjects - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # taskflow oslo-versionedobjects==3.11.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt packaging==26.2 - # via - # -c upper-constraints.txt.master - # kombu - # oslo-utils - # python-keystoneclient - # wheel paste==3.10.1 - # via - # -c upper-constraints.txt.master - # oslo-service pastedeploy==3.1.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # oslo-service pbr==7.0.3 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # -r watcher-base/pythonbuilddeps.txt - # keystonemiddleware - # openstacksdk - # os-resource-classes - # os-service-types - # osc-lib - # oslo-concurrency - # oslo-context - # oslo-i18n - # oslo-log - # oslo-messaging - # oslo-metrics - # oslo-middleware - # oslo-reports - # oslo-utils - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # taskflow pecan==1.8.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt platformdirs==4.10.0 - # via - # -c upper-constraints.txt.master - # openstacksdk - # python-ironicclient prettytable==3.18.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # automaton - # cliff - # oslo-upgradecheck - # python-manilaclient prometheus-client==0.25.0 - # via - # -c upper-constraints.txt.master - # oslo-metrics prompt-toolkit==3.0.52 - # via - # -c upper-constraints.txt.master - # cmd2 psutil==7.2.2 - # via - # -c upper-constraints.txt.master - # openstacksdk - # oslo-reports - # oslo-utils pycadf==4.1.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pycparser==3.0 - # via - # -c upper-constraints.txt.master - # cffi pydot==4.0.1 - # via - # -c upper-constraints.txt.master - # taskflow pygments==2.20.0 - # via - # -c upper-constraints.txt.master - # rich pyjwt==2.13.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware pymemcache==4.0.0 - # via - # -c upper-constraints.txt.master - # oslo-cache pymysql==1.2.0 - # via - # -c upper-constraints.txt.master - # oslo-db pyparsing==3.3.2 - # via - # -c upper-constraints.txt.master - # oslo-utils - # pydot pyperclip==1.11.0 - # via - # -c upper-constraints.txt.master - # cmd2 python-binary-memcached==0.32.0 - # via - # -c upper-constraints.txt.master - # oslo-cache python-dateutil==2.9.0.post0 - # via - # -c upper-constraints.txt.master - # croniter - # gnocchiclient - # oslo-log python-ironicclient==6.2.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt python-keystoneclient==5.8.0 - # via - # -c upper-constraints.txt.master - # keystonemiddleware - # python-openstackclient python-manilaclient==6.2.0 - # via - # -c upper-constraints.txt.master - # python-openstackclient python-memcached==1.62 - # via - # -c upper-constraints.txt.master - # oslo-cache python-observabilityclient==1.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt python-openstackclient==10.2.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt pytz==2026.2 - # via - # -c upper-constraints.txt.master - # wsme pyyaml==6.0.3 - # via - # -c upper-constraints.txt.master - # cliff - # openstacksdk - # oslo-config - # oslo-messaging - # oslo-policy - # oslo-utils - # python-ironicclient - # python-observabilityclient redis==8.0.1 - # via - # -c upper-constraints.txt.master - # oslo-cache referencing==0.37.0 - # via - # -c upper-constraints.txt.master - # jsonschema - # jsonschema-specifications repoze-lru==0.8 - # via - # -c upper-constraints.txt.master - # routes requests==2.34.2 - # via - # -c upper-constraints.txt.master - # keystoneauth1 - # keystonemiddleware - # osc-lib - # oslo-config - # oslo-policy - # python-ironicclient - # python-keystoneclient - # python-manilaclient - # python-openstackclient rfc3986==2.0.0 - # via - # -c upper-constraints.txt.master - # oslo-config rich==15.0.0 - # via - # -c upper-constraints.txt.master - # cmd2 - # rich-argparse rich-argparse==1.8.0 - # via - # -c upper-constraints.txt.master - # cmd2 routes==2.5.1 - # via - # -c upper-constraints.txt.master - # oslo-service rpds-py==2026.5.1 - # via - # -c upper-constraints.txt.master - # jsonschema - # referencing setproctitle==1.3.7 - # via - # -c upper-constraints.txt.master - # cotyledon simplegeneric==0.8.1 - # via - # -c upper-constraints.txt.master - # wsme six==1.17.0 - # via - # -c upper-constraints.txt.master - # python-binary-memcached - # python-dateutil - # routes sqlalchemy==2.0.51 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # alembic - # oslo-db statsd==4.0.1 - # via - # -c upper-constraints.txt.master - # oslo-middleware stevedore==5.9.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # cliff - # dogpile-cache - # keystoneauth1 - # osc-lib - # oslo-config - # oslo-db - # oslo-messaging - # oslo-middleware - # oslo-policy - # python-ironicclient - # python-keystoneclient - # python-openstackclient - # taskflow taskflow==6.3.0 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt tenacity==9.1.4 - # via - # -c upper-constraints.txt.master - # taskflow testresources==2.1.2 - # via - # -c upper-constraints.txt.master - # oslo-db testscenarios==0.6.2 - # via - # -c upper-constraints.txt.master - # oslo-db typing-extensions==4.15.0 - # via - # -c upper-constraints.txt.master - # alembic - # automaton - # cotyledon - # keystoneauth1 - # os-service-types - # oslo-context - # oslo-middleware - # referencing - # sqlalchemy tzdata==2026.2 - # via - # -c upper-constraints.txt.master - # kombu tzlocal==5.4.3 - # via - # -c upper-constraints.txt.master - # apscheduler uhashring==2.4 - # via - # -c upper-constraints.txt.master - # python-binary-memcached ujson==5.13.0 - # via - # -c upper-constraints.txt.master - # gnocchiclient urllib3==2.7.0 - # via - # -c upper-constraints.txt.master - # requests vine==5.1.0 - # via - # -c upper-constraints.txt.master - # amqp - # kombu wcwidth==0.8.1 - # via - # -c upper-constraints.txt.master - # prettytable - # prompt-toolkit webob==1.8.10 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt - # keystonemiddleware - # microversion-parse - # oslo-messaging - # oslo-middleware - # oslo-service - # pecan - # wsme wheel==0.47.0 - # via -r watcher-base/pythonbuilddeps.txt wrapt==2.2.2 - # via - # -c upper-constraints.txt.master - # debtcollector wsme==0.12.1 - # via - # -c upper-constraints.txt.master - # -r src/watcher/requirements.txt yappi==1.7.6 - # via - # -c upper-constraints.txt.master - # oslo-service zipp==4.1.0 - # via - # -c upper-constraints.txt.master - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/containers/watcher/rpms.in.yaml b/containers/watcher/rpms.in.yaml index ada31b87..9a0608dc 100644 --- a/containers/watcher/rpms.in.yaml +++ b/containers/watcher/rpms.in.yaml @@ -27,6 +27,7 @@ packages: - libxslt-devel - mod_ssl - openssl-devel + - openssl-libs - python3 - python3-cryptography - python3-devel diff --git a/containers/watcher/watcher-base/bindeps.txt b/containers/watcher/watcher-base/bindeps.txt index ade6c10f..6b9ce44f 100644 --- a/containers/watcher/watcher-base/bindeps.txt +++ b/containers/watcher/watcher-base/bindeps.txt @@ -9,3 +9,4 @@ python3-cryptography libffi libxml2 libxslt +openssl-libs diff --git a/docs/TESTING.md b/docs/TESTING.md index d76bb1a3..9c50e7e2 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,13 +1,15 @@ # Testing -Run the repository's current test environment with: +Run the stdlib unittest suite through any supported alias: ```console +tox -e unit tox -e test +tox -e py3 ``` -This executes `tests/test_update_sources.sh`, which exercises source update and -lock-generation behavior with local temporary Git repositories. +The suite exercises source update and lock-generation behavior with local +temporary Git repositories. Run repository lint checks with: @@ -19,5 +21,17 @@ The linter environment checks tracked Containerfiles and shell content through pre-commit. Use the narrowest applicable command first, then run both environments before proposing changes to build or source-maintenance behavior. +Regenerate dependency locks from committed source pins with Python 3.12: + +```console +uvx --python 3.12 tox -e update-lockfiles +``` + +The environment rejects other Python minor versions because environment +markers can resolve a different package set. A clean regeneration must leave no +tracked diff under `containers/`. Inspect the corresponding +`.tmp/source-maintenance/frozen-source-refs..tsv` manifest to confirm +that pinned regeneration used `committed-pin` authority throughout. + Additional change-specific validation is described in the [developer guide](developer-guide.md#ci-path-filtering). diff --git a/docs/developer-guide.md b/docs/developer-guide.md index df913c69..cae48b2e 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -241,24 +241,43 @@ STREAM=master ./build.sh update-sources ``` This will: -1. Clone each source repo at the branch tip to resolve the latest commit hash. -2. Update `sources.txt` with the new pinned hashes. -3. Fetch `upper-constraints.txt` from the requirements repo. +1. Resolve every selected branch or tag to one exact commit before mutation. +2. Update `sources.txt` with the frozen commits. +3. Fetch `upper-constraints.txt` from the same frozen input set. 4. Generate `rpms.in.yaml` from all `bindeps.txt` + `builddeps.txt` files. 5. Run `pip-compile` to generate `requirements.lock.`. 6. Run `pybuild-deps compile` to generate `buildrequirements.lock.`. 7. Create default-stream symlinks if `STREAM == DEFAULT_STREAM`. -Auto-cloned repos in `src/` are cleaned up automatically on exit. -Pre-existing checkouts in `src/` are used as-is and not removed. +The preflight record is written to +`.tmp/source-maintenance/frozen-source-refs..tsv`. It records each +source manifest, declared ref, committed pin, frozen commit, and authority. +Advancing runs use `declared-ref`; pinned runs use `committed-pin`; intentional +Git checkouts already under `src/` use `pre-existing-checkout`. A slash in a +stream name is encoded as `%2F`, and unsafe stream names are rejected before +filesystem mutation. + +All selected records must pass preflight before a tracked source manifest is +changed. The fetched repositories are retained for the complete run so a moving +branch cannot provide different content after preflight. Auto-created source +checkouts are cleaned up on exit. Pre-existing checkouts are used as-is and are +not removed. To regenerate lockfiles without updating pinned hashes (steps 1--3 are -skipped; repos are cloned at the existing pinned hashes instead): +skipped; repos are cloned at the existing pinned hashes instead), use the +canonical Python 3.12 environment: ```bash -STREAM=master SKIP_HASH_UPDATE=1 ./build.sh update-sources +STREAM=master uvx --python 3.12 tox -e update-lockfiles -- ``` +Python environment markers are evaluated by the generator interpreter, so +other Python minor versions can produce a different dependency set. The tox +environment rejects non-3.12 interpreters and pins the generator tool versions. +Generated lock headers, resolver annotations, and package-index directives are +removed because they describe the generation environment rather than the +resolved dependency set. + ### Building images ```bash @@ -289,6 +308,89 @@ All image tags are verified to exist locally before any push begins. ./build.sh list ``` +## Zuul content provider + +The `s2i-openstack-container-content-provider` job runs on the +CentOS Stream 10 nodeset host named `builder`. All host preparation, registry +validation, builds, publication, result generation, and cleanup target that +host explicitly. The Zuul executor controls Ansible but does not perform those +mutations. + +In this repository the provider defaults to `all`, so every maintained image +is built from its exact `sources.txt` pins. A child job can set `s2i_ci_images` +to an explicit list of image targets; the provider adds `base`, resolves the +selection through `build.sh`, and publishes only that set. + +To compose the provider in an operator repository, inherit the job, add the +container repository and any related repositories to `required-projects`, set +`s2i_ci_container_project`, and override `s2i_ci_images` with the required +subset. Zuul places those projects in the shared buildset workspace, but this +provider does not yet consume speculative service checkouts or build operators. +Those integrations remain follow-up scope; the current provider always uses +maintained source pins. + +### Image deployment metadata + +The repository-level `containers/image-mappings.yaml` associates exact build +targets with OpenStackVersion custom-image fields. Unlisted targets have no +deployment mapping but are still built and returned. The consolidated +`watcher/watcher-base` image declares: + +```yaml +openstack_version: + custom_container_images: + watcher/watcher-base: + - watcherAPIImage + - watcherApplierImage + - watcherDecisionEngineImage +``` + +All three keys resolve to the same exact `openstack-watcher-base` reference. +The image contains the API, applier, and decision-engine entry points and the +union of their runtime dependencies. Watcher is intentionally not split into +process-specific images. + +Both Cyborg images build and publish but are absent from the central mapping. +Their exact references therefore appear in provider diagnostics without adding +fields to the default deployment map. + +A child job may provide `s2i_ci_image_mappings` as a mapping from a selected +image target to a replacement list of keys. Replacement is per image rather +than additive. The provider records whether each effective list came from +tracked or inventory metadata and rejects malformed values, unknown or unbuilt +image targets, empty key strings, duplicate keys, and a key assigned to more +than one image. + +### Registry and returned data + +The provider starts or inherits a Zuul buildset registry, validates push and +pull with a dedicated UBI tag, builds and pushes the selected image set, and +pulls every exact result back. Credentials and certificate data remain in +Zuul secret data. Returned public diagnostics use the buildset registry's +reachable host or IP and port, never the builder-local registry alias. + +`s2i_ci_content.images` contains every exact successful reference, including +base and both unmapped Cyborg images. The partial +`content_provider_os_custom_container_images` map contains only effective +keys joined to exact successful references. The legacy global OS registry URL +remains the neutral `null` sentinel, while its namespace/tag and gating-repo +fields remain empty or false because this selective provider does not publish a +complete OpenStack image namespace. `cifmw_build_images_output` +remains an empty mapping and is not repurposed for service images. + +Intended references are written before build mutation. Post-run cleanup +removes only those exact Podman pullback and Buildah build tags, verifies exact +absence, and removes a buildset registry only when its ownership marker is +valid. +Per-image parallel logs and registry/result manifests are retained under +`zuul-output/logs/container-build/`. + +The provider pauses while dependent jobs run. Private onboarding may attach a +trivial child that prints the returned registry paths and maps. That debug job +does not pull images, patch an OpenStackVersion resource, deploy OpenStack, or +invoke a downstream repository's playbooks. Downstream consumption is separate +work. + ## Build architecture ### Base image (`openstack-base`) diff --git a/molecule/provider-contract/cleanup.yml b/molecule/provider-contract/cleanup.yml new file mode 100644 index 00000000..9b147fe0 --- /dev/null +++ b/molecule/provider-contract/cleanup.yml @@ -0,0 +1,13 @@ +--- +- name: Remove provider contract fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_root: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract + + tasks: + - name: Remove provider contract fixture state + ansible.builtin.file: + path: "{{ s2i_test_root }}" + state: absent diff --git a/molecule/provider-contract/converge.yml b/molecule/provider-contract/converge.yml new file mode 100644 index 00000000..bb0c1ba1 --- /dev/null +++ b/molecule/provider-contract/converge.yml @@ -0,0 +1,126 @@ +--- +- name: Validate the fake normalized registry + ansible.builtin.import_playbook: >- + ../../playbooks/container-ci/shared/validate-registry.yaml + vars: + s2i_ci_output_dir: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract/zuul-output + +- name: Prepare exact cleanup fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + s2i_test_images: + - registry.test:5000/openstack/openstack-base:test + - registry.test:5000/openstack/openstack-watcher-base:test + s2i_ci_container_repo: "{{ s2i_test_repo_root }}" + s2i_ci_selected_images: + - base + - cyborg/cyborg + - watcher/watcher-base + s2i_ci_image_mappings: + cyborg/cyborg: + - futureCyborgImage + + tasks: + - name: Initialize metadata expansion fixtures + ansible.builtin.set_fact: + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Load tracked image mappings + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/load-image-mappings.yaml + + - name: Expand tracked and inventory image metadata + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-deployment-keys.yaml + + - name: Require tracked and replacement metadata expansion + ansible.builtin.assert: + that: + - s2i_ci_effective_image_mappings['base'] == [] + - s2i_ci_effective_image_mappings['cyborg/cyborg'] == + ['futureCyborgImage'] + - s2i_ci_effective_image_mappings['watcher/watcher-base'] | length == 3 + - s2i_ci_mapping_sources['cyborg/cyborg'] == 'inventory' + - s2i_ci_mapping_sources['watcher/watcher-base'] == 'tracked' + + - name: Initialize exact reference expansion fixtures + ansible.builtin.set_fact: + s2i_ci_target_references: + base: registry.test:5000/openstack/openstack-base:test + cyborg/cyborg: registry.test:5000/openstack/openstack-cyborg:test + watcher/watcher-base: >- + registry.test:5000/openstack/openstack-watcher-base:test + s2i_ci_custom_container_images: {} + + - name: Expand exact metadata references + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + + - name: Require consolidated Watcher reference expansion + ansible.builtin.assert: + that: + - s2i_ci_custom_container_images.watcherAPIImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.watcherApplierImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.watcherDecisionEngineImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.futureCyborgImage == + s2i_ci_target_references['cyborg/cyborg'] + + - name: Read fake container storage after registry validation + ansible.builtin.slurp: + src: "{{ s2i_test_root }}/state.json" + register: s2i_test_state_data + + - name: Decode fake container storage + ansible.builtin.set_fact: + s2i_test_state: >- + {{ s2i_test_state_data.content | b64decode | from_json }} + + - name: Add exact workflow tags to both storage clients + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/state.json" + mode: "0600" + content: | + {{ s2i_test_state | + combine({ + 'podman': s2i_test_state.podman + s2i_test_images, + 'buildah': s2i_test_images + }) | to_nice_json }} + + - name: Write intended exact cleanup metadata + ansible.builtin.copy: + dest: >- + {{ s2i_test_root }}/zuul-output/logs/container-build/intended-images.json + mode: "0644" + content: | + {{ { + 'registry': 'registry.test:5000', + 'namespace': 'openstack', + 'images': s2i_test_images, + 'selected_images': ['base', 'watcher/watcher-base'], + 'target': 'watcher/watcher-base' + } | to_nice_json }} + +- name: Exercise exact shared cleanup + ansible.builtin.import_playbook: >- + ../../playbooks/container-ci/shared/cleanup-images.yaml + vars: + s2i_ci_output_dir: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract/zuul-output diff --git a/molecule/provider-contract/duplicate-metadata.yml b/molecule/provider-contract/duplicate-metadata.yml new file mode 100644 index 00000000..faedb612 --- /dev/null +++ b/molecule/provider-contract/duplicate-metadata.yml @@ -0,0 +1,16 @@ +--- +- name: Reject duplicate deployment keys + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_effective_image_mappings: + base: + - duplicateImage + watcher/watcher-base: + - duplicateImage + + tasks: + - name: Validate duplicate fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-deployment-keys.yaml diff --git a/molecule/provider-contract/files/fake-container-client.py b/molecule/provider-contract/files/fake-container-client.py new file mode 100755 index 00000000..ade2a2cb --- /dev/null +++ b/molecule/provider-contract/files/fake-container-client.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib +import sys + + +state_path = pathlib.Path(os.environ["S2I_FAKE_STATE"]) +client = pathlib.Path(sys.argv[0]).name +arguments = sys.argv[1:] +state = json.loads(state_path.read_text(encoding="utf-8")) +state["commands"].append([client, *arguments]) +images = state[client] +return_code = 0 + +if client == "podman": + if arguments[:1] == ["pull"]: + images.append(arguments[-1]) + elif arguments[:1] == ["tag"]: + images.append(arguments[-1]) + elif arguments[:1] == ["push"]: + pass + elif arguments[:1] == ["untag"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + elif arguments[:2] in (["image", "inspect"], ["image", "exists"]): + return_code = 0 if arguments[-1] in images else 1 + elif arguments[:2] == ["image", "rm"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + else: + return_code = 1 + elif arguments[:1] == ["images"]: + print("\n".join(images)) + else: + return_code = 2 +elif client == "buildah": + if arguments[:1] == ["inspect"]: + return_code = 0 if arguments[-1] in images else 125 + elif arguments[:1] == ["rmi"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + else: + return_code = 125 + elif arguments[:1] == ["images"]: + print("\n".join(images)) + else: + return_code = 2 +else: + return_code = 2 + +state[client] = list(dict.fromkeys(images)) +state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") +sys.exit(return_code) diff --git a/molecule/provider-contract/malformed-mapping.yml b/molecule/provider-contract/malformed-mapping.yml new file mode 100644 index 00000000..60c20335 --- /dev/null +++ b/molecule/provider-contract/malformed-mapping.yml @@ -0,0 +1,18 @@ +--- +- name: Reject malformed mapping value + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_container_repo: "{{ s2i_test_repo_root }}" + s2i_ci_image_mappings: + base: not-a-list + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + tasks: + - name: Validate malformed mapping fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/image-metadata-item.yaml + vars: + s2i_ci_image: base diff --git a/molecule/provider-contract/molecule.yml b/molecule/provider-contract/molecule.yml new file mode 100644 index 00000000..d3eed451 --- /dev/null +++ b/molecule/provider-contract/molecule.yml @@ -0,0 +1,34 @@ +--- +driver: + name: default + options: + managed: false + ansible_connection_options: + ansible_connection: local + +platforms: + - name: localhost + groups: + - builder + +provisioner: + name: ansible + env: + ANSIBLE_LOCAL_TEMP: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/ansible/local-tmp" + PATH: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/molecule/provider-contract/bin:${PATH}" + S2I_FAKE_STATE: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/molecule/provider-contract/state.json" + inventory: + host_vars: + localhost: + ansible_connection: local + +verifier: + name: ansible + +scenario: + test_sequence: + - syntax + - prepare + - converge + - verify + - cleanup diff --git a/molecule/provider-contract/prepare.yml b/molecule/provider-contract/prepare.yml new file mode 100644 index 00000000..3874ab96 --- /dev/null +++ b/molecule/provider-contract/prepare.yml @@ -0,0 +1,57 @@ +--- +- name: Prepare provider contract fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + + tasks: + - name: Reset provider contract fixture state + ansible.builtin.file: + path: "{{ s2i_test_root }}" + state: absent + + - name: Create provider contract fixture directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ s2i_test_root }}/bin" + - "{{ s2i_test_root }}/zuul-output/logs/container-build" + + - name: Install the fake container client + ansible.builtin.copy: + src: fake-container-client.py + dest: "{{ s2i_test_root }}/bin/fake-container-client.py" + mode: "0755" + + - name: Link fake Podman and Buildah clients + ansible.builtin.file: + src: "{{ s2i_test_root }}/bin/fake-container-client.py" + dest: "{{ s2i_test_root }}/bin/{{ item }}" + state: link + loop: + - podman + - buildah + + - name: Initialize fake container storage + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/state.json" + mode: "0600" + content: | + {{ {'podman': [], 'buildah': [], 'commands': []} | to_nice_json }} + + - name: Write normalized registry connection + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/zuul-output/.s2i-ci-registry-connection.json" + mode: "0600" + content: | + {{ { + 'endpoint': 'registry.test:5000', + 'public_host': '192.0.2.10', + 'public_port': 5000, + 'auth_file': '', + 'cert_dir': '' + } | to_nice_json }} diff --git a/molecule/provider-contract/unknown-mapping.yml b/molecule/provider-contract/unknown-mapping.yml new file mode 100644 index 00000000..73dec15b --- /dev/null +++ b/molecule/provider-contract/unknown-mapping.yml @@ -0,0 +1,17 @@ +--- +- name: Reject unknown mapping target + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_selected_images: + - base + - watcher/watcher-base + s2i_ci_image_mappings: + cyborg/cyborg: + - unknownImage + + tasks: + - name: Validate unknown mapping fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-mapping-overrides.yaml diff --git a/molecule/provider-contract/verify.yml b/molecule/provider-contract/verify.yml new file mode 100644 index 00000000..0243d40a --- /dev/null +++ b/molecule/provider-contract/verify.yml @@ -0,0 +1,89 @@ +--- +- name: Verify provider registry and cleanup contracts + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + s2i_test_validation_image: >- + registry.test:5000/s2i-validation/ubi10-minimal:latest + s2i_test_images: + - registry.test:5000/openstack/openstack-base:test + - registry.test:5000/openstack/openstack-watcher-base:test + + tasks: + - name: Exercise invalid metadata contracts + ansible.builtin.command: + argv: + - ansible-playbook + - -i + - builder, + - -c + - local + - "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/{{ item }}" + loop: + - duplicate-metadata.yml + - unknown-mapping.yml + - malformed-mapping.yml + register: s2i_test_invalid_metadata + changed_when: false + failed_when: false + + - name: Require invalid metadata to fail deterministically + ansible.builtin.assert: + that: + - s2i_test_invalid_metadata.results | + map(attribute='rc') | reject('equalto', 0) | list | length == 3 + - >- + 'A deployment key may be assigned to only one image' in + (s2i_test_invalid_metadata.results[0].stdout + + s2i_test_invalid_metadata.results[0].stderr) + - >- + 'Image mapping overrides must name selected image targets' in + (s2i_test_invalid_metadata.results[1].stdout + + s2i_test_invalid_metadata.results[1].stderr) + - >- + 'Invalid effective image mapping for base' in + (s2i_test_invalid_metadata.results[2].stdout + + s2i_test_invalid_metadata.results[2].stderr) + + - name: Read final fake container storage + ansible.builtin.slurp: + src: "{{ s2i_test_root }}/state.json" + register: s2i_test_state_data + + - name: Decode final fake container storage + ansible.builtin.set_fact: + s2i_test_state: >- + {{ s2i_test_state_data.content | b64decode | from_json }} + + - name: Validate exact cleanup and retained source image + ansible.builtin.assert: + that: + - s2i_test_images | intersect(s2i_test_state.podman) | length == 0 + - s2i_test_images | intersect(s2i_test_state.buildah) | length == 0 + - s2i_test_validation_image not in s2i_test_state.podman + - >- + 'registry.access.redhat.com/ubi10/ubi-minimal:latest' in + s2i_test_state.podman + - >- + ['podman', 'push', '--remove-signatures', + 'registry.test:5000/s2i-validation/ubi10-minimal:latest'] in + s2i_test_state.commands + - >- + ['podman', 'image', 'rm', '--force', s2i_test_images[0]] in + s2i_test_state.commands + - >- + ['buildah', 'rmi', s2i_test_images[0]] in s2i_test_state.commands + + - name: Check public registry state artifact + ansible.builtin.stat: + path: >- + {{ s2i_test_root }}/zuul-output/logs/container-build/registry-state.json + register: s2i_test_registry_state + + - name: Require registry state artifact + ansible.builtin.assert: + that: + - s2i_test_registry_state.stat.exists + - s2i_test_registry_state.stat.mode == '0644' diff --git a/playbooks/container-ci/shared/cleanup-images.yaml b/playbooks/container-ci/shared/cleanup-images.yaml new file mode 100644 index 00000000..2100cba9 --- /dev/null +++ b/playbooks/container-ci/shared/cleanup-images.yaml @@ -0,0 +1,208 @@ +--- +- name: Remove exact images recorded by the workflow + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_cleanup_images: true + + tasks: + - name: Check intended image cleanup metadata + when: s2i_ci_cleanup_images | bool + ansible.builtin.stat: + path: "{{ s2i_ci_log_dir }}/intended-images.json" + register: s2i_ci_intended_images_metadata + + - name: Load intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.slurp: + src: "{{ s2i_ci_log_dir }}/intended-images.json" + register: s2i_ci_intended_images_data + + - name: Decode intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.set_fact: + s2i_ci_cleanup_metadata: >- + {{ s2i_ci_intended_images_data.content | b64decode | from_json }} + + - name: Validate intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.assert: + that: + - s2i_ci_cleanup_metadata is mapping + - s2i_ci_cleanup_metadata.registry is string + - s2i_ci_cleanup_metadata.registry is match('^[A-Za-z0-9.:-]+$') + - s2i_ci_cleanup_metadata.namespace is string + - s2i_ci_cleanup_metadata.namespace is match('^[A-Za-z0-9_.-]+$') + - s2i_ci_cleanup_metadata.images is sequence + - s2i_ci_cleanup_metadata.images is not string + - s2i_ci_cleanup_metadata.images | unique | list | length == + s2i_ci_cleanup_metadata.images | length + - s2i_ci_cleanup_metadata.images | + select('match', '^' + s2i_ci_cleanup_metadata.registry | + regex_escape + '/' + s2i_ci_cleanup_metadata.namespace | + regex_escape + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_cleanup_metadata.images | length + fail_msg: Refusing unsafe or malformed intended image cleanup metadata + + - name: Record exact intended images for cleanup + ansible.builtin.set_fact: + s2i_ci_cleanup_images_exact: >- + {{ s2i_ci_cleanup_metadata.images | reverse | list }} + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + + - name: Default exact intended image list + ansible.builtin.set_fact: + s2i_ci_cleanup_images_exact: [] + when: + - not s2i_ci_cleanup_images | bool or + not s2i_ci_intended_images_metadata.stat.exists + + - name: Check Podman pullback tags + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + register: s2i_ci_podman_image_checks + changed_when: false + failed_when: s2i_ci_podman_image_checks.rc not in [0, 1] + + - name: Remove exact Podman pullback tags + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "{{ item.item }}" + loop: "{{ s2i_ci_podman_image_checks.results }}" + when: item.rc == 0 + changed_when: true + + - name: Check Buildah-created image tags + ansible.builtin.command: + argv: + - buildah + - inspect + - --type + - image + - "{{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + register: s2i_ci_buildah_image_checks + changed_when: false + failed_when: s2i_ci_buildah_image_checks.rc not in [0, 125] + + - name: Remove exact Buildah-created image tags + ansible.builtin.command: + argv: + - buildah + - rmi + - "{{ item.item }}" + loop: "{{ s2i_ci_buildah_image_checks.results }}" + when: item.rc == 0 + changed_when: true + + - name: List remaining Podman image references + ansible.builtin.command: + argv: + - podman + - images + - --format + - "{{ '{{.Repository}}:{{.Tag}}' }}" + register: s2i_ci_remaining_podman_images + changed_when: false + + - name: List remaining Buildah image references + ansible.builtin.command: + argv: + - buildah + - images + - --format + - "{{ '{{.Name}}:{{.Tag}}' }}" + register: s2i_ci_remaining_buildah_images + changed_when: false + + - name: Require exact workflow references to be absent + ansible.builtin.assert: + that: + - item not in s2i_ci_remaining_podman_images.stdout_lines + - item not in s2i_ci_remaining_buildah_images.stdout_lines + fail_msg: "Image cleanup left exact workflow reference {{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + + - name: Check registry validation metadata + ansible.builtin.stat: + path: "{{ s2i_ci_log_dir }}/registry-state.json" + register: s2i_ci_registry_state_metadata + + - name: Load registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.slurp: + src: "{{ s2i_ci_log_dir }}/registry-state.json" + register: s2i_ci_registry_state_data + + - name: Decode registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.set_fact: + s2i_ci_registry_state: >- + {{ s2i_ci_registry_state_data.content | b64decode | from_json }} + + - name: Validate registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.assert: + that: + - s2i_ci_registry_state is mapping + - s2i_ci_registry_state.endpoint is string + - s2i_ci_registry_state.endpoint is match('^[A-Za-z0-9.:-]+$') + - s2i_ci_registry_state.validation_image is string + - s2i_ci_registry_state.validation_image.startswith( + s2i_ci_registry_state.endpoint + '/') + + - name: Check the Podman registry validation tag + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ s2i_ci_registry_state.validation_image }}" + register: s2i_ci_validation_image_check + changed_when: false + failed_when: s2i_ci_validation_image_check.rc not in [0, 1] + + - name: Remove the exact Podman registry validation tag + when: + - s2i_ci_registry_state_metadata.stat.exists + - s2i_ci_validation_image_check.rc == 0 + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "{{ s2i_ci_registry_state.validation_image }}" + changed_when: true + + - name: Require registry validation tag to be absent + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ s2i_ci_registry_state.validation_image }}" + register: s2i_ci_validation_image_absent + changed_when: false + failed_when: s2i_ci_validation_image_absent.rc != 1 diff --git a/playbooks/container-ci/shared/deployment-map-item.yaml b/playbooks/container-ci/shared/deployment-map-item.yaml new file mode 100644 index 00000000..c38a4e07 --- /dev/null +++ b/playbooks/container-ci/shared/deployment-map-item.yaml @@ -0,0 +1,7 @@ +--- +- name: Expand image reference for each deployment key + ansible.builtin.set_fact: + s2i_ci_custom_container_images: >- + {{ s2i_ci_custom_container_images | + combine({item: s2i_ci_target_references[s2i_ci_image]}) }} + loop: "{{ s2i_ci_effective_image_mappings[s2i_ci_image] }}" diff --git a/playbooks/container-ci/shared/image-metadata-item.yaml b/playbooks/container-ci/shared/image-metadata-item.yaml new file mode 100644 index 00000000..c9119f47 --- /dev/null +++ b/playbooks/container-ci/shared/image-metadata-item.yaml @@ -0,0 +1,42 @@ +--- +- name: Check the selected image Containerfile + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/containers/{{ s2i_ci_image }}/Containerfile" + register: s2i_ci_image_containerfile + +- name: Require the selected image Containerfile + ansible.builtin.assert: + that: + - s2i_ci_image_containerfile.stat.isreg | default(false) + fail_msg: "Selected image {{ s2i_ci_image }} requires a Containerfile" + +- name: Select effective image deployment keys + ansible.builtin.set_fact: + s2i_ci_effective_image_keys: >- + {{ s2i_ci_image_mappings[s2i_ci_image] + if s2i_ci_image in s2i_ci_image_mappings + else s2i_ci_tracked_image_mappings.get(s2i_ci_image, []) }} + s2i_ci_effective_mapping_source: >- + {{ 'inventory' if s2i_ci_image in s2i_ci_image_mappings else 'tracked' }} + +- name: Validate effective image deployment keys + ansible.builtin.assert: + that: + - s2i_ci_effective_image_keys is sequence + - s2i_ci_effective_image_keys is not string + - s2i_ci_effective_image_keys | select('string') | list | length == + s2i_ci_effective_image_keys | length + - s2i_ci_effective_image_keys | + reject('match', '^[A-Za-z][A-Za-z0-9]*$') | list | length == 0 + - s2i_ci_effective_image_keys | unique | list | length == + s2i_ci_effective_image_keys | length + fail_msg: "Invalid effective image mapping for {{ s2i_ci_image }}" + +- name: Record effective image metadata + ansible.builtin.set_fact: + s2i_ci_effective_image_mappings: >- + {{ s2i_ci_effective_image_mappings | + combine({s2i_ci_image: s2i_ci_effective_image_keys | list}) }} + s2i_ci_mapping_sources: >- + {{ s2i_ci_mapping_sources | + combine({s2i_ci_image: s2i_ci_effective_mapping_source}) }} diff --git a/playbooks/container-ci/shared/load-image-mappings.yaml b/playbooks/container-ci/shared/load-image-mappings.yaml new file mode 100644 index 00000000..4b44834f --- /dev/null +++ b/playbooks/container-ci/shared/load-image-mappings.yaml @@ -0,0 +1,67 @@ +--- +- name: Check the tracked image mapping manifest + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/containers/image-mappings.yaml" + register: s2i_ci_image_mapping_manifest + +- name: Require the tracked image mapping manifest + ansible.builtin.assert: + that: + - s2i_ci_image_mapping_manifest.stat.isreg | default(false) + fail_msg: containers/image-mappings.yaml is required + +- name: Read the tracked image mapping manifest + ansible.builtin.slurp: + src: "{{ s2i_ci_container_repo }}/containers/image-mappings.yaml" + register: s2i_ci_image_mapping_data + +- name: Decode the tracked image mapping manifest + ansible.builtin.set_fact: + s2i_ci_image_mapping_document: >- + {{ s2i_ci_image_mapping_data.content | b64decode | from_yaml }} + +- name: Validate the tracked image mapping schema + ansible.builtin.assert: + that: + - s2i_ci_image_mapping_document is mapping + - s2i_ci_image_mapping_document.openstack_version is mapping + - s2i_ci_image_mapping_document.openstack_version.custom_container_images is mapping + fail_msg: Malformed containers/image-mappings.yaml + +- name: Record tracked image mappings + ansible.builtin.set_fact: + s2i_ci_tracked_image_mappings: >- + {{ s2i_ci_image_mapping_document.openstack_version.custom_container_images }} + s2i_ci_tracked_deployment_keys: [] + +- name: Resolve all available image targets + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - resolve + - all + args: + chdir: "{{ s2i_ci_container_repo }}" + register: s2i_ci_available_images + changed_when: false + +- name: Reject mappings for unknown image targets + ansible.builtin.assert: + that: + - s2i_ci_tracked_image_mappings.keys() | + difference(s2i_ci_available_images.stdout_lines) | length == 0 + fail_msg: containers/image-mappings.yaml names an unknown image target + +- name: Validate every tracked image mapping + ansible.builtin.include_tasks: validate-tracked-mapping-item.yaml + loop: "{{ s2i_ci_tracked_image_mappings.keys() | list }}" + loop_control: + loop_var: s2i_ci_tracked_mapping_target + label: "{{ s2i_ci_tracked_mapping_target }}" + +- name: Reject globally duplicated tracked deployment keys + ansible.builtin.assert: + that: + - s2i_ci_tracked_deployment_keys | unique | list | length == + s2i_ci_tracked_deployment_keys | length + fail_msg: A tracked deployment key may be assigned to only one image diff --git a/playbooks/container-ci/shared/prepare-host.yaml b/playbooks/container-ci/shared/prepare-host.yaml new file mode 100644 index 00000000..64104d47 --- /dev/null +++ b/playbooks/container-ci/shared/prepare-host.yaml @@ -0,0 +1,46 @@ +--- +- name: Prepare the shared container image build tools + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + + tasks: + - name: Ensure the container build log directory exists + ansible.builtin.file: + path: "{{ s2i_ci_log_dir }}" + state: directory + mode: "0755" + + - name: Update the apt package cache + when: + - s2i_ci_install_host_packages | default(true) | bool + - ansible_facts.os_family == "Debian" + become: true + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install container build host packages + when: s2i_ci_install_host_packages | default(true) | bool + become: true + ansible.builtin.package: + name: + - buildah + - ca-certificates + - git + - openssl + state: present + + - name: Ensure Podman is installed and usable + when: s2i_ci_install_host_packages | default(true) | bool + ansible.builtin.include_role: + name: ensure-podman + vars: + ensure_podman_rootless: false + ensure_podman_validate: true + + - name: Validate Buildah + when: s2i_ci_install_host_packages | default(true) | bool + ansible.builtin.command: buildah version + changed_when: false diff --git a/playbooks/container-ci/shared/published-image-item.yaml b/playbooks/container-ci/shared/published-image-item.yaml new file mode 100644 index 00000000..1b30ee8a --- /dev/null +++ b/playbooks/container-ci/shared/published-image-item.yaml @@ -0,0 +1,10 @@ +--- +- name: Record exact reference for selected image + ansible.builtin.set_fact: + s2i_ci_target_references: >- + {{ s2i_ci_target_references | + combine({s2i_ci_image: s2i_ci_published_images[s2i_ci_image_index]}) }} + s2i_ci_local_target_references: >- + {{ s2i_ci_local_target_references | + combine({s2i_ci_image: + s2i_ci_local_published_images[s2i_ci_image_index]}) }} diff --git a/playbooks/container-ci/shared/run.yaml b/playbooks/container-ci/shared/run.yaml new file mode 100644 index 00000000..9e8a1081 --- /dev/null +++ b/playbooks/container-ci/shared/run.yaml @@ -0,0 +1,361 @@ +--- +- name: Build and publish selected container images + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_workspace_root: "{{ zuul_user_dir | default(ansible_user_dir) }}" + + tasks: + - name: Default optional selective provider inputs + ansible.builtin.set_fact: + s2i_ci_namespace: "{{ s2i_ci_namespace | default('openstack') }}" + s2i_ci_tag: >- + {{ s2i_ci_tag | default( + (zuul.change | default('build') | string) ~ '-' ~ + (zuul.patchset | default('latest') | string) + ) }} + s2i_ci_stream: "{{ s2i_ci_stream | default('master') }}" + s2i_ci_parallel: >- + {{ s2i_ci_parallel | + default(ansible_facts.processor_vcpus | default(1)) }} + s2i_ci_images: "{{ s2i_ci_images | default('all') }}" + s2i_ci_image_mappings: "{{ s2i_ci_image_mappings | default({}) }}" + + - name: Validate the selected container project + ansible.builtin.assert: + that: + - s2i_ci_container_project in zuul.projects + fail_msg: The selected container project is not in the Zuul workspace + + - name: Resolve the inventory-described container repository + ansible.builtin.set_fact: + s2i_ci_container_repo: >- + {{ s2i_ci_workspace_root }}/{{ + zuul.projects[s2i_ci_container_project].src_dir }} + + - name: Inspect the container build entry point on the builder + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/build.sh" + follow: true + register: s2i_ci_build_entry_point + + - name: Canonicalize the builder workspace root + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_workspace_root }}"] + register: s2i_ci_workspace_root_realpath + changed_when: false + failed_when: false + + - name: Canonicalize the container repository on the builder + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_container_repo }}"] + register: s2i_ci_container_repo_realpath + changed_when: false + failed_when: false + + - name: Validate selective provider inputs + ansible.builtin.assert: + that: + - s2i_ci_build_entry_point.stat.isreg | default(false) + - s2i_ci_workspace_root_realpath.rc == 0 + - s2i_ci_container_repo_realpath.rc == 0 + - s2i_ci_container_repo_realpath.stdout.startswith( + s2i_ci_workspace_root_realpath.stdout + '/') + - >- + s2i_ci_images == 'all' or + (s2i_ci_images is sequence and s2i_ci_images is not string and + s2i_ci_images | length > 0 and + s2i_ci_images | select('string') | list | length == + s2i_ci_images | length and + s2i_ci_images | + reject('match', '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$') | + list | length == 0 and + s2i_ci_images | unique | list | length == s2i_ci_images | length) + - s2i_ci_image_mappings is mapping + - s2i_ci_tag is string + - "',' not in s2i_ci_tag" + fail_msg: Selective provider inputs are invalid + + - name: Record the requested target expression + ansible.builtin.set_fact: + s2i_ci_target_expression: >- + {{ 'all' if s2i_ci_images == 'all' + else ((['base'] + s2i_ci_images) | unique | list | join(',')) }} + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Resolve selected image targets + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - resolve + - "{{ s2i_ci_target_expression }}" + args: + chdir: "{{ s2i_ci_container_repo }}" + register: s2i_ci_resolved_images + changed_when: false + + - name: Record normalized selected targets + ansible.builtin.set_fact: + s2i_ci_selected_images: "{{ s2i_ci_resolved_images.stdout_lines }}" + + - name: Validate optional mapping overrides + ansible.builtin.include_tasks: validate-mapping-overrides.yaml + + - name: Load tracked image mappings + ansible.builtin.include_tasks: load-image-mappings.yaml + + - name: Load and validate selected image metadata + ansible.builtin.include_tasks: image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: validate-deployment-keys.yaml + + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + no_log: true + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + no_log: true + + - name: Validate normalized registry connection + ansible.builtin.assert: + that: + - s2i_ci_registry is mapping + - s2i_ci_registry.endpoint is string + - s2i_ci_registry.endpoint | length > 0 + - s2i_ci_registry.public_host is string + - s2i_ci_registry.public_host | length > 0 + - s2i_ci_registry.public_port | int > 0 + - s2i_ci_registry.auth_file | default('') is string + - s2i_ci_registry.cert_dir | default('') is string + + - name: Resolve intended exact image references before mutation + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_intended_refs + changed_when: false + + - name: Record intended exact image references + ansible.builtin.set_fact: + s2i_ci_intended_images: >- + {{ s2i_ci_intended_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Validate intended exact image references + ansible.builtin.assert: + that: + - s2i_ci_intended_images | length == s2i_ci_selected_images | length + - s2i_ci_intended_images | unique | list | length == + s2i_ci_intended_images | length + - s2i_ci_intended_images | + select('match', '^' + s2i_ci_registry.endpoint | regex_escape + + '/' + s2i_ci_namespace + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_intended_images | length + fail_msg: Intended image references are incomplete or unsafe + + - name: Persist intended references for partial-failure cleanup + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/intended-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_registry.endpoint, + 'namespace': s2i_ci_namespace, + 'images': s2i_ci_intended_images, + 'selected_images': s2i_ci_selected_images, + 'target': s2i_ci_target_expression + } | to_nice_json }} + + - name: Build selected images from committed source pins + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - build-parallel + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + PARALLEL: "{{ s2i_ci_parallel | string }}" + BUILD_LOGS_DIR: "{{ s2i_ci_log_dir }}" + register: s2i_ci_build + changed_when: s2i_ci_build.rc == 0 + + - name: Save build command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/build.log" + mode: "0644" + content: | + {{ s2i_ci_build.stdout }} + {{ s2i_ci_build.stderr }} + + - name: Push selected images to the buildset registry + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - push + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + REGISTRY_AUTH_FILE: "{{ s2i_ci_registry.auth_file | default('') }}" + REGISTRY_CERT_DIR: "{{ s2i_ci_registry.cert_dir | default('') }}" + register: s2i_ci_push + changed_when: s2i_ci_push.rc == 0 + + - name: Save push command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/push.log" + mode: "0644" + content: | + {{ s2i_ci_push.stdout }} + {{ s2i_ci_push.stderr }} + + - name: Resolve successfully published image references + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_refs + changed_when: false + + - name: Record exact builder-local published image references + ansible.builtin.set_fact: + s2i_ci_local_published_images: >- + {{ s2i_ci_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Require intended and published references to match + ansible.builtin.assert: + that: + - s2i_ci_local_published_images == s2i_ci_intended_images + fail_msg: Published references differ from the pre-recorded intent + + - name: Record the public buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_public_registry_endpoint: >- + {{ s2i_ci_registry.public_host }}:{{ s2i_ci_registry.public_port }} + + - name: Convert published references to the public buildset endpoint + ansible.builtin.set_fact: + s2i_ci_published_images: >- + {{ s2i_ci_local_published_images | + map('regex_replace', + '^' + (s2i_ci_registry.endpoint | regex_escape), + s2i_ci_public_registry_endpoint) | list }} + + - name: Validate public exact published references + ansible.builtin.assert: + that: + - s2i_ci_published_images | length == + s2i_ci_local_published_images | length + - s2i_ci_published_images | + select('match', '^' + + (s2i_ci_public_registry_endpoint | regex_escape) + '/') | + list | length == s2i_ci_published_images | length + - s2i_ci_published_images | + select('search', s2i_ci_registry.endpoint | regex_escape) | + list | length == 0 + fail_msg: Public references must not expose the builder-local alias + + - name: Pull published images back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry.auth_file] + if s2i_ci_registry.auth_file | default('') | length > 0 else []) + + (['--cert-dir', s2i_ci_registry.cert_dir] + if s2i_ci_registry.cert_dir | default('') | length > 0 else []) + + [item] }} + loop: "{{ s2i_ci_local_published_images }}" + changed_when: true + + - name: Initialize exact reference expansion + ansible.builtin.set_fact: + s2i_ci_target_references: {} + s2i_ci_local_target_references: {} + s2i_ci_custom_container_images: {} + + - name: Map selected targets to exact references + ansible.builtin.include_tasks: published-image-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + index_var: s2i_ci_image_index + label: "{{ s2i_ci_image }}" + + - name: Expand exact references into deployment keys + ansible.builtin.include_tasks: deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate consolidated Watcher process entry points + ansible.builtin.command: + argv: + - podman + - run + - --rm + - --user + - watcher + - --entrypoint + - /bin/sh + - "{{ s2i_ci_local_target_references['watcher/watcher-base'] }}" + - -ec + - >- + for command in watcher-api watcher-applier watcher-decision-engine; + do command -v "$command"; "$command" --help >/dev/null; done + when: "'watcher/watcher-base' in s2i_ci_selected_images" + changed_when: false + + - name: Save published image metadata + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/published-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_public_registry_endpoint, + 'namespace': s2i_ci_namespace, + 'tag': s2i_ci_tag, + 'target': s2i_ci_target_expression, + 'stream': s2i_ci_stream, + 'selected_images': s2i_ci_selected_images, + 'images': s2i_ci_published_images, + 'image_mappings': s2i_ci_effective_image_mappings, + 'mapping_sources': s2i_ci_mapping_sources, + 'custom_container_images': + s2i_ci_custom_container_images + } | to_nice_json }} diff --git a/playbooks/container-ci/shared/validate-deployment-keys.yaml b/playbooks/container-ci/shared/validate-deployment-keys.yaml new file mode 100644 index 00000000..f9750707 --- /dev/null +++ b/playbooks/container-ci/shared/validate-deployment-keys.yaml @@ -0,0 +1,12 @@ +--- +- name: Record all effective deployment keys + ansible.builtin.set_fact: + s2i_ci_all_deployment_keys: >- + {{ s2i_ci_effective_image_mappings.values() | flatten | list }} + +- name: Reject duplicate deployment keys + ansible.builtin.assert: + that: + - s2i_ci_all_deployment_keys | unique | list | length == + s2i_ci_all_deployment_keys | length + fail_msg: A deployment key may be assigned to only one image diff --git a/playbooks/container-ci/shared/validate-mapping-overrides.yaml b/playbooks/container-ci/shared/validate-mapping-overrides.yaml new file mode 100644 index 00000000..081c990a --- /dev/null +++ b/playbooks/container-ci/shared/validate-mapping-overrides.yaml @@ -0,0 +1,8 @@ +--- +- name: Reject mapping overrides for unselected images + ansible.builtin.assert: + that: + - s2i_ci_image_mappings is mapping + - s2i_ci_image_mappings.keys() | + difference(s2i_ci_selected_images) | length == 0 + fail_msg: Image mapping overrides must name selected image targets diff --git a/playbooks/container-ci/shared/validate-registry.yaml b/playbooks/container-ci/shared/validate-registry.yaml new file mode 100644 index 00000000..cad25f04 --- /dev/null +++ b/playbooks/container-ci/shared/validate-registry.yaml @@ -0,0 +1,109 @@ +--- +- name: Validate the prepared image registry connection + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_validation_source_image: registry.access.redhat.com/ubi10/ubi-minimal:latest + s2i_ci_validation_repository: s2i-validation/ubi10-minimal:latest + + tasks: + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry_connection: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + + - name: Require normalized registry connection data + ansible.builtin.assert: + that: + - s2i_ci_registry_connection is mapping + - s2i_ci_registry_connection.endpoint is string + - s2i_ci_registry_connection.endpoint | length > 0 + - s2i_ci_registry_connection.public_host is string + - s2i_ci_registry_connection.public_host | length > 0 + - s2i_ci_registry_connection.public_port | int > 0 + - s2i_ci_registry_connection.auth_file | default('') is string + - s2i_ci_registry_connection.cert_dir | default('') is string + + - name: Set normalized registry command parameters + ansible.builtin.set_fact: + s2i_ci_registry_endpoint: "{{ s2i_ci_registry_connection.endpoint }}" + s2i_ci_registry_auth_file: >- + {{ s2i_ci_registry_connection.auth_file | default('') }} + s2i_ci_registry_cert_dir: >- + {{ s2i_ci_registry_connection.cert_dir | default('') }} + + - name: Pull the UBI 10 base image + ansible.builtin.command: + argv: + - podman + - pull + - "{{ s2i_ci_validation_source_image }}" + register: s2i_ci_ubi_pull + changed_when: "'Copying blob' in s2i_ci_ubi_pull.stdout" + + - name: Tag the UBI 10 image for registry validation + ansible.builtin.command: + argv: + - podman + - tag + - "{{ s2i_ci_validation_source_image }}" + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: true + + - name: Push the UBI 10 validation image + ansible.builtin.command: + argv: >- + {{ ['podman', 'push', '--remove-signatures'] + + (['--authfile', s2i_ci_registry_auth_file] + if s2i_ci_registry_auth_file | length > 0 else []) + + (['--cert-dir', s2i_ci_registry_cert_dir] + if s2i_ci_registry_cert_dir | length > 0 else []) + + [s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository] }} + changed_when: true + + - name: Remove the validation tag before pulling it back + ansible.builtin.command: + argv: + - podman + - untag + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: true + + - name: Pull the UBI 10 validation image back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry_auth_file] + if s2i_ci_registry_auth_file | length > 0 else []) + + (['--cert-dir', s2i_ci_registry_cert_dir] + if s2i_ci_registry_cert_dir | length > 0 else []) + + [s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository] }} + changed_when: false + + - name: Inspect the UBI 10 validation image from the registry + ansible.builtin.command: + argv: + - podman + - image + - inspect + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: false + + - name: Record non-secret registry state + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/registry-state.json" + mode: "0644" + content: | + {{ { + 'endpoint': s2i_ci_registry_endpoint, + 'validation_source': s2i_ci_validation_source_image, + 'validation_image': s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository + } | to_nice_json }} diff --git a/playbooks/container-ci/shared/validate-tracked-mapping-item.yaml b/playbooks/container-ci/shared/validate-tracked-mapping-item.yaml new file mode 100644 index 00000000..95f885bc --- /dev/null +++ b/playbooks/container-ci/shared/validate-tracked-mapping-item.yaml @@ -0,0 +1,24 @@ +--- +- name: Select tracked deployment keys for validation + ansible.builtin.set_fact: + s2i_ci_tracked_mapping_keys: >- + {{ s2i_ci_tracked_image_mappings[s2i_ci_tracked_mapping_target] }} + +- name: Validate tracked deployment keys + ansible.builtin.assert: + that: + - s2i_ci_tracked_mapping_keys is sequence + - s2i_ci_tracked_mapping_keys is not string + - s2i_ci_tracked_mapping_keys | select('string') | list | length == + s2i_ci_tracked_mapping_keys | length + - s2i_ci_tracked_mapping_keys | + reject('match', '^[A-Za-z][A-Za-z0-9]*$') | list | length == 0 + - s2i_ci_tracked_mapping_keys | unique | list | length == + s2i_ci_tracked_mapping_keys | length + fail_msg: >- + Invalid tracked image mapping for {{ s2i_ci_tracked_mapping_target }} + +- name: Accumulate tracked deployment keys + ansible.builtin.set_fact: + s2i_ci_tracked_deployment_keys: >- + {{ s2i_ci_tracked_deployment_keys + s2i_ci_tracked_mapping_keys }} diff --git a/playbooks/container-ci/zuul/content-provider-return.yaml b/playbooks/container-ci/zuul/content-provider-return.yaml new file mode 100644 index 00000000..f4e1abe3 --- /dev/null +++ b/playbooks/container-ci/zuul/content-provider-return.yaml @@ -0,0 +1,32 @@ +--- +- name: Return selective image metadata and pause the provider + zuul_return: + data: + zuul: + pause: true + s2i_ci_content: + registry: "{{ s2i_ci_public_registry_endpoint }}" + registry_host: "{{ s2i_ci_registry.public_host }}" + registry_port: "{{ s2i_ci_registry.public_port }}" + namespace: "{{ s2i_ci_namespace }}" + tag: "{{ s2i_ci_tag }}" + target: "{{ s2i_ci_target_expression }}" + stream: "{{ s2i_ci_stream }}" + selected_images: "{{ s2i_ci_selected_images }}" + images: "{{ s2i_ci_published_images }}" + image_mappings: "{{ s2i_ci_effective_image_mappings }}" + mapping_sources: "{{ s2i_ci_mapping_sources }}" + custom_container_images: "{{ s2i_ci_custom_container_images }}" + content_provider_os_custom_container_images: >- + {{ s2i_ci_custom_container_images }} + content_provider_os_registry_url: "null" + content_provider_os_registry_namespace: "" + content_provider_os_registry_tag: "" + content_provider_dlrn_md5_hash: "" + content_provider_gating_repo_available: false + content_provider_gating_repo_url: "" + content_provider_registry_available: true + content_provider_registry_ip: "{{ s2i_ci_registry.public_host }}" + content_provider_registry_ip_port: >- + {{ s2i_ci_public_registry_endpoint }} + cifmw_build_images_output: {} diff --git a/playbooks/container-ci/zuul/post.yaml b/playbooks/container-ci/zuul/post.yaml new file mode 100644 index 00000000..6b63478d --- /dev/null +++ b/playbooks/container-ci/zuul/post.yaml @@ -0,0 +1,129 @@ +--- +- name: Collect Zuul container logs + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + + tasks: + - name: Ensure the container build log directory exists + ansible.builtin.file: + path: "{{ s2i_ci_log_dir }}" + state: directory + mode: "0755" + failed_when: false + + - name: Collect buildset registry logs + block: + - name: Run the trusted container log collection role + ansible.builtin.include_role: + name: collect-container-logs + vars: + container_command: podman + rescue: + - name: Report non-fatal Zuul container log collection failure + ansible.builtin.debug: + msg: Container log collection failed; registry cleanup will continue + +- name: Remove exact workflow images + ansible.builtin.import_playbook: ../shared/cleanup-images.yaml + +- name: Remove the project-owned Zuul buildset registry + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_registry_port: 5000 + + tasks: + - name: Check buildset registry ownership + ansible.builtin.stat: + path: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + register: s2i_ci_registry_owner_marker + + - name: Default buildset registry ownership to unloaded + ansible.builtin.set_fact: + s2i_ci_registry_owner_loaded: false + s2i_ci_registry_owner_is_valid: false + + - name: Load buildset registry ownership + when: s2i_ci_registry_owner_marker.stat.exists | default(false) + block: + - name: Read the buildset registry ownership marker + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + register: s2i_ci_registry_owner_data + + - name: Decode the buildset registry ownership marker + ansible.builtin.set_fact: + s2i_ci_registry_owner: >- + {{ s2i_ci_registry_owner_data.content | b64decode | from_json }} + + - name: Record successful ownership loading + ansible.builtin.set_fact: + s2i_ci_registry_owner_loaded: true + rescue: + - name: Refuse unreadable registry ownership metadata + ansible.builtin.fail: + msg: Refusing to clean a registry whose ownership marker is invalid + + - name: Validate buildset registry ownership + when: s2i_ci_registry_owner_loaded | bool + ansible.builtin.assert: + that: + - s2i_ci_registry_owner.port | default(-1) | int == + s2i_ci_registry_port | int + - s2i_ci_registry_owner.container | default('') == + ('buildset_registry' if s2i_ci_registry_port | int == 5000 + else 'buildset_registry_' + s2i_ci_registry_port | string) + fail_msg: Refusing to clean a registry with an invalid ownership marker + + - name: Record valid buildset registry ownership + when: s2i_ci_registry_owner_loaded | bool + ansible.builtin.set_fact: + s2i_ci_registry_owner_is_valid: true + + - name: Remove the project-owned buildset registry container + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ s2i_ci_registry_owner.container }}" + register: s2i_ci_removed_buildset_registry + changed_when: s2i_ci_removed_buildset_registry.stdout | length > 0 + + - name: Stop the project-owned buildset registry tunnel + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.shell: | + pids=$(pgrep -f '^socat -d -d TCP6-LISTEN:{{ s2i_ci_registry_owner.port }},fork TCP:127.0.0.1:1{{ s2i_ci_registry_owner.port }}$' || true) + if [ -n "$pids" ]; then + printf '%s\n' "$pids" + kill $pids + fi + args: + executable: /bin/bash + register: s2i_ci_stopped_registry_tunnel + changed_when: s2i_ci_stopped_registry_tunnel.stdout | length > 0 + + - name: Remove project-owned buildset registry state + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.file: + path: "{{ s2i_ci_registry_root }}" + state: absent + + - name: Remove persisted buildset registry connection data + ansible.builtin.file: + path: "{{ s2i_ci_registry_connection_file }}" + state: absent diff --git a/playbooks/container-ci/zuul/pre.yaml b/playbooks/container-ci/zuul/pre.yaml new file mode 100644 index 00000000..49f52bae --- /dev/null +++ b/playbooks/container-ci/zuul/pre.yaml @@ -0,0 +1,127 @@ +--- +- name: Prepare shared build tools + ansible.builtin.import_playbook: ../shared/prepare-host.yaml + +- name: Prepare the Zuul buildset registry + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_registry_port: 5000 + s2i_ci_registry_image: quay.io/zuul-ci/zuul-registry@sha256:0bc02a2eed546daa570ebb18829579057cefceefd9ee15660dcf5a1a752d8f78 + + tasks: + - name: Record whether this job owns the buildset registry + ansible.builtin.set_fact: + s2i_ci_owns_buildset_registry: "{{ buildset_registry is not defined }}" + + - name: Discard a stale ownership marker for an inherited registry + when: not s2i_ci_owns_buildset_registry | bool + ansible.builtin.file: + path: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + state: absent + + - name: Remove a stale project-owned buildset registry container + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.command: + argv: [podman, rm, --force, buildset_registry] + register: s2i_ci_stale_buildset_registry + changed_when: s2i_ci_stale_buildset_registry.stdout | length > 0 + failed_when: s2i_ci_stale_buildset_registry.rc not in [0, 1] + + - name: Start the Zuul buildset registry + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.include_role: + name: run-buildset-registry + vars: + buildset_registry_root: "{{ s2i_ci_registry_root }}" + buildset_registry_port: "{{ s2i_ci_registry_port }}" + buildset_registry_image: "{{ s2i_ci_registry_image }}" + container_command: podman + + - name: Record ownership of the started buildset registry + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.copy: + dest: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + mode: "0600" + content: | + {{ { + 'container': ('buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string), + 'port': s2i_ci_registry_port + } | to_nice_json }} + + - name: Remove the unlabeled buildset registry container + when: + - s2i_ci_owns_buildset_registry | bool + - ansible_selinux.status | default("disabled") == "enabled" + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + changed_when: true + + - name: Start the buildset registry with SELinux-labeled volumes + when: + - s2i_ci_owns_buildset_registry | bool + - ansible_selinux.status | default("disabled") == "enabled" + ansible.builtin.command: + argv: + - podman + - run + - --detach + - "--name={{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + - --restart=always + - "--publish=1{{ s2i_ci_registry_port }}:5000" + - "--volume={{ s2i_ci_registry_root }}/tls:/tls:Z" + - "--volume={{ s2i_ci_registry_root }}/conf:/conf:Z" + - "{{ s2i_ci_registry_image }}" + - zuul-registry + - -d + changed_when: true + + - name: Wait for the buildset registry authentication endpoint + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.uri: + url: "https://127.0.0.1:{{ s2i_ci_registry_port }}/v2/" + validate_certs: false + status_code: 401 + register: s2i_ci_buildset_registry_api + retries: 30 + delay: 1 + until: s2i_ci_buildset_registry_api.status == 401 + + - name: Configure the host to use the Zuul buildset registry + ansible.builtin.include_role: + name: use-buildset-registry + vars: + buildset_registry_namespaces: + - [docker.io, https://registry-1.docker.io] + - [quay.io, https://quay.io] + - [gcr.io, https://gcr.io] + - [registry.k8s.io, https://registry.k8s.io] + - [registry.access.redhat.com, https://registry.access.redhat.com] + + - name: Set the Zuul buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_registry_endpoint: >- + {{ buildset_registry_alias }}:{{ buildset_registry.port }} + + - name: Write the normalized registry connection + ansible.builtin.copy: + dest: "{{ s2i_ci_registry_connection_file }}" + mode: "0600" + content: | + {{ { + 'endpoint': s2i_ci_registry_endpoint, + 'public_host': buildset_registry.host, + 'public_port': buildset_registry.port, + 'auth_file': '', + 'cert_dir': '' + } | to_nice_json }} + +- name: Validate the normalized buildset registry + ansible.builtin.import_playbook: ../shared/validate-registry.yaml diff --git a/playbooks/container-ci/zuul/run.yaml b/playbooks/container-ci/zuul/run.yaml new file mode 100644 index 00000000..e8afdfdf --- /dev/null +++ b/playbooks/container-ci/zuul/run.yaml @@ -0,0 +1,12 @@ +--- +- name: Run shared selective image publication + ansible.builtin.import_playbook: ../shared/run.yaml + +- name: Return published image metadata to Zuul + hosts: builder + gather_facts: false + + tasks: + - name: Return content-provider metadata to Zuul + when: s2i_ci_content_provider | default(false) | bool + ansible.builtin.include_tasks: content-provider-return.yaml diff --git a/tests/test_provider_architecture.py b/tests/test_provider_architecture.py new file mode 100644 index 00000000..011d5856 --- /dev/null +++ b/tests/test_provider_architecture.py @@ -0,0 +1,254 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pathlib +import re +import unittest + + +class ProviderArchitectureTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + self.container_ci = self.repo_root / "playbooks" / "container-ci" + + def _read(self, path): + return (self.repo_root / path).read_text(encoding="utf-8") + + def _image_mappings(self): + mappings = {} + current_target = None + for line in self._read("containers/image-mappings.yaml").splitlines(): + target = re.match(r"^ (\S+):$", line) + if target: + current_target = target.group(1) + mappings[current_target] = [] + continue + key = re.match(r"^ - (\S+)$", line) + if key and current_target: + mappings[current_target].append(key.group(1)) + return mappings + + def test_deployment_mappings_are_centralized(self): + self.assertTrue( + (self.repo_root / "containers/image-mappings.yaml").is_file() + ) + self.assertFalse( + list((self.repo_root / "containers").glob("**/image.yaml")) + ) + + def test_tracked_deployment_mappings_are_intentional(self): + self.assertEqual( + { + "glance/glance-api": ["glanceAPIImage"], + "manila/manila-api": ["manilaAPIImage"], + "manila/manila-scheduler": ["manilaSchedulerImage"], + "manila/manila-share": ["manilaShareImage"], + "watcher/watcher-base": [ + "watcherAPIImage", + "watcherApplierImage", + "watcherDecisionEngineImage", + ], + }, + self._image_mappings(), + ) + + def test_container_ci_mutations_target_only_builder(self): + playbooks = sorted(self.container_ci.glob("**/*.yaml")) + + self.assertTrue(playbooks) + for playbook in playbooks: + content = playbook.read_text(encoding="utf-8") + self.assertNotIn("hosts: all", content, playbook) + self.assertNotIn("hosts: localhost", content, playbook) + self.assertNotIn("delegate_to: localhost", content, playbook) + self.assertNotIn("local_action:", content, playbook) + for host_pattern in re.findall(r"^ hosts: (.+)$", content, re.M): + self.assertEqual("builder", host_pattern, playbook) + + def test_exact_cleanup_does_not_hide_failures(self): + cleanup = self._read( + "playbooks/container-ci/shared/cleanup-images.yaml" + ) + + self.assertNotIn("failed_when: false", cleanup) + self.assertIn("podman", cleanup) + self.assertIn("buildah", cleanup) + self.assertIn( + "Require exact workflow references to be absent", cleanup + ) + + def test_shared_and_zuul_ownership_is_separated(self): + shared = { + path.name for path in (self.container_ci / "shared").glob("*") + } + zuul = {path.name for path in (self.container_ci / "zuul").glob("*")} + + self.assertTrue( + { + "prepare-host.yaml", + "validate-registry.yaml", + "run.yaml", + "cleanup-images.yaml", + }.issubset(shared) + ) + self.assertTrue( + { + "pre.yaml", + "run.yaml", + "post.yaml", + "content-provider-return.yaml", + }.issubset(zuul) + ) + self.assertNotIn("reset-static-node.yaml", zuul) + self.assertNotIn("content-provider-return.yaml", shared) + + def test_provider_job_keeps_explicit_builder_contract(self): + zuul = self._read("zuul.d/jobs.yaml") + + self.assertIn("name: builder", zuul) + self.assertEqual( + 2, + zuul.count("nodeset: s2i-openstack-containers-image-builder"), + ) + self.assertIn("s2i_ci_images: all", zuul) + self.assertNotIn("- project:", zuul) + self.assertNotIn("abstract: true", zuul) + + def test_upstream_github_check_runs_configured_jobs(self): + layout = self._read("zuul.d/projects.yaml") + jobs = re.findall(r"^ - (\S+):$", layout, re.M) + + self.assertEqual( + [ + "s2i-openstack-containers-molecule", + "s2i-openstack-container-content-provider", + ], + jobs, + ) + self.assertIn("irrelevant-files:", layout) + self.assertNotIn("noop", layout) + + def test_provider_validates_repository_on_builder(self): + run = self._read("playbooks/container-ci/shared/run.yaml") + + self.assertIn("ansible.builtin.stat:", run) + self.assertIn("s2i_ci_build_entry_point.stat.isreg", run) + self.assertIn("argv: [realpath, --canonicalize-existing", run) + self.assertNotIn(" is file", run) + self.assertNotIn(" | realpath", run) + + def test_central_mapping_validates_all_tracked_entries(self): + loader = self._read( + "playbooks/container-ci/shared/load-image-mappings.yaml" + ) + item = self._read( + "playbooks/container-ci/shared/validate-tracked-mapping-item.yaml" + ) + + self.assertIn("resolve", loader) + self.assertIn( + "difference(s2i_ci_available_images.stdout_lines)", loader + ) + self.assertIn("Validate every tracked image mapping", loader) + self.assertIn("globally duplicated tracked deployment keys", loader) + self.assertIn("s2i_ci_tracked_mapping_keys is sequence", item) + self.assertIn("reject('match', '^[A-Za-z][A-Za-z0-9]*$')", item) + + def test_inventory_provider_inputs_are_not_masked_by_play_vars(self): + shared_run = self._read("playbooks/container-ci/shared/run.yaml") + zuul_run = self._read("playbooks/container-ci/zuul/run.yaml") + prepare_host = self._read( + "playbooks/container-ci/shared/prepare-host.yaml" + ) + + shared_play_vars = shared_run.split(" tasks:", 1)[0] + for variable in ( + "s2i_ci_namespace:", + "s2i_ci_tag:", + "s2i_ci_stream:", + "s2i_ci_parallel:", + "s2i_ci_images:", + "s2i_ci_image_mappings:", + ): + self.assertNotIn(variable, shared_play_vars) + self.assertNotIn("s2i_ci_content_provider: false", zuul_run) + self.assertNotIn("s2i_ci_install_host_packages: true", prepare_host) + self.assertIn("s2i_ci_images | default('all')", shared_run) + self.assertIn("s2i_ci_target_expression", shared_run) + self.assertIn("build.sh", shared_run) + self.assertIn("resolve", shared_run) + self.assertIn( + "s2i_ci_content_provider | default(false) | bool", zuul_run + ) + + def test_return_contract_is_selective_and_secret_free(self): + returned = self._read( + "playbooks/container-ci/zuul/content-provider-return.yaml" + ) + + for field in ( + "s2i_ci_content:", + "content_provider_os_custom_container_images:", + 'content_provider_os_registry_url: "null"', + "content_provider_dlrn_md5_hash:", + "content_provider_gating_repo_available: false", + "content_provider_gating_repo_url:", + "content_provider_registry_ip:", + "content_provider_registry_ip_port:", + "cifmw_build_images_output: {}", + "pause: true", + ): + self.assertIn(field, returned) + for secret in ("password", "username", "auth_file", "cert_dir"): + self.assertNotIn(secret, returned) + self.assertIn("s2i_ci_public_registry_endpoint", returned) + self.assertNotIn("s2i_ci_registry.endpoint", returned) + + def test_watcher_image_is_process_neutral(self): + containerfile = self._read( + "containers/watcher/watcher-base/Containerfile" + ) + bindeps = set( + line + for line in self._read( + "containers/watcher/watcher-base/bindeps.txt" + ).splitlines() + if line and not line.startswith("#") + ) + + self.assertIn( + "Consolidated Watcher API, applier, and decision-engine", + containerfile, + ) + self.assertTrue( + { + "httpd", + "python3-mod_wsgi", + "libffi", + "libxml2", + "libxslt", + }.issubset(bindeps) + ) + + def test_c2_has_no_oib_or_local_adapter(self): + self.assertFalse((self.repo_root / "openstack_image_builder").exists()) + self.assertFalse((self.container_ci / "local").exists()) + all_content = "\n".join( + path.read_text(encoding="utf-8") + for path in self.container_ci.glob("**/*.yaml") + ) + for forbidden in ("S2I_CONTEXTS_ROOT", "ERROR_ON_CLONE", "oib"): + self.assertNotIn(forbidden, all_content) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_provider_shell.py b/tests/test_provider_shell.py new file mode 100644 index 00000000..3fb076ff --- /dev/null +++ b/tests/test_provider_shell.py @@ -0,0 +1,200 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pathlib +import subprocess +import tempfile +import time +import unittest + + +class ProviderShellTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + temporary_root = self.repo_root / ".tmp" + temporary_root.mkdir(exist_ok=True) + self.temporary_directory = tempfile.TemporaryDirectory( + dir=temporary_root, prefix="provider-shell." + ) + self.addCleanup(self.temporary_directory.cleanup) + self.root = pathlib.Path(self.temporary_directory.name) + (self.root / "build.sh").symlink_to(self.repo_root / "build.sh") + self.bin_dir = self.root / "bin" + self.bin_dir.mkdir() + self.logs_dir = self.root / "logs" + self._create_images() + self._create_fake_buildah() + + def _create_images(self): + for target in ("base", "alpha/one", "beta/two"): + image_root = self.root / "containers" / target + image_root.mkdir(parents=True) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + project = target.split("/", maxsplit=1)[0] + project_root = self.root / "containers" / project + (project_root / "requirements.lock.master").touch() + if "/" in target: + (project_root / "src" / project).mkdir( + parents=True, exist_ok=True + ) + + def _create_fake_buildah(self): + fake = self.bin_dir / "buildah" + fake.write_text( + """#!/usr/bin/env python3 +import os +import pathlib +import sys +import time + +args = sys.argv[1:] +if args[0] == "bud": + containerfile = pathlib.Path(args[args.index("-f") + 1]) + image = containerfile.parent.name + if containerfile.parent.parent.name != "containers": + image = f"{containerfile.parent.parent.name}/{image}" + if image != "base": + print(f"LIVE {image}", flush=True) + time.sleep(0.75) + if os.environ.get("FAIL_IMAGE") and image.endswith(os.environ["FAIL_IMAGE"]): + print(f"FAIL {image}", flush=True) + sys.exit(9) + print(f"DONE {image}", flush=True) + sys.exit(0) +if args[0] == "inspect": + sys.exit(0) +if args[0] == "push": + sys.exit(0) +sys.exit(2) +""", + encoding="utf-8", + ) + fake.chmod(0o755) + + def _environment(self): + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "STREAM": "master", + "REGISTRY": "registry.test:5000", + "NAMESPACE": "openstack", + "TAG": "test", + "PARALLEL": "2", + "BUILD_LOGS_DIR": str(self.logs_dir), + } + ) + return environment + + def _run(self, *arguments, environment=None): + return subprocess.run( + [str(self.root / "build.sh"), *arguments], + cwd=self.root, + env=environment or self._environment(), + check=False, + capture_output=True, + text=True, + ) + + def test_explicit_union_is_ordered_and_includes_base(self): + result = self._run("refs", "beta/two,alpha/one,beta/two") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + [ + "registry.test:5000/openstack/openstack-base:test", + "registry.test:5000/openstack/openstack-two:test", + "registry.test:5000/openstack/openstack-one:test", + ], + result.stdout.splitlines(), + ) + + def test_single_target_semantics_are_unchanged(self): + result = self._run("refs", "alpha/one") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["registry.test:5000/openstack/openstack-one:test"], + result.stdout.splitlines(), + ) + + def test_resolve_all_returns_machine_readable_targets(self): + result = self._run("resolve", "all") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["base", "alpha/one", "beta/two"], result.stdout.splitlines() + ) + + def test_explicit_union_rejects_unknown_target(self): + result = self._run("refs", "alpha/one,unknown/image") + + self.assertNotEqual(0, result.returncode) + self.assertIn("Unknown image or project", result.stderr) + + def test_parallel_output_is_live_and_logs_are_retained(self): + process = subprocess.Popen( + [ + str(self.root / "build.sh"), + "build-parallel", + "alpha/one,beta/two", + ], + cwd=self.root, + env=self._environment(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.addCleanup( + lambda: process.kill() if process.poll() is None else None + ) + self.addCleanup(process.stdout.close) + output = [] + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + line = process.stdout.readline() + output.append(line) + if "LIVE" in line: + break + + self.assertIn("LIVE", "".join(output)) + self.assertIsNone( + process.poll(), "build exited before live output arrived" + ) + output.append(process.stdout.read()) + process.wait(timeout=10) + self.assertEqual(0, process.returncode, "".join(output)) + self.assertIn("[alpha/one] LIVE alpha/one", "".join(output)) + self.assertIn("[beta/two] LIVE beta/two", "".join(output)) + self.assertTrue((self.logs_dir / "base.log").is_file()) + self.assertTrue((self.logs_dir / "alpha_one.log").is_file()) + self.assertTrue((self.logs_dir / "beta_two.log").is_file()) + self.assertEqual(1, "".join(output).count("LIVE alpha/one")) + + def test_parallel_failure_propagates(self): + environment = self._environment() + environment["FAIL_IMAGE"] = "two" + + result = self._run( + "build-parallel", "alpha/one,beta/two", environment=environment + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("FAIL beta/two", result.stdout + result.stderr) + self.assertIn("stopping remaining builds", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_update_sources.py b/tests/test_update_sources.py new file mode 100644 index 00000000..2d57b45e --- /dev/null +++ b/tests/test_update_sources.py @@ -0,0 +1,533 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for build.sh update-sources behavior.""" + +import csv +import os +import pathlib +import subprocess +import tempfile +import unittest + + +class UpdateSourcesTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + tmp_root = self.repo_root / ".tmp" + tmp_root.mkdir(exist_ok=True) + self.temporary_directory = tempfile.TemporaryDirectory( + dir=tmp_root, prefix="update-sources-test." + ) + self.addCleanup(self.temporary_directory.cleanup) + self.test_root = pathlib.Path(self.temporary_directory.name) + upstream_root = self.test_root / "upstream" + upstream_root.mkdir() + + self.upstream_requirements = upstream_root / "requirements.git" + self.requirements_old, self.requirements_new = self.create_remote( + self.upstream_requirements, + [ + {"upper-constraints.txt": "six==1.17.0\n"}, + {"upper-constraints.txt": ("six==1.17.0\npbr==7.0.3\n")}, + ], + ) + self.upstream_service = upstream_root / "test-svc.git" + self.service_old, self.service_new = self.create_remote( + self.upstream_service, + [ + {"requirements.txt": "six\npbr\n"}, + {"requirements.txt": "six\npbr\n# v2\n"}, + ], + ) + (self.test_root / "build.sh").symlink_to(self.repo_root / "build.sh") + self.project_root = self.test_root / "containers" / "test-svc" + image_root = self.project_root / "test-svc" + (self.project_root / "src").mkdir(parents=True) + (image_root / "src").mkdir(parents=True) + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} master {self.service_old}\n" + ) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + (image_root / "bindeps.txt").write_text("python3\n", encoding="utf-8") + (image_root / "builddeps.txt").write_text("gcc\n", encoding="utf-8") + (image_root / "pythondeps.txt").touch() + (image_root / "pythonbuilddeps.txt").touch() + + self.additional_services = {} + for name in ("test-svc2", "test-svc3"): + remote = upstream_root / f"{name}.git" + old, new = self.create_remote( + remote, + [ + {"requirements.txt": "six\npbr\n"}, + {"requirements.txt": "six\npbr\n# v2\n"}, + ], + ) + self.additional_services[name] = (remote, old, new) + self.create_project_fixture(name, remote, old) + + self.manifest_path = ( + self.test_root + / ".tmp/source-maintenance/frozen-source-refs.master.tsv" + ) + + def run_command(self, command, cwd=None): + result = subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"command failed with {result.returncode}: {command}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result.stdout.strip() + + def create_remote(self, destination, commits): + work = self.test_root / f"work-{destination.stem}" + self.run_command(["git", "init", "-b", "master", str(work)]) + self.run_command( + ["git", "config", "user.email", "test@example.com"], work + ) + self.run_command(["git", "config", "user.name", "Test"], work) + self.run_command(["git", "config", "commit.gpgsign", "false"], work) + hashes = [] + for index, files in enumerate(commits, start=1): + for name, content in files.items(): + (work / name).write_text(content, encoding="utf-8") + self.run_command(["git", "add", "-A"], work) + self.run_command(["git", "commit", "-m", f"v{index}"], work) + hashes.append(self.run_command(["git", "rev-parse", "HEAD"], work)) + self.run_command( + ["git", "clone", "--bare", str(work), str(destination)] + ) + return hashes[0], hashes[-1] + + def write_sources(self, content): + (self.project_root / "sources.txt").write_text( + content, encoding="utf-8" + ) + + def create_project_fixture(self, name, remote, pinned_hash): + project_root = self.test_root / "containers" / name + image_root = project_root / name + (project_root / "src").mkdir(parents=True) + (image_root / "src").mkdir(parents=True) + (project_root / "sources.txt").write_text( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + f"master {name} {remote} master {pinned_hash}\n", + encoding="utf-8", + ) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + (image_root / "bindeps.txt").write_text("python3\n", encoding="utf-8") + (image_root / "builddeps.txt").write_text("gcc\n", encoding="utf-8") + (image_root / "pythondeps.txt").touch() + (image_root / "pythonbuilddeps.txt").touch() + + def run_update_result(self, *targets, **environment): + command_environment = os.environ.copy() + command_environment.update({"STREAM": "master"}) + command_environment.update(environment) + if not targets: + targets = ("test-svc",) + result = subprocess.run( + ["bash", "./build.sh", "update-sources", *targets], + cwd=self.test_root, + check=False, + capture_output=True, + text=True, + env=command_environment, + ) + (self.test_root / "build.log").write_text( + result.stdout + result.stderr, encoding="utf-8" + ) + return result + + def run_update(self, *targets, **environment): + result = self.run_update_result(*targets, **environment) + if result.returncode != 0: + self.fail( + f"update-sources failed with {result.returncode}:\n" + f"{result.stdout}{result.stderr}" + ) + return result.stdout + result.stderr + + def manifest_for_stream(self, stream): + filename = f"frozen-source-refs.{stream.replace('/', '%2F')}.tsv" + return self.test_root / ".tmp/source-maintenance" / filename + + def manifest_rows(self): + with self.manifest_path.open(encoding="utf-8", newline="") as stream: + return list(csv.DictReader(stream, delimiter="\t")) + + def source_field(self, name, field, project_root=None): + if project_root is None: + project_root = self.project_root + for line in ( + (project_root / "sources.txt") + .read_text(encoding="utf-8") + .splitlines() + ): + columns = line.split() + if columns[:2] == ["master", name]: + return columns[field - 1] + self.fail(f"source entry was not found: {name}") + + def test_updates_hashes_to_frozen_branch_tips(self): + self.run_update() + + self.assertEqual( + self.requirements_new, + self.source_field("upper-constraints", 5), + ) + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + rows = self.manifest_rows() + self.assertEqual( + ["upper-constraints", "test-svc"], [row["name"] for row in rows] + ) + self.assertEqual( + [self.requirements_new, self.service_new], + [row["frozen_commit"] for row in rows], + ) + self.assertEqual({"declared-ref"}, {row["authority"] for row in rows}) + + def test_fetches_upper_constraints(self): + self.run_update() + + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertIn("pbr==7.0.3", constraints) + + def test_generates_rpms_in_yaml(self): + self.run_update() + + rpms = (self.project_root / "rpms.in.yaml").read_text(encoding="utf-8") + self.assertIn("python3", rpms) + self.assertIn("gcc", rpms) + + def test_generates_requirements_lock(self): + self.run_update() + + lock = (self.project_root / "requirements.lock.master").read_text( + encoding="utf-8" + ) + self.assertIn("six", lock) + + def test_generates_buildrequirements_lock(self): + self.run_update() + + build_lock = self.project_root / "buildrequirements.lock.master" + self.assertTrue(build_lock.is_file()) + self.assertNotIn("# via", build_lock.read_text(encoding="utf-8")) + + def test_creates_default_stream_symlinks(self): + self.run_update(DEFAULT_STREAM="master") + + expected = { + "upper-constraints.txt": "upper-constraints.txt.master", + "requirements.lock": "requirements.lock.master", + "buildrequirements.lock": "buildrequirements.lock.master", + } + for name, target in expected.items(): + link = self.project_root / name + self.assertTrue(link.is_symlink()) + self.assertEqual(target, os.readlink(link)) + + def test_skips_symlinks_for_non_default_stream(self): + self.run_update(DEFAULT_STREAM="other") + + for name in ( + "upper-constraints.txt", + "requirements.lock", + "buildrequirements.lock", + ): + self.assertFalse((self.project_root / name).is_symlink()) + + def test_skip_hash_update_preserves_and_records_committed_hashes(self): + self.run_update(SKIP_HASH_UPDATE="1") + + self.assertEqual( + self.requirements_old, + self.source_field("upper-constraints", 5), + ) + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + self.assertTrue( + (self.project_root / "requirements.lock.master").is_file() + ) + rows = self.manifest_rows() + self.assertEqual( + [self.requirements_old, self.service_old], + [row["frozen_commit"] for row in rows], + ) + self.assertEqual({"committed-pin"}, {row["authority"] for row in rows}) + + def test_skip_hash_update_uses_pinned_constraints(self): + self.run_update(SKIP_HASH_UPDATE="1") + + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertNotIn("pbr", constraints) + + def test_hash_in_branch_field_selects_constraints_commit(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} {self.requirements_old} " + f"{self.requirements_new}\n" + "master test-svc " + f"{self.upstream_service} master {self.service_old}\n" + ) + + self.run_update() + + self.assertEqual( + self.requirements_old, + self.source_field("upper-constraints", 5), + ) + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertNotIn("pbr", constraints) + + def test_hash_in_branch_field_selects_regular_repo_commit(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} {self.service_old} " + f"{self.service_new}\n" + ) + + self.run_update() + + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + self.assertTrue( + (self.project_root / "requirements.lock.master").is_file() + ) + + def test_lockfile_excludes_rpm_python_packages(self): + image_root = self.project_root / "test-svc" + (image_root / "bindeps.txt").write_text( + "python3\npython3-six\n", encoding="utf-8" + ) + + output = self.run_update() + + lock = (self.project_root / "requirements.lock.master").read_text( + encoding="utf-8" + ) + self.assertNotIn("six==", lock) + self.assertIn("Filtering RPM-provided packages", output) + + def test_preexisting_checkout_is_preserved_and_recorded(self): + source = self.project_root / "src" / "test-svc" + self.run_command( + ["git", "clone", str(self.upstream_service), str(source)] + ) + (source / "MARKER").write_text("local-dev\n", encoding="utf-8") + + self.run_update() + + self.assertEqual( + "local-dev\n", + (source / "MARKER").read_text(encoding="utf-8"), + ) + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + service_row = self.manifest_rows()[1] + self.assertEqual("pre-existing-checkout", service_row["authority"]) + self.assertEqual(self.service_new, service_row["frozen_commit"]) + + def test_unversioned_preexisting_source_fails_before_mutation(self): + source = self.project_root / "src" / "test-svc" + source.mkdir() + (source / "requirements.txt").write_text("six\n", encoding="utf-8") + original_sources = (self.project_root / "sources.txt").read_bytes() + + result = self.run_update_result() + + self.assertNotEqual(0, result.returncode) + self.assertIn("is not a Git checkout", result.stderr) + self.assertEqual( + original_sources, (self.project_root / "sources.txt").read_bytes() + ) + self.assertFalse( + (self.project_root / "upper-constraints.txt.master").exists() + ) + self.assertFalse(self.manifest_path.exists()) + + def test_slash_stream_uses_safe_manifest_filename(self): + result = self.run_update_result(STREAM="stable/test") + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + manifest = self.manifest_for_stream("stable/test") + self.assertTrue(manifest.is_file()) + self.assertEqual( + self.test_root / ".tmp/source-maintenance", manifest.parent + ) + self.assertFalse( + (self.test_root / ".tmp/source-maintenance/stable").exists() + ) + + def test_unsafe_stream_fails_before_filesystem_mutation(self): + original_sources = (self.project_root / "sources.txt").read_bytes() + + result = self.run_update_result(STREAM="../../escape") + + self.assertNotEqual(0, result.returncode) + self.assertIn("Unsafe stream name", result.stderr) + self.assertEqual( + original_sources, (self.project_root / "sources.txt").read_bytes() + ) + self.assertFalse((self.test_root / ".tmp").exists()) + + def test_unresolvable_ref_fails_before_mutation(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} missing-branch {self.service_old}\n" + ) + original_sources = (self.project_root / "sources.txt").read_bytes() + + result = self.run_update_result() + + self.assertNotEqual(0, result.returncode) + self.assertIn("Could not freeze ref 'missing-branch'", result.stderr) + self.assertEqual( + original_sources, (self.project_root / "sources.txt").read_bytes() + ) + self.assertFalse( + (self.project_root / "upper-constraints.txt.master").exists() + ) + self.assertFalse(self.manifest_path.exists()) + + def test_failed_preflight_removes_stale_manifest(self): + self.run_update() + self.assertTrue(self.manifest_path.is_file()) + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_new}\n" + "master test-svc " + f"{self.upstream_service} missing-branch {self.service_new}\n" + ) + + result = self.run_update_result() + + self.assertNotEqual(0, result.returncode) + self.assertFalse(self.manifest_path.exists()) + + def test_list_discovers_all_projects(self): + result = subprocess.run( + ["bash", "./build.sh", "list"], + cwd=self.test_root, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("test-svc/test-svc", result.stdout) + self.assertIn("test-svc2/test-svc2", result.stdout) + self.assertIn("test-svc3/test-svc3", result.stdout) + + def test_multiple_targets_update_only_selected_projects(self): + self.run_update("test-svc", "test-svc2", DEFAULT_STREAM="master") + + service2_new = self.additional_services["test-svc2"][2] + service3_old = self.additional_services["test-svc3"][1] + project2 = self.test_root / "containers/test-svc2" + project3 = self.test_root / "containers/test-svc3" + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + self.assertEqual( + service2_new, + self.source_field("test-svc2", 5, project2), + ) + self.assertEqual( + service3_old, + self.source_field("test-svc3", 5, project3), + ) + for project in (self.project_root, project2): + self.assertTrue( + (project / "upper-constraints.txt.master").is_file() + ) + self.assertTrue((project / "requirements.lock.master").is_file()) + self.assertTrue((project / "rpms.in.yaml").is_file()) + self.assertTrue((project / "requirements.lock").is_symlink()) + self.assertFalse((project3 / "upper-constraints.txt.master").exists()) + self.assertFalse((project3 / "requirements.lock.master").exists()) + self.assertFalse((project3 / "rpms.in.yaml").exists()) + + def test_single_target_does_not_affect_other_projects(self): + self.run_update("test-svc") + + project2 = self.test_root / "containers/test-svc2" + self.assertEqual( + self.additional_services["test-svc2"][1], + self.source_field("test-svc2", 5, project2), + ) + self.assertFalse((project2 / "requirements.lock.master").exists()) + + def test_all_updates_every_project(self): + self.run_update("all") + + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + for name in ("test-svc2", "test-svc3"): + project = self.test_root / "containers" / name + self.assertEqual( + self.additional_services[name][2], + self.source_field(name, 5, project), + ) + self.assertTrue((project / "requirements.lock.master").is_file()) + + def test_unknown_target_fails_before_mutation(self): + result = self.run_update_result("nonexistent") + + self.assertNotEqual(0, result.returncode) + self.assertIn("ERROR: Unknown image or project", result.stderr) + self.assertFalse(self.manifest_path.exists()) + + def test_duplicate_ref_records_retain_deterministic_rows(self): + image_sources = self.project_root / "test-svc" / "sources.txt" + image_sources.write_text( + "master helper " + f"{self.upstream_service} master {self.service_old}\n", + encoding="utf-8", + ) + + self.run_update() + + rows = self.manifest_rows() + self.assertEqual( + ["upper-constraints", "test-svc", "helper"], + [row["name"] for row in rows], + ) + self.assertEqual(rows[1]["frozen_commit"], rows[2]["frozen_commit"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_update_sources.sh b/tests/test_update_sources.sh deleted file mode 100755 index 9a3ed9b6..00000000 --- a/tests/test_update_sources.sh +++ /dev/null @@ -1,529 +0,0 @@ -#!/usr/bin/env bash -# Tests for build.sh update-sources functionality. -# -# Uses local bare git repos as fake remotes so tests run offline and fast. -# pip-compile and pybuild-deps must be on PATH for lockfile tests. -# -# Usage: -# PATH=".tox/update-sources/bin:$PATH" bash tests/test_update_sources.sh -# tox -e test -# -set -uo pipefail - -# ── Test runner ────────────────────────────────────────────────────────── - -_PASS=0 -_FAIL=0 -_SKIP=0 - -assert() { - local desc="$1" - shift - if "$@"; then - return 0 - fi - echo " ASSERTION FAILED: ${desc}" - echo " command: $*" - return 1 -} - -assert_file_exists() { assert "file exists: $1" test -f "$1"; } -assert_symlink() { assert "symlink exists: $1" test -L "$1"; } -assert_no_symlink() { assert "no symlink: $1" test ! -L "$1"; } -assert_grep() { assert "grep '$1' in $2" grep -q "$1" "$2"; } -assert_no_grep() { - if grep -q "$1" "$2" 2>/dev/null; then - echo " ASSERTION FAILED: '$1' should not appear in $2" - return 1 - fi -} - -assert_link_target() { - local link="$1" expected="$2" - local actual - actual="$(readlink "$1")" - assert "symlink $1 -> $2 (actual: ${actual})" test "${actual}" = "${expected}" -} - -assert_field() { - local file="$1" stream="$2" name="$3" field="$4" expected="$5" - local actual - actual=$(awk -v s="${stream}" -v n="${name}" '$1==s && $2==n {print $'${field}'}' "${file}") - assert "sources.txt ${name} field ${field} == ${expected} (actual: ${actual})" \ - test "${actual}" = "${expected}" -} - -run_test() { - local name="$1" - - _setup_fixture - - local rc=0 - # Run in subshell with set -e so first failed assertion stops the test - ( set -e; "${name}" ) || rc=$? - - if [[ ${rc} -eq 0 ]]; then - echo " PASS ${name}" - ((_PASS++)) - elif [[ ${rc} -eq 99 ]]; then - echo " SKIP ${name}" - ((_SKIP++)) - else - echo " FAIL ${name}" - ((_FAIL++)) - if [[ -f "${TEST_DIR}/build.log" ]]; then - echo " --- build.log (last 20 lines) ---" - tail -20 "${TEST_DIR}/build.log" | sed 's/^/ /' - echo " ---" - fi - fi - - _teardown_fixture -} - -skip_test() { - echo " skipping: $1" - return 99 -} - -# ── Fixture ────────────────────────────────────────────────────────────── - -SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -TEST_DIR="" -UPSTREAM_REQ="" -UPSTREAM_SVC="" -UPSTREAM_SVC2="" -UPSTREAM_SVC3="" -REQ_HASH_OLD="" -REQ_HASH_NEW="" -SVC_HASH_OLD="" -SVC_HASH_NEW="" -SVC2_HASH_OLD="" -SVC2_HASH_NEW="" -SVC3_HASH_OLD="" -SVC3_HASH_NEW="" - -_init_work_repo() { - local dir="$1" - git init -b master "${dir}" >/dev/null 2>&1 - git -C "${dir}" config user.email "test@test.com" - git -C "${dir}" config user.name "Test" -} - -_setup_fixture() { - TEST_DIR="$(mktemp -d)" - local work - - # ── Upstream requirements repo (2 commits) ── - UPSTREAM_REQ="${TEST_DIR}/upstream/requirements.git" - mkdir -p "${TEST_DIR}/upstream" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six==1.17.0" > "${work}/upper-constraints.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six==1.17.0\npbr==7.0.3\n' > "${work}/upper-constraints.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_REQ}" >/dev/null 2>&1 - rm -rf "${work}" - - REQ_HASH_OLD="$(git -C "${UPSTREAM_REQ}" rev-parse master~1)" - REQ_HASH_NEW="$(git -C "${UPSTREAM_REQ}" rev-parse master)" - - # ── Upstream service repo (2 commits) ── - UPSTREAM_SVC="${TEST_DIR}/upstream/test-svc.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC_HASH_OLD="$(git -C "${UPSTREAM_SVC}" rev-parse master~1)" - SVC_HASH_NEW="$(git -C "${UPSTREAM_SVC}" rev-parse master)" - - # ── Upstream service 2 repo (2 commits) ── - UPSTREAM_SVC2="${TEST_DIR}/upstream/test-svc2.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC2}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC2_HASH_OLD="$(git -C "${UPSTREAM_SVC2}" rev-parse master~1)" - SVC2_HASH_NEW="$(git -C "${UPSTREAM_SVC2}" rev-parse master)" - - # ── Upstream service 3 repo (2 commits) ── - UPSTREAM_SVC3="${TEST_DIR}/upstream/test-svc3.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC3}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC3_HASH_OLD="$(git -C "${UPSTREAM_SVC3}" rev-parse master~1)" - SVC3_HASH_NEW="$(git -C "${UPSTREAM_SVC3}" rev-parse master)" - - # ── Symlink build.sh ── - ln -s "${SCRIPT_DIR}/build.sh" "${TEST_DIR}/build.sh" - - # ── Containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc/src" - mkdir -p "${TEST_DIR}/containers/test-svc/test-svc/src" - - cat > "${TEST_DIR}/containers/test-svc/sources.txt" < "${TEST_DIR}/containers/test-svc/test-svc/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc/test-svc/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc/test-svc/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc/test-svc/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc/test-svc/pythonbuilddeps.txt" - - # ── Second project containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc2/src" - mkdir -p "${TEST_DIR}/containers/test-svc2/test-svc2/src" - - cat > "${TEST_DIR}/containers/test-svc2/sources.txt" < "${TEST_DIR}/containers/test-svc2/test-svc2/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc2/test-svc2/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc2/test-svc2/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc2/test-svc2/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc2/test-svc2/pythonbuilddeps.txt" - - # ── Third project containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc3/src" - mkdir -p "${TEST_DIR}/containers/test-svc3/test-svc3/src" - - cat > "${TEST_DIR}/containers/test-svc3/sources.txt" < "${TEST_DIR}/containers/test-svc3/test-svc3/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc3/test-svc3/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc3/test-svc3/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc3/test-svc3/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc3/test-svc3/pythonbuilddeps.txt" -} - -_teardown_fixture() { - [[ -n "${TEST_DIR}" ]] && rm -rf "${TEST_DIR}" -} - -# Helper: run build.sh inside TEST_DIR with env vars passed as arguments. -# Usage: _run_build STREAM=master [SKIP_HASH_UPDATE=1 ...] -_run_build() { - (cd "${TEST_DIR}" && env "$@" ./build.sh update-sources test-svc) >"${TEST_DIR}/build.log" 2>&1 -} - -# Helper: run build.sh with arbitrary action and targets. -# Usage: _run_cmd -- [targets...] -_run_cmd() { - local env_args=() - while [[ $# -gt 0 && "$1" != "--" ]]; do - env_args+=("$1") - shift - done - [[ "$1" == "--" ]] && shift - (cd "${TEST_DIR}" && env "${env_args[@]}" ./build.sh "$@") >"${TEST_DIR}/build.log" 2>&1 -} - -# ── Tests ──────────────────────────────────────────────────────────────── - -test_updates_hashes_to_branch_tip() { - _run_build STREAM=master - - local src="${TEST_DIR}/containers/test-svc/sources.txt" - assert_field "${src}" master upper-constraints 5 "${REQ_HASH_NEW}" - assert_field "${src}" master test-svc 5 "${SVC_HASH_NEW}" -} - -test_fetches_upper_constraints() { - _run_build STREAM=master - - local uc="${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${uc}" - assert_grep "six==1.17.0" "${uc}" - assert_grep "pbr==7.0.3" "${uc}" -} - -test_generates_rpms_in_yaml() { - _run_build STREAM=master - - local rpms="${TEST_DIR}/containers/test-svc/rpms.in.yaml" - assert_file_exists "${rpms}" - assert_grep "python3" "${rpms}" - assert_grep "gcc" "${rpms}" -} - -test_generates_requirements_lock() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_build STREAM=master - - local lock="${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${lock}" - assert_grep "six" "${lock}" -} - -test_generates_buildrequirements_lock() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master - - assert_file_exists "${TEST_DIR}/containers/test-svc/buildrequirements.lock.master" -} - -test_creates_default_stream_symlinks() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master DEFAULT_STREAM=master - - local d="${TEST_DIR}/containers/test-svc" - assert_symlink "${d}/upper-constraints.txt" - assert_symlink "${d}/requirements.lock" - assert_symlink "${d}/buildrequirements.lock" - assert_link_target "${d}/requirements.lock" "requirements.lock.master" - assert_link_target "${d}/buildrequirements.lock" "buildrequirements.lock.master" - assert_link_target "${d}/upper-constraints.txt" "upper-constraints.txt.master" -} - -test_skips_symlinks_for_non_default_stream() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master DEFAULT_STREAM=other - - local d="${TEST_DIR}/containers/test-svc" - assert_no_symlink "${d}/requirements.lock" - assert_no_symlink "${d}/buildrequirements.lock" - assert_no_symlink "${d}/upper-constraints.txt" -} - -test_skip_hash_update_preserves_hashes() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_build STREAM=master SKIP_HASH_UPDATE=1 - - local src="${TEST_DIR}/containers/test-svc/sources.txt" - assert_field "${src}" master upper-constraints 5 "${REQ_HASH_OLD}" - assert_field "${src}" master test-svc 5 "${SVC_HASH_OLD}" - - assert_file_exists "${TEST_DIR}/containers/test-svc/requirements.lock.master" -} - -test_skip_hash_update_fetches_constraints_at_pinned_hash() { - _run_build STREAM=master SKIP_HASH_UPDATE=1 - - local uc="${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${uc}" - assert_grep "six==1.17.0" "${uc}" - assert_no_grep "pbr" "${uc}" -} - -test_hash_in_branch_field_upper_constraints() { - cat > "${TEST_DIR}/containers/test-svc/sources.txt" </dev/null 2>&1 || skip_test "pip-compile not on PATH" - - cat > "${TEST_DIR}/containers/test-svc/sources.txt" </dev/null 2>&1 || skip_test "pip-compile not on PATH" - - printf 'python3\npython3-six\n' > "${TEST_DIR}/containers/test-svc/test-svc/bindeps.txt" - - _run_build STREAM=master - - local lock="${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${lock}" - assert_no_grep "^six==" "${lock}" - assert_grep "Filtering RPM-provided packages" "${TEST_DIR}/build.log" -} - -test_preexisting_checkout_is_preserved() { - local src_dir="${TEST_DIR}/containers/test-svc/src/test-svc" - mkdir -p "${src_dir}" - echo "local-dev" > "${src_dir}/MARKER" - echo "six" > "${src_dir}/requirements.txt" - - _run_build STREAM=master - - assert_file_exists "${src_dir}/MARKER" - assert_grep "local-dev" "${src_dir}/MARKER" - assert_field "${TEST_DIR}/containers/test-svc/sources.txt" master test-svc 5 "${SVC_HASH_OLD}" -} - -# ── Multi-target tests ────────────────────────────────────────────────── - -test_list_discovers_all_projects() { - _run_cmd STREAM=master -- list - assert_grep "test-svc/test-svc" "${TEST_DIR}/build.log" - assert_grep "test-svc2/test-svc2" "${TEST_DIR}/build.log" -} - -test_update_sources_multiple_targets() { - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - local src1="${TEST_DIR}/containers/test-svc/sources.txt" - local src2="${TEST_DIR}/containers/test-svc2/sources.txt" - local src3="${TEST_DIR}/containers/test-svc3/sources.txt" - assert_field "${src1}" master test-svc 5 "${SVC_HASH_NEW}" - assert_field "${src2}" master test-svc2 5 "${SVC2_HASH_NEW}" - assert_field "${src3}" master test-svc3 5 "${SVC3_HASH_OLD}" -} - -test_update_sources_multiple_targets_fetches_constraints() { - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - assert_file_exists "${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${TEST_DIR}/containers/test-svc2/upper-constraints.txt.master" - assert "no constraints for test-svc3" test ! -f "${TEST_DIR}/containers/test-svc3/upper-constraints.txt.master" -} - -test_update_sources_multiple_targets_generates_lockfiles() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - assert_file_exists "${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${TEST_DIR}/containers/test-svc2/requirements.lock.master" - assert "no lockfile for test-svc3" test ! -f "${TEST_DIR}/containers/test-svc3/requirements.lock.master" -} - -test_update_sources_multiple_targets_generates_rpms_in() { - _run_cmd STREAM=master -- update-sources test-svc test-svc2 - - assert_file_exists "${TEST_DIR}/containers/test-svc/rpms.in.yaml" - assert_file_exists "${TEST_DIR}/containers/test-svc2/rpms.in.yaml" - assert "no rpms.in.yaml for test-svc3" test ! -f "${TEST_DIR}/containers/test-svc3/rpms.in.yaml" -} - -test_update_sources_single_target_does_not_affect_other() { - _run_cmd STREAM=master -- update-sources test-svc - - local src2="${TEST_DIR}/containers/test-svc2/sources.txt" - assert_field "${src2}" master test-svc2 5 "${SVC2_HASH_OLD}" -} - -test_update_sources_all_updates_everything() { - _run_cmd STREAM=master -- update-sources all - - local src1="${TEST_DIR}/containers/test-svc/sources.txt" - local src2="${TEST_DIR}/containers/test-svc2/sources.txt" - local src3="${TEST_DIR}/containers/test-svc3/sources.txt" - assert_field "${src1}" master test-svc 5 "${SVC_HASH_NEW}" - assert_field "${src2}" master test-svc2 5 "${SVC2_HASH_NEW}" - assert_field "${src3}" master test-svc3 5 "${SVC3_HASH_NEW}" -} - -test_update_sources_unknown_target_fails() { - if _run_cmd STREAM=master -- update-sources nonexistent 2>/dev/null; then - echo " ASSERTION FAILED: expected failure for unknown target" - return 1 - fi - assert_grep "ERROR: Unknown image or project" "${TEST_DIR}/build.log" -} - -test_update_sources_multiple_targets_symlinks() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_cmd STREAM=master DEFAULT_STREAM=master -- update-sources test-svc test-svc2 - - for proj in test-svc test-svc2; do - local d="${TEST_DIR}/containers/${proj}" - assert_symlink "${d}/requirements.lock" - assert_link_target "${d}/requirements.lock" "requirements.lock.master" - done -} - -# ── Run all tests ──────────────────────────────────────────────────────── - -echo "=== update-sources tests ===" -echo "" - -TESTS=( - test_updates_hashes_to_branch_tip - test_fetches_upper_constraints - test_generates_rpms_in_yaml - test_generates_requirements_lock - test_generates_buildrequirements_lock - test_creates_default_stream_symlinks - test_skips_symlinks_for_non_default_stream - test_skip_hash_update_preserves_hashes - test_skip_hash_update_fetches_constraints_at_pinned_hash - test_hash_in_branch_field_upper_constraints - test_hash_in_branch_field_regular_repo - test_lockfile_excludes_rpm_python_packages - test_preexisting_checkout_is_preserved - test_list_discovers_all_projects - test_update_sources_multiple_targets - test_update_sources_multiple_targets_fetches_constraints - test_update_sources_multiple_targets_generates_lockfiles - test_update_sources_multiple_targets_generates_rpms_in - test_update_sources_single_target_does_not_affect_other - test_update_sources_all_updates_everything - test_update_sources_unknown_target_fails - test_update_sources_multiple_targets_symlinks -) - -for t in "${TESTS[@]}"; do - run_test "${t}" -done - -echo "" -echo "=== ${_PASS} passed, ${_FAIL} failed, ${_SKIP} skipped ===" - -[[ ${_FAIL} -eq 0 ]] diff --git a/tox.ini b/tox.ini index 2a9ebb16..1c4494d0 100644 --- a/tox.ini +++ b/tox.ini @@ -20,19 +20,25 @@ passenv = SKIP_HASH_UPDATE PIP_NO_BINARY REGISTRY_AUTH_FILE + REGISTRY_CERT_DIR PARALLEL BUILD_LOGS_DIR UPDATE_LOCKFILES_TARGETS # pin to pip==26.1.2 as 26.2 has broken pip-tools -# pin pip-tools<7.6.1: pybuild-deps 0.5.0 passes generate_hashes to +# pin pip-tools==7.6.0: pybuild-deps 0.5.0 passes generate_hashes to # OutputWriter which was removed in pip-tools 7.6.1 deps = - pip-tools<7.6.1 - pybuild-deps + pip-tools==7.6.0 + pybuild-deps==0.5.0 pip==26.1.2 allowlist_externals = bash +[testenv:{unit,test,py3}] +description = Run all non-Ansible tests through stdlib unittest +commands = + python -m unittest discover -s {toxinidir}/tests -p 'test_*.py' -v {posargs} + [testenv:linters] description = Run formatting and style checks through pre-commit deps = @@ -47,26 +53,47 @@ setenv = commands = pre-commit run -a --show-diff-on-failure +[testenv:molecule] +description = Exercise provider registry and cleanup contracts with Molecule +deps = + ansible-core==2.18.7 + molecule==26.6.0 +setenv = + PYTHONPYCACHEPREFIX = {toxinidir}/.tmp/python-cache + MOLECULE_EPHEMERAL_DIRECTORY = {toxinidir}/.tmp/molecule/ephemeral/{envname} + ANSIBLE_HOME = {toxinidir}/.tmp/ansible/home + ANSIBLE_COLLECTIONS_PATH = {toxinidir}/.tmp/ansible/collections + ANSIBLE_LOCAL_TEMP = {toxinidir}/.tmp/ansible/local-tmp + ANSIBLE_ROLES_PATH = {toxinidir}/.tmp/ansible/roles +commands = + molecule reset -s provider-contract + molecule test -s provider-contract + [testenv:update-sources] +description = Refresh source pins and generated lockfiles +setenv = + XDG_CACHE_HOME = {envtmpdir}/cache +commands_pre = + python -c "import sys; assert sys.version_info[:2] == (3, 12), 'dependency generation requires Python 3.12'" commands = - bash -c "rm -f ~/.cache/pybuild-deps/find-build-deps" + bash -c "rm -f {env:XDG_CACHE_HOME}/pybuild-deps/find-build-deps" bash {toxinidir}/build.sh update-sources {posargs:all} [testenv:update-lockfiles] +description = Refresh generated lockfiles without advancing source pins setenv = + XDG_CACHE_HOME = {envtmpdir}/cache SKIP_HASH_UPDATE = 1 +commands_pre = + {[testenv:update-sources]commands_pre} commands = - bash -c "rm -f ~/.cache/pybuild-deps/find-build-deps" + bash -c "rm -f {env:XDG_CACHE_HOME}/pybuild-deps/find-build-deps" bash {toxinidir}/build.sh update-sources {posargs:all} [testenv:build] commands = bash {toxinidir}/build.sh build {posargs:all} -[testenv:test] -commands = - bash {toxinidir}/tests/test_update_sources.sh - [testenv:custom] commands = bash {toxinidir}/build.sh {posargs} diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml new file mode 100644 index 00000000..99fe2b35 --- /dev/null +++ b/zuul.d/jobs.yaml @@ -0,0 +1,38 @@ +--- +- job: + name: s2i-openstack-containers-molecule + parent: tox + description: Run provider registry and cleanup contract scenarios. + nodeset: s2i-openstack-containers-image-builder + vars: + python_version: "3.12" + tox_envlist: molecule + +- nodeset: + name: s2i-openstack-containers-image-builder + nodes: + - name: builder + label: cloud-centos-10-stream + +- job: + name: s2i-openstack-container-content-provider + parent: base + description: | + Build and publish OpenStack service container images from committed + source pins for dependent jobs. This repository builds all maintained + images by default; child jobs can select a subset and add related + required projects. Speculative source consumption remains follow-up work. + timeout: 7200 + nodeset: s2i-openstack-containers-image-builder + required-projects: + - github.com/openstack-k8s-operators/s2i-openstack-containers + - opendev.org/zuul/zuul-jobs + pre-run: playbooks/container-ci/zuul/pre.yaml + run: playbooks/container-ci/zuul/run.yaml + post-run: playbooks/container-ci/zuul/post.yaml + vars: + s2i_ci_container_project: >- + github.com/openstack-k8s-operators/s2i-openstack-containers + s2i_ci_images: all + s2i_ci_stream: master + s2i_ci_content_provider: true diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index ca302dea..4bd2a3be 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -1,8 +1,12 @@ --- -# Minimal RDO github-check wiring so Zuul loads in-repo config from main. -# Expand with real jobs once webhook/status checks are confirmed. - project: name: openstack-k8s-operators/s2i-openstack-containers github-check: jobs: - - noop + - s2i-openstack-containers-molecule: + irrelevant-files: &documentation-only + - ^docs/.* + - ^.*\.md$ + - ^LICENSE.*$ + - s2i-openstack-container-content-provider: + irrelevant-files: *documentation-only