diff --git a/changelog.d/2-wire-builds/dump-version-info b/changelog.d/2-wire-builds/dump-version-info new file mode 100644 index 000000000..562586784 --- /dev/null +++ b/changelog.d/2-wire-builds/dump-version-info @@ -0,0 +1 @@ +Added: Dump version information post creating the artifacts on the stdout diff --git a/changelog.d/2-wire-builds/q2-2026 b/changelog.d/2-wire-builds/q2-2026 new file mode 100644 index 000000000..e1d78f4f2 --- /dev/null +++ b/changelog.d/2-wire-builds/q2-2026 @@ -0,0 +1 @@ +Added: Update the wire-builds reference for 2026-q2 diff --git a/changelog.d/5-bug-fixes/pipeline-fixes b/changelog.d/5-bug-fixes/pipeline-fixes new file mode 100644 index 000000000..381d78f0d --- /dev/null +++ b/changelog.d/5-bug-fixes/pipeline-fixes @@ -0,0 +1 @@ +Fixed: Update the docker pull logic to retry if initial pull fails and wait when connecting to fresh VMs in hetzner for cd diff --git a/nix/scripts/create-container-dump.sh b/nix/scripts/create-container-dump.sh index 84bee63ce..cecf792eb 100644 --- a/nix/scripts/create-container-dump.sh +++ b/nix/scripts/create-container-dump.sh @@ -13,34 +13,72 @@ export REGISTRY_TIMEOUT=600 # Registry specific timeout output_dir=$1 mkdir -p $1 + # Download all the docker images into $1, and append its name to an index.txt # If this errors out for you, copy default-policy.json from the skopeo repo to # /etc/containers/policy.json while IFS= read -r image; do - # sanitize the image file name, replace slashes with underscores, suffix with .tar - image_filename=$(sed -r "s/[:\/]/_/g" <<< $image) - image_path=$(realpath $1)/${image_filename}.tar - if [[ -e $image_path ]];then - echo "Skipping $image_filename…" + +# sanitize the image file name, replace slashes with underscores, suffix with .tar + image_filename=$(sed -r "s/[:\/]/_/g" <<< "$image") + image_path="$(realpath "$1")/${image_filename}.tar" + + if [[ -s "$image_path" ]]; then + echo "Skipping $image_filename…" + continue + fi + + echo "Fetching $image_filename…" + + # All of these images should be publicly fetchable, especially given we + # ship public tarballs containing these images. + # ci.sh already honors DOCKER_LOGIN, so do the same here, otherwise + # fallback to unauthorized fetching. + + # If an image has both a tag and digest, remove the tag. Return the original if there is no match. + image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/') + + tmp_path="${image_path}.tmp" + rm -f "$tmp_path" + + success=false + + for attempt in {1..5}; do + echo "Attempt $attempt/5 for $image_trimmed" + + if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]]; then + skopeo copy --insecure-policy \ + --src-creds "$DOCKER_LOGIN" \ + --retry-times 10 \ + "docker://$image_trimmed" \ + "docker-archive:${tmp_path}" \ + --additional-tag "$image" || rc=$? else - echo "Fetching $image_filename…" - - # All of these images should be publicly fetchable, especially given we - # ship public tarballs containing these images. - # ci.sh already honors DOCKER_LOGIN, so do the same here, otherwise - # fallback to unauthorized fetching. - - # If an image has both a tag and digest, remove the tag. Return the original if there is no match. - image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/') - if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]];then - skopeo copy --insecure-policy --src-creds "$DOCKER_LOGIN" --retry-times 10 \ - docker://$image_trimmed docker-archive:${image_path} --additional-tag $image - else - skopeo copy --insecure-policy --retry-times 10 \ - docker://$image_trimmed docker-archive:${image_path} --additional-tag $image - fi - echo "${image_filename}.tar" >> $(realpath "$1")/index.txt - # passing image and $output_dir - create-build-entry $image $output_dir + skopeo copy --insecure-policy \ + --retry-times 10 \ + "docker://$image_trimmed" \ + "docker-archive:${tmp_path}" \ + --additional-tag "$image" || rc=$? + fi + + rc=$? + + if [[ $rc -eq 0 && -s "$tmp_path" ]]; then + mv "$tmp_path" "$image_path" + success=true + break fi + + echo "Fetch failed for $image_trimmed with rc=$rc; retrying…" + rm -f "$tmp_path" + sleep $((attempt * 20)) + done + + if [[ "$success" != true ]]; then + echo "ERROR: failed to fetch $image after retries" >&2 + exit 1 + fi + + echo "${image_filename}.tar" >> "$(realpath "$1")/index.txt" + create-build-entry "$image" "$output_dir" done diff --git a/offline/default-build/build.sh b/offline/default-build/build.sh index a94855536..239b5937c 100755 --- a/offline/default-build/build.sh +++ b/offline/default-build/build.sh @@ -116,3 +116,16 @@ done # Create the tar archive with relative paths tar czf "$OUTPUT_TAR" "${ITEMS_TO_ARCHIVE[@]}" + +# Dumping details of versions for the build and packed +echo "Dump of versions/helm_image_tree.json" +cat "${OUTPUT_DIR}/versions/helm_image_tree.json" + +echo "Dump of versions/containers_system_images.json" +cat "${OUTPUT_DIR}/versions/containers_system_images.json" + +echo "Dump of versions/wire-binaries.json" +cat "${OUTPUT_DIR}/versions/wire-binaries.json" + +echo "Dump of wire-builds used" +cat "${OUTPUT_DIR}/build.json" diff --git a/offline/demo-build/build.sh b/offline/demo-build/build.sh index 7aafee54f..a33349b1d 100755 --- a/offline/demo-build/build.sh +++ b/offline/demo-build/build.sh @@ -83,3 +83,10 @@ done # Create the tar archive with relative paths tar czf "$OUTPUT_TAR" "${ITEMS_TO_ARCHIVE[@]}" + +# Dumping details of versions for the build and packed +echo "Dump of versions/helm_image_tree.json" +cat "${OUTPUT_DIR}/versions/helm_image_tree.json" + +echo "Dump of wire-builds used" +cat "${OUTPUT_DIR}/build.json" diff --git a/offline/min-build/build.sh b/offline/min-build/build.sh index 00f0f2421..94567feee 100755 --- a/offline/min-build/build.sh +++ b/offline/min-build/build.sh @@ -94,3 +94,10 @@ done # Create the tar archive with relative paths tar czf "$OUTPUT_TAR" "${ITEMS_TO_ARCHIVE[@]}" + +# Dumping details of versions for the build and packed +echo "Dump of versions/helm_image_tree.json" +cat "${OUTPUT_DIR}/versions/helm_image_tree.json" + +echo "Dump of wire-builds used" +cat "${OUTPUT_DIR}/build.json" diff --git a/offline/tasks/proc_pull_charts.sh b/offline/tasks/proc_pull_charts.sh index 8c0f57d17..41392babe 100755 --- a/offline/tasks/proc_pull_charts.sh +++ b/offline/tasks/proc_pull_charts.sh @@ -36,13 +36,14 @@ echo "Excluding following charts from the release: $HELM_CHART_EXCLUDE_LIST" wire_build_chart_release () { wire_build="$1" - curl "$wire_build" | jq -r --argjson HELM_CHART_EXCLUDE_LIST "$HELM_CHART_EXCLUDE_LIST" ' + curl "$wire_build" -o "${OUTPUT_DIR}/build.json" + jq -r --argjson HELM_CHART_EXCLUDE_LIST "$HELM_CHART_EXCLUDE_LIST" ' .helmCharts | with_entries(select(.key as $k | $HELM_CHART_EXCLUDE_LIST | index($k) | not)) | to_entries | map("\(.key) \(.value.repo) \(.value.version)") | join("\n") - ' + ' "${OUTPUT_DIR}/build.json" } # pull_charts() accepts charts in format @@ -84,5 +85,5 @@ pull_charts() { } -wire_build="https://raw.githubusercontent.com/wireapp/wire-builds/818524e35d2894f5486c50b9ed9ed967ac099561/build.json" +wire_build="https://raw.githubusercontent.com/wireapp/wire-builds/refs/heads/2026-q2/build.json" wire_build_chart_release "$wire_build" | pull_charts diff --git a/terraform/examples/wiab-staging-hetzner/outputs.tf b/terraform/examples/wiab-staging-hetzner/outputs.tf index 8fc10ec4e..c3fa5037b 100644 --- a/terraform/examples/wiab-staging-hetzner/outputs.tf +++ b/terraform/examples/wiab-staging-hetzner/outputs.tf @@ -55,7 +55,7 @@ output "static-inventory" { } } vars = { - ansible_ssh_common_args = "-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null -o ControlMaster=auto -o ControlPersist=60s -o BatchMode=yes -o ConnectionAttempts=10 -o ServerAliveInterval=60 -o ServerAliveCountMax=3" + ansible_ssh_common_args = "-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null -o ControlMaster=auto -o ControlPersist=60s -o BatchMode=yes -o ConnectionAttempts=10 -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=10" } } private = { @@ -66,7 +66,7 @@ output "static-inventory" { adminhost_local = {} } vars = { - ansible_ssh_common_args = "-o ProxyCommand=\"ssh -i ssh_private_key -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null -W %h:%p -q root@${hcloud_server.adminhost.ipv4_address}\" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null -o ControlMaster=auto -o ControlPersist=60s -o BatchMode=yes -o ConnectionAttempts=10 -o ServerAliveInterval=60 -o ServerAliveCountMax=3" + ansible_ssh_common_args = "-o ProxyCommand=\"ssh -i ssh_private_key -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null -W %h:%p -q root@${hcloud_server.adminhost.ipv4_address}\" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null -o ControlMaster=auto -o ControlPersist=60s -o BatchMode=yes -o ConnectionAttempts=10 -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=10" } } adminhost_local = { diff --git a/terraform/examples/wiab-staging-hetzner/setup_nodes.yml b/terraform/examples/wiab-staging-hetzner/setup_nodes.yml index 0bb041ea9..b77b9ec3d 100644 --- a/terraform/examples/wiab-staging-hetzner/setup_nodes.yml +++ b/terraform/examples/wiab-staging-hetzner/setup_nodes.yml @@ -1,4 +1,20 @@ --- +- name: Wait for adminhost private SSH + hosts: adminhost + gather_facts: no + tasks: + - name: Wait for SSH on public adminhost + wait_for_connection: + timeout: 300 + delay: 5 + + - name: Wait until adminhost private IP is reachable from public adminhost + wait_for: + host: "{{ hostvars['adminhost_local'].ansible_host }}" + port: 22 + timeout: 300 + delay: 5 + - name: Setup adminhost with dnsmasq and Docker hosts: adminhost_local become: yes diff --git a/values/sftd/demo-values.example.yaml b/values/sftd/demo-values.example.yaml index 91dc2c885..1335d8d15 100644 --- a/values/sftd/demo-values.example.yaml +++ b/values/sftd/demo-values.example.yaml @@ -3,10 +3,10 @@ host: sftd.example.com replicaCount: 1 joinCall: replicaCount: 1 - image: - repository: docker.io/bitnamilegacy/nginx - pullPolicy: IfNotPresent - tag: "1.27.3-debian-12-r5" +# image: +# repository: docker.io/bitnamilegacy/nginx +# pullPolicy: IfNotPresent +# tag: "1.27.3-debian-12-r5" tls: issuerRef: name: letsencrypt-http01 diff --git a/values/sftd/prod-values.example.yaml b/values/sftd/prod-values.example.yaml index 1c2374f9e..7eca857ab 100644 --- a/values/sftd/prod-values.example.yaml +++ b/values/sftd/prod-values.example.yaml @@ -10,14 +10,14 @@ tls: name: letsencrypt-http01 kind: ClusterIssuer -joinCall: -# this value should be set to 3 when deployed in a full production DMZ manner -# replicaCount = 1 is to support the simple wiab-staging solution - replicaCount: 1 - image: - repository: docker.io/bitnamilegacy/nginx - pullPolicy: IfNotPresent - tag: "1.27.3-debian-12-r5" +#joinCall: +## this value should be set to 3 when deployed in a full production DMZ manner +## replicaCount = 1 is to support the simple wiab-staging solution +# replicaCount: 1 +# image: +# repository: docker.io/bitnamilegacy/nginx +# pullPolicy: IfNotPresent +# tag: "1.27.3-debian-12-r5" # Uncomment to enable SFT to SFT communication for federated calls # multiSFT: diff --git a/values/wire-server/prod-values.example.yaml b/values/wire-server/prod-values.example.yaml index 5ab911af7..a2d987bfe 100644 --- a/values/wire-server/prod-values.example.yaml +++ b/values/wire-server/prod-values.example.yaml @@ -3,6 +3,7 @@ tags: proxy: false # enable if you want/need giphy/youtube/etc proxying legalhold: false # Enable if you need legalhold federation: false # Enable to use federation + mlsstats: true cassandra-migrations: # images: