[backport-gcp-2.22] Sync CI configuration with main branch

.buildkite/hooks/post-checkout

@@ -1,5 +1,18 @@
 #!/bin/bash
 
+# ******************************* WARNING ******************************************
+# This post-checkout hook is not the same as in the rest of repositories e.g. beats
+# because some steps in this pipeline (in PR context) take a very long time and we
+# want to make sure that THE SAME COMMIT FROM TARGET BRANCH gets merged in every
+# pipeline step. Otherwise, HEAD (target branch) may have changed in the meantime
+# and therefore, some steps (e.g. sonarqube) may end up testing a different commit.
+#
+# Running builds from branches or tags (out of PR context) maintains the same behavior
+# as in the rest of the repositories.
+#
+# Reference: https://github.com/elastic/integrations/pull/10397
+# **********************************************************************************
+
 set -euo pipefail
 
 checkout_merge() {
@@ -13,8 +26,15 @@ checkout_merge() {
     fi
 
     git fetch -v origin "${target_branch}"
-    git checkout FETCH_HEAD
-    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    if [[ ${REPOSITORY_TARGET_BRANCH_COMMIT} == "" ]]; then
+        git checkout FETCH_HEAD
+        echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    else
+        # Use the same commit from target branch as in the other steps.
+        echo "Retrieved commit from meta-data: ${REPOSITORY_TARGET_BRANCH_COMMIT}"
+        git checkout "${REPOSITORY_TARGET_BRANCH_COMMIT}"
+        echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    fi
 
     # create temporal branch to merge the PR with the target branch
     git checkout -b ${merge_branch}
@@ -44,6 +64,9 @@ PR_COMMIT="${BUILDKITE_COMMIT}"
 PR_ID=${BUILDKITE_PULL_REQUEST}
 MERGE_BRANCH="pr_merge_${PR_ID}"
 
+# This meta-data field is populated in the pre-command hook
+REPOSITORY_TARGET_BRANCH_COMMIT=$(buildkite-agent meta-data get "REPOSITORY_TARGET_BRANCH_COMMIT" --default "")
+
 checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"

.buildkite/hooks/pre-command

@@ -11,7 +11,10 @@ git config --global core.pager 'cat'
 export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export SERVERLESS=${SERVERLESS:-"false"}
 export STACK_VERSION=${STACK_VERSION:-""}
+export ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""}
+export STACK_LOGSDB_ENABLED=${STACK_LOGSDB_ENABLED:-"false"}
 export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
+export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
 
 BASE_DIR=$(pwd)
 export BASE_DIR
@@ -22,32 +25,82 @@ export GO_VERSION
 REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 export REPO_NAME
 
-export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
-export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
-
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 export REPO_BUILD_TAG
 
-AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
-PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
-
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
-GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 
-EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 
+export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
+
+# -------------
+# variables required by packages using Terraform as a service deployer
+# https://github.com/elastic/elastic-package/blob/f8f2f15a04bcc25eca00887fb147bd7f8a0f32b3/internal/servicedeployer/_static/terraform_deployer.yml#L8
+
+export ENVIRONMENT="ci"
+export REPO="${REPO_NAME}"
+
+branch_name_label() {
+    local branch="$1"
+
+    if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]] ; then
+        # remove fork from branch name
+        branch=$(echo $branch | cut -d : -f 2)
+    fi
+
+    # From Jenkins
+    # BRANCH_NAME_LOWER_CASE = "${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}"
+    # to lower case and replace characters
+    branch=$(echo "$branch" | tr '[:upper:]' '[:lower:]' | tr '_/\:. ' '-')
+
+    # truncate up to 63 characters limit
+    echo $branch | head -c 63
+}
+
+BRANCH_NAME_LOWER_CASE=$(branch_name_label "$BUILDKITE_BRANCH")
+export BRANCH_NAME_LOWER_CASE
+# This variable contains the build number https://buildkite.com/elastic/elastic-package/<number>
+export BUILD_ID="${BUILDKITE_BUILD_NUMBER}"
+# get current timestamp in milliseconds
+# From Jenkins
+# CREATED_DATE = "${new Date().getTime()}"
+CREATED_DATE=$(date +%s%3N)
+export CREATED_DATE
+
+# -------------
+
+if [[ "${ELASTIC_PACKAGE_CUSTOMIZE_SERVICE_TEST_RUN_ID:-"false"}" == "true" ]]; then
+    # Required to customize the RunID value mainly for those packages creating resources in cloud providers
+    # via the terraform service deployer.
+    # Get the latest 4 digits of the BUILDKITE_STEP_ID
+    export ELASTIC_PACKAGE_PREFIX_SERVICE_TEST_RUN_ID="${BUILDKITE_STEP_ID: -4}"
+fi
+
 if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     # first upload pipeline does not have the environment variables defined in the YAML
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
 
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
+if [[ ( "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ) &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
+    # Get the commit from target branch in the first step (reference-target-branch).
+    # This step MUST be the first one and not run in parallel with any other step to ensure
+    # that there is just one value for this variable
+    if is_pr ; then
+        git fetch -v origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        commit_main=$(git rev-parse --verify FETCH_HEAD)
+        buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}"
+    fi
+fi
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
         export BUILDKITE_API_TOKEN
     fi
+fi
 
+if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
@@ -56,51 +109,15 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
-        GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
-        export GITHUB_TOKEN
-    fi
-
-    if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
-        ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_SECRET_KEY
-        ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_ACCESS_KEY
-
-        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
-        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
-
-        # Environment variables required by the service deployer
-        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
-        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
-
-        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
-        export BUILDKITE_API_TOKEN
     fi
 fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
-        ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_SECRET_KEY
-        ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_ACCESS_KEY
-
-        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
-        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
-
-        # Environment variables required by the service deployer
-        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
-        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
 
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
 
-        EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
-        export EC_API_KEY_SECRET
-        EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
-        export EC_HOST_SECRET
         EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
         export EC_REGION_SECRET
     fi

.buildkite/hooks/pre-exit

@@ -4,44 +4,51 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
-    # FIXME: update condition depending on the pipeline steps triggered
-    if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
-        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-        unset AWS_ACCESS_KEY_ID
-        unset AWS_SECRET_ACCESS_KEY
+should_run_stack_down() {
+    # as first check, ensure that the elastic-package binary exists
+    if [ ! -f "${ELASTIC_PACKAGE_BIN}" ]; then
+        return 1
+    fi
 
-        # Ensure that kind cluster is deleted
-        delete_kind_cluster
+    if is_serverless; then
+        return 0
+    fi
 
-        # Ensure elastic stack is stopped
-        if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
-            echo "--- Take down the Elastic stack"
-            ${ELASTIC_PACKAGE_BIN} stack down -v
-        fi
+    if is_stack_created; then
+        return 0
     fi
+
+    return 1
+}
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
+  cd "${WORKSPACE}"
+  git config remote.origin.url "https://github.com/elastic/integrations.git"
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
-        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-        unset AWS_ACCESS_KEY_ID
-        unset AWS_SECRET_ACCESS_KEY
+exit_code=0
+if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack|integrations-serverless)$ ]]; then
+    # it should match "^test-integration-" steps created in the integrations and integrations-test-stack pipelines (e.g. test-integration-apache or test-integration-aws)
+    # as well as the step ID "test-integrations-serverless-project" from the "integrations-serverless" pipeline
+    if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
 
         # Ensure that kind cluster is deleted
         delete_kind_cluster
 
         # Ensure elastic stack is stopped
-        if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
+        if should_run_stack_down; then
             echo "--- Take down the Elastic stack"
-            EC_API_KEY=${EC_API_KEY_SECRET} EC_HOST=${EC_HOST_SECRET} ${ELASTIC_PACKAGE_BIN} stack down -v
+            if ! ${ELASTIC_PACKAGE_BIN} stack down -v ; then
+                exit_code=1
+            fi
         fi
+
+        echo "+++ :bookmark: Documentation to access logs"
+        inline_link "https://docs.elastic.dev/ingest-dev-docs/elastic-packages/ecosystem-ci-pipelines#private-logs"
     fi
 fi
 
+echo "--- Cleaning up"
 unset_secrets
-cleanup
 
-google_cloud_logout_active_account
+exit "${exit_code}"

.buildkite/pipeline.backport-dispatch.yml

@@ -0,0 +1,43 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.6.0"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  YQ_VERSION: 'v4.35.2'
+  GH_CLI_VERSION: "2.29.0"
+  NOTIFY_TO: "ecosystem-team@elastic.co"
+
+steps:
+  - label: ":ballot_box_with_check: Backports inventory validation"
+    key: "check-backports-inventory"
+    command: ".buildkite/scripts/check_backports_inventory.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+    if_changed:
+      - ".backports.yml"
+      - ".buildkite/scripts/run_dev_scripts_tests.sh"
+      - "dev/scripts/**.sh"
+      - "dev/backports/**"
+    if: |
+      build.env('BUILDKITE_PULL_REQUEST') == "false" && build.branch == "main"
+
+  - label: ":git: Create backport branches for new entries"
+    key: "trigger-backport-create"
+    command: ".buildkite/scripts/trigger_backport.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+    plugins:
+      - elastic/vault-github-token#v0.1.0:
+    depends_on:
+      - step: "check-backports-inventory"
+        allow_failure: false
+    if_changed:
+      - ".backports.yml"
+    if: |
+      build.env('BUILDKITE_PULL_REQUEST') == "false" &&
+      build.branch == "main" &&
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations-backport-dispatch"
+
+notify:
+  - email: "$NOTIFY_TO"
+    if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"