From ca31ae815125e33fb68a30f42e5f80fcd3c4e1ed Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 5 Aug 2026 18:02:39 +0200 Subject: [PATCH 1/6] Sync CI configuration with main branch --- .buildkite/hooks/post-checkout | 27 +- .buildkite/hooks/pre-command | 107 +- .buildkite/hooks/pre-exit | 55 +- .buildkite/pipeline.backport-dispatch.yml | 43 + .buildkite/pipeline.backport.yml | 108 +- .buildkite/pipeline.publish.yml | 52 + .buildkite/pipeline.schedule-daily.yml | 86 +- .buildkite/pipeline.schedule-weekly.yml | 42 + .buildkite/pipeline.serverless.yml | 70 +- .buildkite/pipeline.yml | 203 ++- .buildkite/pull-requests.json | 75 +- .buildkite/scripts/backport_branch.sh | 385 +++++ .buildkite/scripts/backport_branch_lib.sh | 60 + .buildkite/scripts/build_packages.sh | 48 +- .buildkite/scripts/check_backport_owners.sh | 133 ++ .buildkite/scripts/check_backport_tool.sh | 27 + .../scripts/check_backports_inventory.sh | 15 + .buildkite/scripts/check_changelog_entries.sh | 217 +++ .../check_changelog_versions_in_main.sh | 11 + .buildkite/scripts/check_sources.sh | 25 +- .buildkite/scripts/common.sh | 852 +++++++--- .../scripts/find_oldest_supported_version.py | 80 +- .buildkite/scripts/non_package_patterns.txt | 58 + .buildkite/scripts/notify_backport_pr.sh | 64 + .../packages/security_detection_engine.sh | 120 ++ .buildkite/scripts/process_benchmarks.sh | 2 +- .buildkite/scripts/report_issues.sh | 40 + .../requirements-ci-python-scripts.txt | 3 + .../scripts/run_buildkite_scripts_tests.sh | 69 + .buildkite/scripts/run_dev_scripts_tests.sh | 10 + .buildkite/scripts/test_backport_branch.sh | 241 +++ .../scripts/test_check_backport_owners.sh | 74 + .../scripts/test_check_changelog_entries.sh | 196 +++ .buildkite/scripts/test_helpers.sh | 55 + .../test_integrations_with_serverless.sh | 40 +- .../scripts/test_non_package_patterns.sh | 148 ++ .buildkite/scripts/test_one_package.sh | 39 +- .buildkite/scripts/test_trigger_backport.sh | 253 +++ .buildkite/scripts/trigger_backport.sh | 107 ++ .buildkite/scripts/trigger_backport_lib.sh | 148 ++ .../trigger_integrations_in_parallel.sh | 67 +- .github/workflows/auto-backport.yml | 177 ++ .../workflows/backport-packages-detect.yml | 60 + .../workflows/bump-elastic-stack-version.yml | 52 + .github/workflows/catalog-info.yml | 24 + .github/workflows/ci-comment.yml | 55 + .github/workflows/docs-build.yml | 13 + .github/workflows/docs-deploy.yml | 14 + .github/workflows/docs-edit-automation.yml | 217 +++ .github/workflows/docs-preview-cleanup.yml | 11 + .../workflows/notify-package-docs-failure.yml | 57 + .github/workflows/post-backport-checklist.yml | 121 ++ .github/workflows/pr-buildkite-detective.yml | 39 + .github/workflows/requires-update.yml | 81 + .../workflows/sweep-dashboard-data-scope.yml | 101 ++ .../sweep-field-mapping-conflicts.yml | 98 ++ .../workflows/sweep-httpjson-pagination.yml | 105 ++ .../sweep-ingest-pipeline-safety.yml | 97 ++ .../sweep-pipeline-error-handling.yml | 116 ++ .github/workflows/sync-backport-changelog.yml | 82 + .../trigger-breaking-change-detector.yml | 29 + .github/workflows/trigger-bug-hunter.yml | 19 + .github/workflows/trigger-docs-patrol.yml | 20 + .../trigger-duplicate-issue-detector.yml | 21 + .github/workflows/trigger-issue-triage.yml | 29 + .../workflows/trigger-mention-in-issue.yml | 31 + .github/workflows/trigger-mention-in-pr.yml | 31 + .../trigger-newbie-contributor-patrol.yml | 18 + .../trigger-package-tests-security-ml.yml | 79 + .../trigger-pr-actions-detective.yml | 48 + .github/workflows/trigger-pr-review.yml | 25 + .github/workflows/trigger-stale-issues.yml | 18 + .github/workflows/trigger-text-auditor.yml | 20 + .../updatecli.d/bump-latest-7x-version.yml | 48 + .../bump-latest-8x-snapshot-version.yml | 84 + .../bump-latest-9x-snapshot-version.yml | 84 + .../sync-packages-to-bug-issue-template.yml | 53 + .github/workflows/updatecli/values.d/scm.yml | 4 + .github/workflows/vale-lint.yml | 106 ++ .github/workflows/vale-report.yml | 40 + .github/workflows/validate-package-docs.yml | 96 ++ .../validate-yaml-dashboards.requirements.txt | 2 + .../workflows/validate-yaml-dashboards.yml | 223 +++ .gitignore | 18 + cmd/backport/.go-version | 1 + cmd/backport/backports/apply/apply.go | 1091 ++++++++++++ .../backports/apply/apply_integration_test.go | 1465 +++++++++++++++++ cmd/backport/backports/apply/apply_test.go | 576 +++++++ cmd/backport/backports/changelog/collect.go | 272 +++ .../backports/changelog/collect_test.go | 229 +++ cmd/backport/backports/changelog/comment.go | 103 ++ .../backports/changelog/comment_test.go | 136 ++ cmd/backport/backports/changelog/extract.go | 57 + .../backports/changelog/extract_test.go | 118 ++ cmd/backport/backports/changelog/insert.go | 67 + .../backports/changelog/insert_test.go | 94 ++ cmd/backport/backports/changelog/resolve.go | 27 + .../backports/changelog/resolve_test.go | 82 + cmd/backport/backports/changelog/sync.go | 140 ++ cmd/backport/backports/changelog/sync_test.go | 86 + cmd/backport/backports/changelog/update.go | 55 + .../backports/changelog/update_test.go | 114 ++ cmd/backport/backports/checklist/checklist.go | 166 ++ .../backports/checklist/checklist_test.go | 362 ++++ cmd/backport/backports/inventory.go | 467 ++++++ cmd/backport/backports/inventory_test.go | 1103 +++++++++++++ cmd/backport/backports/owners/check.go | 105 ++ cmd/backport/backports/owners/check_test.go | 71 + cmd/backport/backports/owners/compare.go | 135 ++ .../owners/compare_integration_test.go | 111 ++ cmd/backport/backports/owners/owners.go | 210 +++ cmd/backport/backports/owners/owners_test.go | 253 +++ cmd/backport/backports/packages/detect.go | 52 + .../backports/packages/detect_test.go | 142 ++ cmd/backport/citools/packagemanifest.go | 88 + cmd/backport/citools/packages.go | 76 + cmd/backport/codeowners/codeowners.go | 151 ++ cmd/backport/gitutil/git.go | 52 + cmd/backport/go.mod | 26 + cmd/backport/go.sum | 81 + cmd/backport/magefile.go | 123 ++ cmd/backport/main.go | 657 ++++++++ cmd/backport/tools.go | 14 + dev/citools/gomod.go | 45 + dev/citools/gomod_test.go | 94 ++ dev/citools/kibana.go | 50 + dev/citools/kibana_test.go | 141 ++ dev/citools/logsdb.go | 59 + dev/citools/logsdb_test.go | 104 ++ dev/citools/packagemanifest.go | 89 + dev/citools/packages.go | 78 + dev/citools/subscription.go | 47 + dev/citools/subscription_test.go | 180 ++ dev/codeowners/codeowners.go | 280 +++- dev/codeowners/codeowners_test.go | 337 ++++ .../testdata/CODEOWNERS-nested-category-owner | 2 + .../testdata/CODEOWNERS-nested-missing-owner | 2 + .../CODEOWNERS-nested-streams-missing-owners | 4 + .../testdata/CODEOWNERS-nested-streams-valid | 5 + .../testdata/CODEOWNERS-nested-valid | 3 + .../CODEOWNERS-owners-packages-datastreams | 10 + .../testdata/CODEOWNERS-owners-trailing-slash | 7 + .../CODEOWNERS-streams-missing-owners | 2 + .../CODEOWNERS-streams-multiple-owners | 3 + .../testdata/CODEOWNERS-streams-valid | 3 + dev/codeowners/testdata/devexp/manifest.yml | 4 + .../data_stream/stream_1/.keep | 0 .../data_stream/stream_2/.keep | 0 .../category/package_nested_1/manifest.yml | 6 + .../category/package_nested_2/manifest.yml | 6 + .../nested_packages/package_top/manifest.yml | 6 + .../package_1/data_stream/stream_1/.keep | 0 .../package_1/data_stream/stream_2/.keep | 0 .../test_packages/package_1/manifest.yml | 6 + dev/coverage/coverage.go | 143 ++ dev/coverage/coverage_test.go | 31 + .../testdata/expected-test-coverage.xml | 23 + dev/coverage/testdata/test-coverage-1.xml | 23 + dev/coverage/testdata/test-coverage-2.xml | 9 + dev/coverage/testdata/test-coverage-3.xml | 23 + dev/gitutil/git.go | 39 + dev/import-beats/README.md | 2 +- dev/import-beats/elasticsearch.go | 2 +- dev/import-beats/fields.go | 6 +- dev/import-beats/packages.go | 7 +- dev/import-beats/variables.go | 2 +- dev/import-beats/variables_compact.go | 2 +- dev/packagenames/packagenames.go | 56 + dev/packagenames/packagenames_test.go | 63 + .../testdata/duplicates/p1/manifest.yml | 4 + .../testdata/duplicates/p2/manifest.yml | 4 + .../invalid_manifests/p1/manifest.yml | 4 + .../invalid_manifests/p2/manifest.yml | 3 + .../nested/duplicates/p3/manifest.yml | 4 + .../duplicates/technology/p1/manifest.yml | 4 + .../duplicates/technology/p2/manifest.yml | 4 + .../nested/no_duplicates/p3/manifest.yml | 4 + .../no_duplicates/technology/p1/manifest.yml | 4 + .../no_duplicates/technology/p2/manifest.yml | 4 + .../testdata/no_duplicates/p1/manifest.yml | 4 + .../testdata/no_duplicates/p2/manifest.yml | 4 + dev/scripts/README.md | 12 + dev/scripts/backport_apply.sh | 83 + dev/scripts/backport_bootstrap_inventory.sh | 337 ++++ dev/scripts/get_release_commit.sh | 85 + dev/scripts/test_get_release_commit.sh | 148 ++ go.mod | 13 +- go.sum | 26 + magefile.go | 139 +- tools.go | 2 + 190 files changed, 18691 insertions(+), 484 deletions(-) create mode 100644 .buildkite/pipeline.backport-dispatch.yml create mode 100644 .buildkite/pipeline.publish.yml create mode 100644 .buildkite/pipeline.schedule-weekly.yml create mode 100755 .buildkite/scripts/backport_branch.sh create mode 100755 .buildkite/scripts/backport_branch_lib.sh create mode 100755 .buildkite/scripts/check_backport_owners.sh create mode 100755 .buildkite/scripts/check_backport_tool.sh create mode 100755 .buildkite/scripts/check_backports_inventory.sh create mode 100755 .buildkite/scripts/check_changelog_entries.sh create mode 100755 .buildkite/scripts/check_changelog_versions_in_main.sh create mode 100644 .buildkite/scripts/non_package_patterns.txt create mode 100755 .buildkite/scripts/notify_backport_pr.sh create mode 100755 .buildkite/scripts/packages/security_detection_engine.sh create mode 100755 .buildkite/scripts/report_issues.sh create mode 100644 .buildkite/scripts/requirements-ci-python-scripts.txt create mode 100755 .buildkite/scripts/run_buildkite_scripts_tests.sh create mode 100755 .buildkite/scripts/run_dev_scripts_tests.sh create mode 100755 .buildkite/scripts/test_backport_branch.sh create mode 100755 .buildkite/scripts/test_check_backport_owners.sh create mode 100755 .buildkite/scripts/test_check_changelog_entries.sh create mode 100644 .buildkite/scripts/test_helpers.sh create mode 100755 .buildkite/scripts/test_non_package_patterns.sh create mode 100755 .buildkite/scripts/test_trigger_backport.sh create mode 100755 .buildkite/scripts/trigger_backport.sh create mode 100644 .buildkite/scripts/trigger_backport_lib.sh create mode 100644 .github/workflows/auto-backport.yml create mode 100644 .github/workflows/backport-packages-detect.yml create mode 100644 .github/workflows/bump-elastic-stack-version.yml create mode 100644 .github/workflows/catalog-info.yml create mode 100644 .github/workflows/ci-comment.yml create mode 100644 .github/workflows/docs-build.yml create mode 100644 .github/workflows/docs-deploy.yml create mode 100644 .github/workflows/docs-edit-automation.yml create mode 100644 .github/workflows/docs-preview-cleanup.yml create mode 100644 .github/workflows/notify-package-docs-failure.yml create mode 100644 .github/workflows/post-backport-checklist.yml create mode 100644 .github/workflows/pr-buildkite-detective.yml create mode 100644 .github/workflows/requires-update.yml create mode 100644 .github/workflows/sweep-dashboard-data-scope.yml create mode 100644 .github/workflows/sweep-field-mapping-conflicts.yml create mode 100644 .github/workflows/sweep-httpjson-pagination.yml create mode 100644 .github/workflows/sweep-ingest-pipeline-safety.yml create mode 100644 .github/workflows/sweep-pipeline-error-handling.yml create mode 100644 .github/workflows/sync-backport-changelog.yml create mode 100644 .github/workflows/trigger-breaking-change-detector.yml create mode 100644 .github/workflows/trigger-bug-hunter.yml create mode 100644 .github/workflows/trigger-docs-patrol.yml create mode 100644 .github/workflows/trigger-duplicate-issue-detector.yml create mode 100644 .github/workflows/trigger-issue-triage.yml create mode 100644 .github/workflows/trigger-mention-in-issue.yml create mode 100644 .github/workflows/trigger-mention-in-pr.yml create mode 100644 .github/workflows/trigger-newbie-contributor-patrol.yml create mode 100644 .github/workflows/trigger-package-tests-security-ml.yml create mode 100644 .github/workflows/trigger-pr-actions-detective.yml create mode 100644 .github/workflows/trigger-pr-review.yml create mode 100644 .github/workflows/trigger-stale-issues.yml create mode 100644 .github/workflows/trigger-text-auditor.yml create mode 100644 .github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml create mode 100644 .github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml create mode 100644 .github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml create mode 100644 .github/workflows/updatecli/updatecli.d/sync-packages-to-bug-issue-template.yml create mode 100644 .github/workflows/updatecli/values.d/scm.yml create mode 100644 .github/workflows/vale-lint.yml create mode 100644 .github/workflows/vale-report.yml create mode 100644 .github/workflows/validate-package-docs.yml create mode 100644 .github/workflows/validate-yaml-dashboards.requirements.txt create mode 100644 .github/workflows/validate-yaml-dashboards.yml create mode 100644 cmd/backport/.go-version create mode 100644 cmd/backport/backports/apply/apply.go create mode 100644 cmd/backport/backports/apply/apply_integration_test.go create mode 100644 cmd/backport/backports/apply/apply_test.go create mode 100644 cmd/backport/backports/changelog/collect.go create mode 100644 cmd/backport/backports/changelog/collect_test.go create mode 100644 cmd/backport/backports/changelog/comment.go create mode 100644 cmd/backport/backports/changelog/comment_test.go create mode 100644 cmd/backport/backports/changelog/extract.go create mode 100644 cmd/backport/backports/changelog/extract_test.go create mode 100644 cmd/backport/backports/changelog/insert.go create mode 100644 cmd/backport/backports/changelog/insert_test.go create mode 100644 cmd/backport/backports/changelog/resolve.go create mode 100644 cmd/backport/backports/changelog/resolve_test.go create mode 100644 cmd/backport/backports/changelog/sync.go create mode 100644 cmd/backport/backports/changelog/sync_test.go create mode 100644 cmd/backport/backports/changelog/update.go create mode 100644 cmd/backport/backports/changelog/update_test.go create mode 100644 cmd/backport/backports/checklist/checklist.go create mode 100644 cmd/backport/backports/checklist/checklist_test.go create mode 100644 cmd/backport/backports/inventory.go create mode 100644 cmd/backport/backports/inventory_test.go create mode 100644 cmd/backport/backports/owners/check.go create mode 100644 cmd/backport/backports/owners/check_test.go create mode 100644 cmd/backport/backports/owners/compare.go create mode 100644 cmd/backport/backports/owners/compare_integration_test.go create mode 100644 cmd/backport/backports/owners/owners.go create mode 100644 cmd/backport/backports/owners/owners_test.go create mode 100644 cmd/backport/backports/packages/detect.go create mode 100644 cmd/backport/backports/packages/detect_test.go create mode 100644 cmd/backport/citools/packagemanifest.go create mode 100644 cmd/backport/citools/packages.go create mode 100644 cmd/backport/codeowners/codeowners.go create mode 100644 cmd/backport/gitutil/git.go create mode 100644 cmd/backport/go.mod create mode 100644 cmd/backport/go.sum create mode 100644 cmd/backport/magefile.go create mode 100644 cmd/backport/main.go create mode 100644 cmd/backport/tools.go create mode 100644 dev/citools/gomod.go create mode 100644 dev/citools/gomod_test.go create mode 100644 dev/citools/kibana.go create mode 100644 dev/citools/kibana_test.go create mode 100644 dev/citools/logsdb.go create mode 100644 dev/citools/logsdb_test.go create mode 100644 dev/citools/packagemanifest.go create mode 100644 dev/citools/packages.go create mode 100644 dev/citools/subscription.go create mode 100644 dev/citools/subscription_test.go create mode 100644 dev/codeowners/testdata/CODEOWNERS-nested-category-owner create mode 100644 dev/codeowners/testdata/CODEOWNERS-nested-missing-owner create mode 100644 dev/codeowners/testdata/CODEOWNERS-nested-streams-missing-owners create mode 100644 dev/codeowners/testdata/CODEOWNERS-nested-streams-valid create mode 100644 dev/codeowners/testdata/CODEOWNERS-nested-valid create mode 100644 dev/codeowners/testdata/CODEOWNERS-owners-packages-datastreams create mode 100644 dev/codeowners/testdata/CODEOWNERS-owners-trailing-slash create mode 100644 dev/codeowners/testdata/CODEOWNERS-streams-missing-owners create mode 100644 dev/codeowners/testdata/CODEOWNERS-streams-multiple-owners create mode 100644 dev/codeowners/testdata/CODEOWNERS-streams-valid create mode 100644 dev/codeowners/testdata/nested_packages/category/package_nested_1/data_stream/stream_1/.keep create mode 100644 dev/codeowners/testdata/nested_packages/category/package_nested_1/data_stream/stream_2/.keep create mode 100644 dev/codeowners/testdata/nested_packages/category/package_nested_1/manifest.yml create mode 100644 dev/codeowners/testdata/nested_packages/category/package_nested_2/manifest.yml create mode 100644 dev/codeowners/testdata/nested_packages/package_top/manifest.yml create mode 100644 dev/codeowners/testdata/test_packages/package_1/data_stream/stream_1/.keep create mode 100644 dev/codeowners/testdata/test_packages/package_1/data_stream/stream_2/.keep create mode 100644 dev/codeowners/testdata/test_packages/package_1/manifest.yml create mode 100644 dev/coverage/coverage.go create mode 100644 dev/coverage/coverage_test.go create mode 100644 dev/coverage/testdata/expected-test-coverage.xml create mode 100644 dev/coverage/testdata/test-coverage-1.xml create mode 100644 dev/coverage/testdata/test-coverage-2.xml create mode 100644 dev/coverage/testdata/test-coverage-3.xml create mode 100644 dev/gitutil/git.go create mode 100644 dev/packagenames/packagenames.go create mode 100644 dev/packagenames/packagenames_test.go create mode 100644 dev/packagenames/testdata/duplicates/p1/manifest.yml create mode 100644 dev/packagenames/testdata/duplicates/p2/manifest.yml create mode 100644 dev/packagenames/testdata/invalid_manifests/p1/manifest.yml create mode 100644 dev/packagenames/testdata/invalid_manifests/p2/manifest.yml create mode 100644 dev/packagenames/testdata/nested/duplicates/p3/manifest.yml create mode 100644 dev/packagenames/testdata/nested/duplicates/technology/p1/manifest.yml create mode 100644 dev/packagenames/testdata/nested/duplicates/technology/p2/manifest.yml create mode 100644 dev/packagenames/testdata/nested/no_duplicates/p3/manifest.yml create mode 100644 dev/packagenames/testdata/nested/no_duplicates/technology/p1/manifest.yml create mode 100644 dev/packagenames/testdata/nested/no_duplicates/technology/p2/manifest.yml create mode 100644 dev/packagenames/testdata/no_duplicates/p1/manifest.yml create mode 100644 dev/packagenames/testdata/no_duplicates/p2/manifest.yml create mode 100644 dev/scripts/README.md create mode 100755 dev/scripts/backport_apply.sh create mode 100755 dev/scripts/backport_bootstrap_inventory.sh create mode 100755 dev/scripts/get_release_commit.sh create mode 100755 dev/scripts/test_get_release_commit.sh diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index b1572902664..08fed38fa66 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,5 +1,18 @@ #!/bin/bash +# ******************************* WARNING ****************************************** +# This post-checkout hook is not the same as in the rest of repositories e.g. beats +# because some steps in this pipeline (in PR context) take a very long time and we +# want to make sure that THE SAME COMMIT FROM TARGET BRANCH gets merged in every +# pipeline step. Otherwise, HEAD (target branch) may have changed in the meantime +# and therefore, some steps (e.g. sonarqube) may end up testing a different commit. +# +# Running builds from branches or tags (out of PR context) maintains the same behavior +# as in the rest of the repositories. +# +# Reference: https://github.com/elastic/integrations/pull/10397 +# ********************************************************************************** + set -euo pipefail checkout_merge() { @@ -13,8 +26,15 @@ checkout_merge() { fi git fetch -v origin "${target_branch}" - git checkout FETCH_HEAD - echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + if [[ ${REPOSITORY_TARGET_BRANCH_COMMIT} == "" ]]; then + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + else + # Use the same commit from target branch as in the other steps. + echo "Retrieved commit from meta-data: ${REPOSITORY_TARGET_BRANCH_COMMIT}" + git checkout "${REPOSITORY_TARGET_BRANCH_COMMIT}" + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + fi # create temporal branch to merge the PR with the target branch git checkout -b ${merge_branch} @@ -44,6 +64,9 @@ PR_COMMIT="${BUILDKITE_COMMIT}" PR_ID=${BUILDKITE_PULL_REQUEST} MERGE_BRANCH="pr_merge_${PR_ID}" +# This meta-data field is populated in the pre-command hook +REPOSITORY_TARGET_BRANCH_COMMIT=$(buildkite-agent meta-data get "REPOSITORY_TARGET_BRANCH_COMMIT" --default "") + checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" echo "Commit information" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index dd9be80cf85..846d46f37e5 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,10 @@ git config --global core.pager 'cat' export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"} export SERVERLESS=${SERVERLESS:-"false"} export STACK_VERSION=${STACK_VERSION:-""} +export ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""} +export STACK_LOGSDB_ENABLED=${STACK_LOGSDB_ENABLED:-"false"} export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"} +export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"} BASE_DIR=$(pwd) export BASE_DIR @@ -22,32 +25,82 @@ export GO_VERSION REPO_NAME=$(repo_name "${BUILDKITE_REPO}") export REPO_NAME -export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}" -export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" - REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)" export REPO_BUILD_TAG -AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth -PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account - BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token -GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token -EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa EC_DATA_PATH=secret/ci/elastic-integrations/ec_data +export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal" + +# ------------- +# variables required by packages using Terraform as a service deployer +# https://github.com/elastic/elastic-package/blob/f8f2f15a04bcc25eca00887fb147bd7f8a0f32b3/internal/servicedeployer/_static/terraform_deployer.yml#L8 + +export ENVIRONMENT="ci" +export REPO="${REPO_NAME}" + +branch_name_label() { + local branch="$1" + + if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]] ; then + # remove fork from branch name + branch=$(echo $branch | cut -d : -f 2) + fi + + # From Jenkins + # BRANCH_NAME_LOWER_CASE = "${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}" + # to lower case and replace characters + branch=$(echo "$branch" | tr '[:upper:]' '[:lower:]' | tr '_/\:. ' '-') + + # truncate up to 63 characters limit + echo $branch | head -c 63 +} + +BRANCH_NAME_LOWER_CASE=$(branch_name_label "$BUILDKITE_BRANCH") +export BRANCH_NAME_LOWER_CASE +# This variable contains the build number https://buildkite.com/elastic/elastic-package/ +export BUILD_ID="${BUILDKITE_BUILD_NUMBER}" +# get current timestamp in milliseconds +# From Jenkins +# CREATED_DATE = "${new Date().getTime()}" +CREATED_DATE=$(date +%s%3N) +export CREATED_DATE + +# ------------- + +if [[ "${ELASTIC_PACKAGE_CUSTOMIZE_SERVICE_TEST_RUN_ID:-"false"}" == "true" ]]; then + # Required to customize the RunID value mainly for those packages creating resources in cloud providers + # via the terraform service deployer. + # Get the latest 4 digits of the BUILDKITE_STEP_ID + export ELASTIC_PACKAGE_PREFIX_SERVICE_TEST_RUN_ID="${BUILDKITE_STEP_ID: -4}" +fi + if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then # first upload pipeline does not have the environment variables defined in the YAML export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH} fi -if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then +if [[ ( "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ) && "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then + # Get the commit from target branch in the first step (reference-target-branch). + # This step MUST be the first one and not run in parallel with any other step to ensure + # that there is just one value for this variable + if is_pr ; then + git fetch -v origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + commit_main=$(git rev-parse --verify FETCH_HEAD) + buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}" + fi +fi + +if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH}) export BUILDKITE_API_TOKEN fi +fi +if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]]; then if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations" ]]; then BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}") export BUILDKITE_API_TOKEN @@ -56,51 +109,15 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}") export BUILDKITE_API_TOKEN - GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}") - export GITHUB_TOKEN - fi - - if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then - ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}") - export ELASTIC_PACKAGE_AWS_SECRET_KEY - ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}") - export ELASTIC_PACKAGE_AWS_ACCESS_KEY - - PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}") - export PRIVATE_CI_GCS_CREDENTIALS_SECRET - export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci" - - # Environment variables required by the service deployer - export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY} - export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY} - - BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}") - export BUILDKITE_API_TOKEN fi fi if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then - ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}") - export ELASTIC_PACKAGE_AWS_SECRET_KEY - ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}") - export ELASTIC_PACKAGE_AWS_ACCESS_KEY - - PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}") - export PRIVATE_CI_GCS_CREDENTIALS_SECRET - export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci" - - # Environment variables required by the service deployer - export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY} - export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY} BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}") export BUILDKITE_API_TOKEN - EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}") - export EC_API_KEY_SECRET - EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}") - export EC_HOST_SECRET EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}") export EC_REGION_SECRET fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 5387aad888c..cb716289e14 100755 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -4,44 +4,51 @@ source .buildkite/scripts/common.sh set -euo pipefail -if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then - # FIXME: update condition depending on the pipeline steps triggered - if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then - unset ELASTIC_PACKAGE_AWS_ACCESS_KEY - unset ELASTIC_PACKAGE_AWS_SECRET_KEY - unset AWS_ACCESS_KEY_ID - unset AWS_SECRET_ACCESS_KEY +should_run_stack_down() { + # as first check, ensure that the elastic-package binary exists + if [ ! -f "${ELASTIC_PACKAGE_BIN}" ]; then + return 1 + fi - # Ensure that kind cluster is deleted - delete_kind_cluster + if is_serverless; then + return 0 + fi - # Ensure elastic stack is stopped - if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then - echo "--- Take down the Elastic stack" - ${ELASTIC_PACKAGE_BIN} stack down -v - fi + if is_stack_created; then + return 0 fi + + return 1 +} + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then + cd "${WORKSPACE}" + git config remote.origin.url "https://github.com/elastic/integrations.git" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then - if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then - unset ELASTIC_PACKAGE_AWS_ACCESS_KEY - unset ELASTIC_PACKAGE_AWS_SECRET_KEY - unset AWS_ACCESS_KEY_ID - unset AWS_SECRET_ACCESS_KEY +exit_code=0 +if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack|integrations-serverless)$ ]]; then + # it should match "^test-integration-" steps created in the integrations and integrations-test-stack pipelines (e.g. test-integration-apache or test-integration-aws) + # as well as the step ID "test-integrations-serverless-project" from the "integrations-serverless" pipeline + if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then # Ensure that kind cluster is deleted delete_kind_cluster # Ensure elastic stack is stopped - if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then + if should_run_stack_down; then echo "--- Take down the Elastic stack" - EC_API_KEY=${EC_API_KEY_SECRET} EC_HOST=${EC_HOST_SECRET} ${ELASTIC_PACKAGE_BIN} stack down -v + if ! ${ELASTIC_PACKAGE_BIN} stack down -v ; then + exit_code=1 + fi fi + + echo "+++ :bookmark: Documentation to access logs" + inline_link "https://docs.elastic.dev/ingest-dev-docs/elastic-packages/ecosystem-ci-pipelines#private-logs" fi fi +echo "--- Cleaning up" unset_secrets -cleanup -google_cloud_logout_active_account +exit "${exit_code}" diff --git a/.buildkite/pipeline.backport-dispatch.yml b/.buildkite/pipeline.backport-dispatch.yml new file mode 100644 index 00000000000..6a82c3bd3b3 --- /dev/null +++ b/.buildkite/pipeline.backport-dispatch.yml @@ -0,0 +1,43 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +env: + SETUP_GVM_VERSION: "v0.6.0" + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + YQ_VERSION: 'v4.35.2' + GH_CLI_VERSION: "2.29.0" + NOTIFY_TO: "ecosystem-team@elastic.co" + +steps: + - label: ":ballot_box_with_check: Backports inventory validation" + key: "check-backports-inventory" + command: ".buildkite/scripts/check_backports_inventory.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + if_changed: + - ".backports.yml" + - ".buildkite/scripts/run_dev_scripts_tests.sh" + - "dev/scripts/**.sh" + - "dev/backports/**" + if: | + build.env('BUILDKITE_PULL_REQUEST') == "false" && build.branch == "main" + + - label: ":git: Create backport branches for new entries" + key: "trigger-backport-create" + command: ".buildkite/scripts/trigger_backport.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "check-backports-inventory" + allow_failure: false + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') == "false" && + build.branch == "main" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations-backport-dispatch" + +notify: + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'" diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index d4e6e54caa5..f3562a2d969 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -1,6 +1,110 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "integrations-backport" + +env: + SETUP_GVM_VERSION: "v0.6.0" + YQ_VERSION: 'v4.35.2' + GH_CLI_VERSION: "2.29.0" + # Agent images used in pipeline steps + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + NOTIFY_TO: "ecosystem-team@elastic.co" + steps: - - label: "init pipeline" - command: echo "It works!" + - label: "Check that it runs from UI" + key: "check-ui" + command: + - "buildkite-agent annotate \"The $${BUILDKITE_PIPELINE_SLUG} pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' or 'integrations-backport-dispatch' pipeline. For the recommended workflow see the [documentation](https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-support-old-package.html).\" --style 'warning'" + - "exit 1" + if: | + !( + (build.source == 'ui' && build.creator.teams includes "ecosystem") || + (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') || + (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations-backport-dispatch') + ) + + # Ensure that the check-ui step runs before any other step + - wait: ~ + + - input: "Input values for the variables" + key: "input-variables" + if: "build.source != 'trigger_job'" + fields: + - select: "DRY_RUN" + key: "DRY_RUN" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "true" + - text: "Enter base commit for the backport branch" + key: "BASE_COMMIT" + required: true + default: "" + - text: "Enter package name (as defined in manifest.yml)" + key: "PACKAGE_NAME" + required: true + default: "" + - text: "Enter package version (examples: 1.5.7, 1.0.0-beta1)" + key: "PACKAGE_VERSION" + required: true + default: "" + - select: "Include to backport-branch only one package?" + key: "REMOVE_OTHER_PACKAGES" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "true" + - text: "Backport branch name (optional, defaults to backport--.)" + key: "BACKPORT_BRANCH_NAME" + required: false + default: "" + - text: "PR number to notify on completion (optional)" + key: "PR_NUMBER" + required: false + default: "" + + - label: "Creating the backport branch" + key: "create-backport-branch" + command: ".buildkite/scripts/backport_branch.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + env: + GITHUB_EMAIL: "elasticmachine@elastic.co" + GITHUB_USERNAME: "elastic-vault-github-plugin-prod" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "input-variables" + allow_failure: false + + - label: ":github: Notify PR" + key: "notify-pr" + command: | + if ! outcome=$$(buildkite-agent step get "outcome" --step "create-backport-branch"); then + echo "Failed to retrieve step outcome, skipping PR notification" + exit 0 + fi + if [[ "$${outcome}" == "passed" ]]; then + NOTIFY_STATUS=success .buildkite/scripts/notify_backport_pr.sh + else + NOTIFY_STATUS=failure .buildkite/scripts/notify_backport_pr.sh + fi + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "create-backport-branch" + allow_failure: true + +notify: + # Only notify for automated backport creation failures (triggered by integrations-backport-dispatch). + # Dry-run failures from PR builds (triggered by integrations) are already surfaced as a failing + # CI status on the PR itself and do not warrant a team email. + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations-backport-dispatch'" diff --git a/.buildkite/pipeline.publish.yml b/.buildkite/pipeline.publish.yml new file mode 100644 index 00000000000..64f5d60c7dc --- /dev/null +++ b/.buildkite/pipeline.publish.yml @@ -0,0 +1,52 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +env: + SETUP_GVM_VERSION: "v0.6.0" + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + DOCKER_COMPOSE_VERSION: "v2.24.1" + DOCKER_VERSION: "false" + YQ_VERSION: 'v4.35.2' + JQ_VERSION: '1.7' + # Elastic package settings + # Manage docker output/logs + ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true" + # Default license to use by `elastic-package build` + ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt" + # Link definitions path (full path to be set in the corresponding step) + ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml" + # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields + ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true" + NOTIFY_TO: "ecosystem-team@elastic.co" + +steps: + - label: ":white_check_mark: Check go sources" + key: "check" + command: ".buildkite/scripts/check_sources.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" + + - label: ":package: Build packages" + key: "build-packages" + command: ".buildkite/scripts/build_packages.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "8G" + env: + ARTIFACTS_FOLDER: "artifacts-to-sign" + # by default it will publish packages + DRY_RUN: "${DRY_RUN:-false}" + # Comma-separated list of package zip filenames to skip (e.g. "foo-1.0.0.zip,bar-2.0.0.zip") + # kubeletstats_input_otel-0.1.0 was deleted but it is kept in the repository for now, in case it is needed to release a new version if needed + SKIP_PACKAGES: "kubeletstats_input_otel-0.1.0.zip" + depends_on: + - step: "check" + allow_failure: false + artifact_paths: + - artifacts-to-sign/*.zip + +notify: + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'" diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index fef5d4c7943..aff5682d0ff 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -2,7 +2,7 @@ name: integrations-schedule-daily env: - SETUP_GVM_VERSION: "v0.5.1" + SETUP_GVM_VERSION: "v0.6.0" LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" # The pipeline is triggered by the scheduler every day @@ -20,24 +20,85 @@ steps: build: env: SERVERLESS: "false" - SKIP_PUBLISHING: "true" FORCE_CHECK_ALL: "true" - STACK_VERSION: 7.17-SNAPSHOT + STACK_VERSION: 7.17.29 depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_7_BRANCH') == "true" - - label: "Check integrations local stacks - Stack Version v8.12" + - label: "Check integrations local stacks - Stack Version v8.19" trigger: "integrations" build: env: SERVERLESS: "false" - SKIP_PUBLISHING: "true" FORCE_CHECK_ALL: "true" - STACK_VERSION: 8.12-SNAPSHOT + STACK_VERSION: 8.19.20-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "true" depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_8_BRANCH') == "true" + + - label: "Check integrations local stacks - Stack Version v8.19 - LogsDB" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 8.19.20-SNAPSHOT + STACK_LOGSDB_ENABLED: "true" + PUBLISH_COVERAGE_REPORTS: "false" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('TEST_PACKAGES_8_BRANCH') == "true" + + - label: "Check integrations local stacks and basic subscription and LogsDB" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_SUBSCRIPTION: "basic" + STACK_LOGSDB_ENABLED: "true" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" + + - label: "Check integrations local stacks and basic subscription" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_SUBSCRIPTION: "basic" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" + + - label: "Check integrations local stacks - Stack Version v9.6" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 9.6.0-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "false" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('TEST_PACKAGES_9_BRANCH') == "true" - label: "Check integrations in serverless - project: Observability" key: "trigger-integrations-serverless-obs" @@ -48,6 +109,8 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_SERVERLESS') == "true" - label: "Check integrations in serverless - project: Security" key: "trigger-integrations-serverless-security" @@ -58,3 +121,14 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_SERVERLESS') == "true" + + - label: ":package: Publish missing packages" + key: "trigger-integrations-publish" + trigger: "integrations-publish" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('REPUBLISH_PACKAGES') == "true" diff --git a/.buildkite/pipeline.schedule-weekly.yml b/.buildkite/pipeline.schedule-weekly.yml new file mode 100644 index 00000000000..a2a0666b312 --- /dev/null +++ b/.buildkite/pipeline.schedule-weekly.yml @@ -0,0 +1,42 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: integrations-schedule-weekly + +env: + SETUP_GVM_VERSION: "v0.6.0" + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + +# The pipeline is triggered by the scheduler every week +steps: + - label: ":white_check_mark: Check go sources" + key: "check" + command: ".buildkite/scripts/check_sources.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" + + - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v8.19" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 8.19.20-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true" + depends_on: + - step: "check" + allow_failure: false + + - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.6" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 9.6.0-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true" + depends_on: + - step: "check" + allow_failure: false diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml index 4f7ecf8a855..9a405774a12 100644 --- a/.buildkite/pipeline.serverless.yml +++ b/.buildkite/pipeline.serverless.yml @@ -1,22 +1,35 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - SETUP_GVM_VERSION: "v0.5.1" + SETUP_GVM_VERSION: "v0.6.0" LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" - DOCKER_COMPOSE_VERSION: "v2.17.2" - KIND_VERSION: 'v0.20.0' - K8S_VERSION: 'v1.28.0' + DOCKER_COMPOSE_VERSION: "v2.24.1" + DOCKER_VERSION: "false" # not required to set since system tests are not running yet + KIND_VERSION: 'v0.32.0' + K8S_VERSION: 'v1.36.1' YQ_VERSION: 'v4.35.2' + IMAGE_UBUNTU_X86_64: "family/platform-obs-integrations-ubuntu-2204" + GH_CLI_VERSION: "2.29.0" + # This pipeline is intended to test packages with Elastic Serverless + SERVERLESS: true + # Elastic package settings # Manage docker output/logs - ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" - ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" + ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true" # Default license to use by `elastic-package build` ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt" # Link definitions path (full path to be set in the corresponding step) ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml" # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true" + # Enable independent Elastic Agents for all packages + ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true" + # Set maximum number of parallel tests to run if package allows it + ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5" + # Enable/Disable the usage of wolfi images for Elastic Agent + ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}" + # Disable checking for newer versions + ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true" steps: - input: "Input values for the variables" @@ -47,13 +60,28 @@ steps: - label: "Check integrations in serverless" key: "test-integrations-serverless-project" command: ".buildkite/scripts/test_integrations_with_serverless.sh" - timeout_in_minutes: 120 + timeout_in_minutes: 360 env: - SERVERLESS: true FORCE_CHECK_ALL: true UPLOAD_SAFE_LOGS: 1 agents: provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + plugins: + # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf + # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID) + - elastic/oblt-aws-auth#v0.1.0: + duration: 21600 # seconds + # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf + # This plugin authenticates to CI Google Cloud using the OIDC token. + - elastic/oblt-google-auth#v1.3.0: + lifetime: 21600 # seconds + project-id: "elastic-observability-ci" + project-number: "911195782929" + - avaly/gcp-secret-manager#v1.2.0: + env: + EC_API_KEY: elastic-cloud-observability-team-qa-api-key + EC_HOST: elastic-cloud-observability-team-qa-endpoint artifact_paths: - "build/test-results/*.xml" - "build/elastic-stack-dump/*/logs/*.log" @@ -66,8 +94,30 @@ steps: continue_on_failure: true - label: ":junit: Junit annotate" + agents: + # requires at least "bash", "curl" and "git" + image: "docker.elastic.co/ci-agent-images/buildkite-junit-annotate:1.0" plugins: - - junit-annotate#v2.4.1: + - junit-annotate#v2.7.0: artifacts: "build/test-results/*.xml" + failed-download-exit-code: 0 # Not fail the build in case there are no XML files + report-skipped: true + always-annotate: true + run-in-docker: false + + - label: ":github: Report failed tests" + key: report-failed-tests + command: ".buildkite/scripts/report_issues.sh" + env: + CI_MAX_TESTS_REPORTED: 30 agents: - provider: "gcp" # junit plugin requires docker + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" + plugins: + - elastic/vault-github-token#v0.1.0: + # not fail build if this step fails + soft_fail: true + # run this step when if it is triggered by the daily job + if: > + build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2d88648077e..24585203cd1 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,26 +1,56 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json - env: - SETUP_GVM_VERSION: "v0.5.1" - LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" - DOCKER_COMPOSE_VERSION: "v2.17.2" + SETUP_GVM_VERSION: "v0.6.0" + DOCKER_COMPOSE_VERSION: "v2.24.1" + DOCKER_VERSION: "26.1.2" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.28.0' YQ_VERSION: 'v4.35.2' JQ_VERSION: '1.7' GH_CLI_VERSION: "2.29.0" + CHANGELOG_SKIP_LABEL: "changelog-link-check:skip" + + # Agent images used in pipeline steps + LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" + IMAGE_UBUNTU_X86_64: "family/platform-obs-integrations-ubuntu-2204" + # Elastic package settings # Manage docker output/logs - ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" - ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" + ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true" # Default license to use by `elastic-package build` ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt" # Link definitions path (full path to be set in the corresponding step) ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml" # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true" + # Enable independent Elastic Agents for all packages + ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true" + # Set maximum number of parallel tests to run if package allows it + ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5" + # Enable/Disable the usage of wolfi images for Elastic Agent + ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}" + # Disable checking for newer versions + ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true" + # Customize Service Test Run ID used by elastic-package + # It requires to be customized in .buildkite/hooks/pre-command + # Defining here in this file an environment variable like this: + # ELASTIC_PACKAGE_CUSTOMIZE_SERVICE_TEST_RUN_ID: "${BUILDKITE_STEP_ID: -4}" + # All steps will have the same value, since it always gets the value + # "BUILDKITE_STEP_ID" from the first "Upload pipeline" step. + ELASTIC_PACKAGE_CUSTOMIZE_SERVICE_TEST_RUN_ID: "true" steps: + - label: "Get reference from target branch" + key: "reference-target-branch" + command: "echo 'Get reference from main'" + agents: + image: "${LINUX_AGENT_IMAGE}" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" + + - wait: ~ + continue_on_failure: false + - label: ":white_check_mark: Check go sources" key: "check" command: ".buildkite/scripts/check_sources.sh" @@ -28,22 +58,129 @@ steps: image: "${LINUX_AGENT_IMAGE}" cpu: "8" memory: "4G" + artifact_paths: + - tests-report.xml + + - label: ":bash: :snake: Buildkite scripts unit tests" + key: "check-buildkite-scripts" + command: ".buildkite/scripts/run_buildkite_scripts_tests.sh" + agents: + # GCP VM used to match the same Python 3 version available in the steps that test packages + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" - - label: ":package: Build packages" - key: "build-packages" - command: ".buildkite/scripts/build_packages.sh" + - label: ":bash: Dev scripts unit tests" + key: "check-dev-scripts" + command: ".buildkite/scripts/run_dev_scripts_tests.sh" + + - label: ":golang: Backport tool tests" + key: "check-backport-tool" + command: ".buildkite/scripts/check_backport_tool.sh" agents: image: "${LINUX_AGENT_IMAGE}" - cpu: "8" - memory: "8G" - env: - ARTIFACTS_FOLDER: "artifacts-to-sign" - DRY_RUN: "false" + if_changed: + - "cmd/backport/**" + + - label: ":ballot_box_with_check: Backports inventory validation" + key: "check-backports-inventory" + command: ".buildkite/scripts/check_backports_inventory.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + if_changed: + - ".backports.yml" + - ".buildkite/scripts/run_dev_scripts_tests.sh" + - "dev/scripts/**.sh" + - "cmd/backport/**" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + + - label: ":git: Trigger backport dry-runs" + key: "trigger-backport-dryrun" + command: ".buildkite/scripts/trigger_backport.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + depends_on: + - step: "check-backports-inventory" + allow_failure: false + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + + - label: ":junit: Sources Junit annotate" + agents: + # requires at least "bash", "curl" and "git" + image: "docker.elastic.co/ci-agent-images/buildkite-junit-annotate:1.0" depends_on: - step: "check" + allow_failure: true + plugins: + - junit-annotate#v2.7.0: + artifacts: "tests-report.xml" + failed-download-exit-code: 0 # Not fail the build in case there are no XML files + report-skipped: true + always-annotate: false + run-in-docker: false + context: junit-sources + + - label: ":scroll: Check changelog versions not in main" + key: "check-changelog-versions-in-main" + command: ".buildkite/scripts/check_changelog_versions_in_main.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "2" + memory: "1G" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') =~ /^backport-/ + if_changed: + - "**/changelog.yml" + + - label: ":scroll: Check changelog PR links" + key: "check-changelog-pr-links" + command: ".buildkite/scripts/check_changelog_entries.sh" + soft_fail: + - exit_status: 2 + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "2" + memory: "1G" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "check-buildkite-scripts" allow_failure: false - artifact_paths: - - artifacts-to-sign/*.zip + # CHANGELOG_SKIP_LABEL is substituted into the regex at pipeline upload time. + # This works because the variable is defined in the global env section above; + # step-level env vars are not available at upload time and cannot be used here. + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + !(build.env("GITHUB_PR_LABELS") =~ /${CHANGELOG_SKIP_LABEL}/) + if_changed: + - "**/changelog.yml" + + - label: ":busts_in_silhouette: Check backport package owners" + key: "check-backport-owners" + command: ".buildkite/scripts/check_backport_owners.sh" + soft_fail: true + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "2" + memory: "1G" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "check-buildkite-scripts" + allow_failure: false + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') =~ /^backport-/ + if_changed: + - "packages/**" + - ".github/CODEOWNERS" - label: "Trigger integrations" key: "test-integrations" @@ -53,6 +190,8 @@ steps: depends_on: - step: "check" allow_failure: false + - step: "check-buildkite-scripts" + allow_failure: false - wait: ~ continue_on_failure: true @@ -66,11 +205,39 @@ steps: image: "${LINUX_AGENT_IMAGE}" cpu: "8" memory: "4G" + plugins: + - elastic/vault-github-token#v0.1.0: + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" - label: ":junit: Junit annotate" + agents: + # requires at least "bash", "curl" and "git" + image: "docker.elastic.co/ci-agent-images/buildkite-junit-annotate:1.0" plugins: - - junit-annotate#v2.4.1: + - junit-annotate#v2.7.0: artifacts: "build/test-results/*.xml" failed-download-exit-code: 0 # Not fail the build in case there are no XML files + report-skipped: true + always-annotate: true + run-in-docker: false + + - label: ":github: Report failed tests" + key: report-failed-tests + command: ".buildkite/scripts/report_issues.sh" + env: + CI_MAX_TESTS_REPORTED: 30 agents: - provider: "gcp" # junit plugin requires docker + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" + plugins: + - elastic/vault-github-token#v0.1.0: + # not fail build if this step fails + soft_fail: true + # run this step when if it is triggered by the daily job + if: | + build.source == "trigger_job" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" && + build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index d5981deec5d..e9be54fd341 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipelineSlug": "integrations", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, @@ -13,6 +13,47 @@ "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$|^/test benchmark fullreport$", "skip_ci_labels": [], "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.agents/skills/", + "^.buildkite/pipeline.backport.yml$", + "^.buildkite/pipeline.backport-dispatch.yml$", + "^.buildkite/pipeline.publish.yml$", + "^.buildkite/pipeline.serverless.yml$", + "^.buildkite/pipeline.schedule-daily.yml$", + "^.buildkite/pipeline.schedule-weekly.yml$", + "^.buildkite/pull-requests.json$", + "^.buildkite/scripts/backport_branch.sh$", + "^.buildkite/scripts/build_packages.sh$", + "^.buildkite/scripts/check_backport_tool.sh$", + "^.buildkite/scripts/notify_backport_pr.sh$", + "^.github/dependabot.yml$", + "^.github/workflows/", + "^.github/stale.yml$", + "^.github/ISSUE_TEMPLATE/", + "^.github/PULL_REQUEST_TEMPLATE.md$", + "^.gitignore", + "^.mergify.yml$", + "^catalog-info.yaml$", + "^docs/", + "^CODE_OF_CONDUCT.md$", + "^CONTRIBUTING.md$", + "^README.md$" + ], + "always_require_ci_on_changed": [] + }, + { + "enabled": true, + "pipelineSlug": "integrations-test-stack", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [], + "set_commit_status": true, + "build_on_commit": false, + "build_on_comment": true, + "trigger_comment_regex": "^/test stack (7|8|9)\\.\\d+\\.\\d+(-SNAPSHOT)?$", + "always_trigger_comment_regex": "^/test stack (7|8|9)\\.\\d+\\.\\d+(-SNAPSHOT)?$", + "skip_ci_labels": [], + "skip_target_branches": [], "skip_ci_on_only_changed": [], "always_require_ci_on_changed": [] }, @@ -47,6 +88,38 @@ "skip_target_branches": [], "skip_ci_on_only_changed": [], "always_require_ci_on_changed": [] + }, + { + "enabled": false, + "pipelineSlug": "integrations-backport-dispatch", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [], + "set_commit_status": false, + "build_on_commit": false, + "build_on_comment": false, + "trigger_comment_regex": "", + "always_trigger_comment_regex": "", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [] + }, + { + "enabled": false, + "pipelineSlug": "integrations-publish", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [], + "set_commit_status": false, + "build_on_commit": false, + "build_on_comment": false, + "trigger_comment_regex": "", + "always_trigger_comment_regex": "", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [] } ] } diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh new file mode 100755 index 00000000000..f0b6c0b5fa4 --- /dev/null +++ b/.buildkite/scripts/backport_branch.sh @@ -0,0 +1,385 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh +source "$(dirname "${BASH_SOURCE[0]}")/backport_branch_lib.sh" + +set -euo pipefail + +cleanup_gh() { + pushd "$WORKSPACE" > /dev/null + git config remote.origin.url "https://github.com/elastic/integrations.git" + popd > /dev/null +} + +cleanup() { + local exit_code=$? + cleanup_gh + exit "${exit_code}" +} + +trap cleanup EXIT + +# annotate_and_echo posts a Buildkite annotation and echoes the same message +# to the build log so it is visible in both the annotation panel and the raw output. +# Usage: annotate_and_echo