Buildkite improvements FY27 Q2

.buildkite/hooks/pre-command

@@ -2,6 +2,17 @@
 
 set -e
 
+# Generate unique schema name for this build
+# Format: zz_bk_integration_tests_<commit_sha>_<build_number>
+BUILD_NUMBER=$BUILDKITE_BUILD_NUMBER
+COMMIT_SHA=${BUILDKITE_COMMIT:0:7}
+PREFIX="zz_bk_integration_tests"
+if [[ "$BUILDKITE_STEP_KEY" == "run_dbt_databricks_sql" ]]; then PREFIX="${PREFIX}_sql"; fi
+BUILD_SCHEMA="${PREFIX}_${COMMIT_SHA}_${BUILD_NUMBER}"
+
+export BUILD_SCHEMA
+echo "Build schema: ${BUILD_SCHEMA}"
+
 # Export secrets for Docker containers.
 # Restrict exposing secrets only to the steps that need them
 export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
@@ -22,4 +33,4 @@ export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secr
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -1,74 +1,104 @@
+env:
+  RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh"
+
 steps:
   - label: ":postgres: Run Tests - Postgres"
-    key: "run-dbt-postgres"
+    key: "run_dbt_postgres"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_POSTGRES_DBT_DBNAME"
             - "CI_POSTGRES_DBT_HOST"
             - "CI_POSTGRES_DBT_PASS"
             - "CI_POSTGRES_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh postgres 
+      curl -s "${RUN_MODELS_URL}" | bash -s postgres 
 
   - label: ":snowflake-db: Run Tests - Snowflake"
     key: "run_dbt_snowflake"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_SNOWFLAKE_DBT_ACCOUNT"
             - "CI_SNOWFLAKE_DBT_DATABASE"
             - "CI_SNOWFLAKE_DBT_PASS"
             - "CI_SNOWFLAKE_DBT_ROLE"
             - "CI_SNOWFLAKE_DBT_USER"
             - "CI_SNOWFLAKE_DBT_WAREHOUSE"
     commands: |
-      bash .buildkite/scripts/run_models.sh snowflake
+      curl -s "${RUN_MODELS_URL}" | bash -s snowflake
 
   - label: ":gcloud: Run Tests - BigQuery"
     key: "run_dbt_bigquery"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "GCLOUD_SERVICE_KEY"
     commands: |
-      bash .buildkite/scripts/run_models.sh bigquery
+      curl -s "${RUN_MODELS_URL}" | bash -s bigquery
 
   - label: ":amazon-redshift: Run Tests - Redshift"
     key: "run_dbt_redshift"
+    concurrency: 3
+    concurrency_group: "warehouse/redshift"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_REDSHIFT_DBT_DBNAME"
             - "CI_REDSHIFT_DBT_HOST"
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      curl -s "${RUN_MODELS_URL}" | bash -s redshift
 
   - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
             - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
-      bash .buildkite/scripts/run_models.sh databricks
+      curl -s "${RUN_MODELS_URL}" | bash -s databricks

integration_tests/ci/test_scenarios.yml

@@ -0,0 +1,8 @@
+schema_variable_name: "amazon_ads_schema"
+default_include_incremental: false
+
+test_scenarios:
+  - name: "portfolio history disabled"
+    vars:
+      amazon_ads__portfolio_history_enabled: false
+    include_incremental: false

integration_tests/dbt_project.yml

@@ -7,7 +7,7 @@ profile: 'integration_tests'
 
 vars:
   amazon_ads:
-    amazon_ads_schema: amazon_ads_integration_tests_03
+    amazon_ads_schema: amazon_ads_integration_tests
     amazon_ads_ad_group_history_identifier: "ad_group_history_data"
     amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data"
     amazon_ads_advertised_product_report_identifier: "advertised_product_report_data"
@@ -41,14 +41,60 @@ seeds:
     show: False
   +quote_columns: "{{ true if target.type == 'redshift' else false }}"
   amazon_ads_integration_tests:
-    +column_types:
-      _fivetran_synced: "timestamp"
-      campaign_budget_amount: "float"
-      click_through_rate: "float"
-      keyword_bid: "float"
-      clicks: "float"
-      impressions: "float"
-      cost: "float"
+    ad_group_level_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    advertised_product_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    campaign_level_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    keyword_history_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+    portfolio_history_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+    product_ad_history_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+    profile_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+    search_term_ad_keyword_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+        keyword_bid: "float"
+    targeting_keyword_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+        keyword_bid: "float"
+    campaign_history_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+    ad_group_history_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
 flags:
   send_anonymous_usage_stats: False
   use_colors: True