From e2ae6c1d349ea56425526e0acc107b31c2f2c52a Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:07:13 -0500 Subject: [PATCH 01/42] test/centralized-bk --- .buildkite/hooks/pre-command | 25 ------- .buildkite/pipeline.yml | 91 +++++------------------- .buildkite/scripts/run_models.sh | 25 ------- integration_tests/ci/sample.profiles.yml | 50 ------------- integration_tests/ci/test_scenarios.yml | 16 +++++ 5 files changed, 35 insertions(+), 172 deletions(-) delete mode 100644 .buildkite/hooks/pre-command delete mode 100644 .buildkite/scripts/run_models.sh delete mode 100644 integration_tests/ci/sample.profiles.yml create mode 100644 integration_tests/ci/test_scenarios.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index 81b7b00..0000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2603602..3347df1 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,74 +1,21 @@ -steps: - - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh postgres - - - label: ":snowflake-db: Run Tests - Snowflake" - key: "run_dbt_snowflake" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" - commands: | - bash .buildkite/scripts/run_models.sh snowflake +# This pipeline uses centralized infrastructure from dbt_package_automations. +# Configure TEST_* flags below to control which warehouses are tested. +# Environment variables and secrets are automatically handled by the central scripts. - - label: ":gcloud: Run Tests - BigQuery" - key: "run_dbt_bigquery" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "GCLOUD_SERVICE_KEY" - commands: | - bash .buildkite/scripts/run_models.sh bigquery +env: + # Configure which warehouse tests to run (true/false) + TEST_POSTGRES: true + TEST_SNOWFLAKE: true + TEST_BIGQUERY: true + TEST_REDSHIFT: true + TEST_DATABRICKS: true + TEST_DATABRICKS_SQL: false + TEST_SQLSERVER: false - - label: ":amazon-redshift: Run Tests - Redshift" - key: "run_dbt_redshift" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh redshift - - - label: ":databricks: Run Tests - Databricks" - key: "run_dbt_databricks" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file +steps: + - label: ":gear: Load Central Pipeline" + key: "load_pipeline" + command: | + # Download and load pipeline from dbt_package_automations + curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ + | buildkite-agent pipeline upload \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index fb75f9b..0000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -dbt seed --target "$db" --full-refresh -dbt run --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh -dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" - -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index e7c86b2..0000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,50 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: amazon_ads_integration_tests_03 - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: amazon_ads_integration_tests_03 - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: amazon_ads_integration_tests_03 - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: amazon_ads_integration_tests_03 - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: amazon_ads_integration_tests_03 - threads: 8 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 0000000..5bb8470 --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,16 @@ +# Test scenarios configuration for Amazon Ads dbt package +# This file defines the different test scenarios to run during CI/CD + +# The dbt variable name that controls which schema to use for testing +schema_variable_name: "amazon_ads_schema" + +# Whether the default scenario should include incremental testing +default_include_incremental: false + +# List of test scenarios to run +test_scenarios: + # Test with portfolio history disabled + - name: "portfolio history disabled" + vars: + amazon_ads__portfolio_history_enabled: false + include_incremental: false \ No newline at end of file From 99e567626d6c08573bf07e5065307dcd1faeda2c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:21:12 -0500 Subject: [PATCH 02/42] move to pre-command --- .buildkite/pipeline.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3347df1..b5e6643 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -12,10 +12,4 @@ env: TEST_DATABRICKS_SQL: false TEST_SQLSERVER: false -steps: - - label: ":gear: Load Central Pipeline" - key: "load_pipeline" - command: | - # Download and load pipeline from dbt_package_automations - curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ - | buildkite-agent pipeline upload \ No newline at end of file +# Pipeline steps will be loaded by pre-command hook \ No newline at end of file From 30f4433e962d40cbd1fce0935f410b7e76c093d2 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:21:27 -0500 Subject: [PATCH 03/42] move to pre-command --- .buildkite/hooks/pre-command | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .buildkite/hooks/pre-command diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 0000000..5bb2555 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,15 @@ +#!/bin/bash + +# Pre-command hook to load centralized pipeline +set -euo pipefail + +# Only run on the first step to load the centralized pipeline +if [[ "${BUILDKITE_STEP_NUMBER:-}" == "1" ]]; then + echo "Loading centralized pipeline from dbt_package_automations..." + + # Download and upload the centralized pipeline definition + curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ + | buildkite-agent pipeline upload + + echo "Central pipeline loaded successfully" +fi \ No newline at end of file From 1be4cdd5b3b12868a633f3505e0e6b11003db851 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:39:34 -0500 Subject: [PATCH 04/42] update pre-command --- .buildkite/hooks/pre-command | 23 +++++++++++++++-------- .buildkite/pipeline.yml | 15 --------------- 2 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5bb2555..1a95348 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -3,13 +3,20 @@ # Pre-command hook to load centralized pipeline set -euo pipefail -# Only run on the first step to load the centralized pipeline -if [[ "${BUILDKITE_STEP_NUMBER:-}" == "1" ]]; then - echo "Loading centralized pipeline from dbt_package_automations..." +echo "Loading centralized pipeline from dbt_package_automations..." - # Download and upload the centralized pipeline definition - curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ - | buildkite-agent pipeline upload +# Export TEST_* variables for central pipeline +export TEST_POSTGRES=true +export TEST_SNOWFLAKE=true +export TEST_BIGQUERY=true +export TEST_REDSHIFT=true +export TEST_DATABRICKS=true +export TEST_DATABRICKS_SQL=false +export TEST_SQLSERVER=false + +# Download and upload the centralized pipeline definition +curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ + | buildkite-agent pipeline upload - echo "Central pipeline loaded successfully" -fi \ No newline at end of file +echo "Central pipeline loaded with TEST_* configuration" + | buildkite-agent pipeline upload diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index b5e6643..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,15 +0,0 @@ -# This pipeline uses centralized infrastructure from dbt_package_automations. -# Configure TEST_* flags below to control which warehouses are tested. -# Environment variables and secrets are automatically handled by the central scripts. - -env: - # Configure which warehouse tests to run (true/false) - TEST_POSTGRES: true - TEST_SNOWFLAKE: true - TEST_BIGQUERY: true - TEST_REDSHIFT: true - TEST_DATABRICKS: true - TEST_DATABRICKS_SQL: false - TEST_SQLSERVER: false - -# Pipeline steps will be loaded by pre-command hook \ No newline at end of file From 7adfef69f3c5ac5a43dfbe0db5b6b3d21b7ccb00 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:42:02 -0500 Subject: [PATCH 05/42] fix --- .buildkite/hooks/pre-command | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 1a95348..1a89106 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -19,4 +19,3 @@ curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/fe | buildkite-agent pipeline upload echo "Central pipeline loaded with TEST_* configuration" - | buildkite-agent pipeline upload From 951bcdc036469ee91c4a3e9406d9ff45a86bdef5 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:50:53 -0500 Subject: [PATCH 06/42] post checkout approach --- .buildkite/hooks/{pre-command => post-checkout} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .buildkite/hooks/{pre-command => post-checkout} (69%) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/post-checkout similarity index 69% rename from .buildkite/hooks/pre-command rename to .buildkite/hooks/post-checkout index 1a89106..66ec10a 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/post-checkout @@ -1,9 +1,9 @@ #!/bin/bash -# Pre-command hook to load centralized pipeline +# Post-checkout hook to configure and load centralized pipeline (runs once per build) set -euo pipefail -echo "Loading centralized pipeline from dbt_package_automations..." +echo "Configuring TEST_* variables and loading centralized pipeline..." # Export TEST_* variables for central pipeline export TEST_POSTGRES=true @@ -18,4 +18,4 @@ export TEST_SQLSERVER=false curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ | buildkite-agent pipeline upload -echo "Central pipeline loaded with TEST_* configuration" +echo "Central pipeline loaded with TEST_* configuration" \ No newline at end of file From 6cce33b2e59eb38494f3fa728382c755d383e4db Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:53:33 -0500 Subject: [PATCH 07/42] try pre-checkout --- .buildkite/hooks/{post-checkout => pre-checkout} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .buildkite/hooks/{post-checkout => pre-checkout} (87%) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/pre-checkout similarity index 87% rename from .buildkite/hooks/post-checkout rename to .buildkite/hooks/pre-checkout index 66ec10a..2da6d1a 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/pre-checkout @@ -1,6 +1,6 @@ #!/bin/bash -# Post-checkout hook to configure and load centralized pipeline (runs once per build) +# Pre-checkout hook to configure and load centralized pipeline (runs before checkout) set -euo pipefail echo "Configuring TEST_* variables and loading centralized pipeline..." From 29a26125eb4cfc2b20b5cb6c55e713d9553a44bb Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:58:06 -0500 Subject: [PATCH 08/42] try again --- .buildkite/hooks/{pre-checkout => post-checkout} | 2 +- .buildkite/pipeline.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) rename .buildkite/hooks/{pre-checkout => post-checkout} (87%) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/hooks/pre-checkout b/.buildkite/hooks/post-checkout similarity index 87% rename from .buildkite/hooks/pre-checkout rename to .buildkite/hooks/post-checkout index 2da6d1a..15699fb 100644 --- a/.buildkite/hooks/pre-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,6 +1,6 @@ #!/bin/bash -# Pre-checkout hook to configure and load centralized pipeline (runs before checkout) +# Post-checkout hook to configure and load centralized pipeline (runs after checkout) set -euo pipefail echo "Configuring TEST_* variables and loading centralized pipeline..." diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..82068bd --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,2 @@ +steps: + - wait \ No newline at end of file From bd70e1743d39fe2698380bec153ca662473594b0 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:14:12 -0500 Subject: [PATCH 09/42] save file first --- .buildkite/hooks/post-checkout | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 15699fb..6d9606c 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -14,8 +14,11 @@ export TEST_DATABRICKS=true export TEST_DATABRICKS_SQL=false export TEST_SQLSERVER=false -# Download and upload the centralized pipeline definition +# Download the centralized pipeline definition curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ - | buildkite-agent pipeline upload + -o .buildkite/pipeline.yml + +# Upload the centralized pipeline definition +buildkite-agent pipeline upload .buildkite/pipeline.yml echo "Central pipeline loaded with TEST_* configuration" \ No newline at end of file From 5a8c68e0a632db09599b4bb5a450b1c5a59f576c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:21:03 -0500 Subject: [PATCH 10/42] download only --- .buildkite/hooks/post-checkout | 5 +---- .buildkite/pipeline.yml | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 6d9606c..049b3b8 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -18,7 +18,4 @@ export TEST_SQLSERVER=false curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ -o .buildkite/pipeline.yml -# Upload the centralized pipeline definition -buildkite-agent pipeline upload .buildkite/pipeline.yml - -echo "Central pipeline loaded with TEST_* configuration" \ No newline at end of file +echo "Centralized pipeline downloaded." \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 82068bd..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,2 +0,0 @@ -steps: - - wait \ No newline at end of file From 52bae2b9c79f86809944c3260823b8115d307c77 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:30:59 -0500 Subject: [PATCH 11/42] test again --- .buildkite/hooks/post-checkout | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 049b3b8..afa796e 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,17 +5,8 @@ set -euo pipefail echo "Configuring TEST_* variables and loading centralized pipeline..." -# Export TEST_* variables for central pipeline -export TEST_POSTGRES=true -export TEST_SNOWFLAKE=true -export TEST_BIGQUERY=true -export TEST_REDSHIFT=true -export TEST_DATABRICKS=true -export TEST_DATABRICKS_SQL=false -export TEST_SQLSERVER=false - -# Download the centralized pipeline definition +# Download the centralized pipeline (now with hardcoded conditions) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ -o .buildkite/pipeline.yml -echo "Centralized pipeline downloaded." \ No newline at end of file +echo "Centralized pipeline loaded." \ No newline at end of file From 51d5ccc489009128eaed09ad256c494b2df71385 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:49:21 -0500 Subject: [PATCH 12/42] add warehouse control --- integration_tests/ci/test_scenarios.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 5bb8470..19b25ec 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -1,6 +1,16 @@ # Test scenarios configuration for Amazon Ads dbt package # This file defines the different test scenarios to run during CI/CD +# Warehouse testing configuration +warehouses: + postgres: true + snowflake: false + bigquery: false + redshift: false + databricks: false + databricks_sql: false + sqlserver: false + # The dbt variable name that controls which schema to use for testing schema_variable_name: "amazon_ads_schema" From 9d212a6e4a1fed57b09252696df960141ed6f8df Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:01:27 -0500 Subject: [PATCH 13/42] test --- .buildkite/hooks/post-checkout | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index afa796e..f1d5d02 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -3,10 +3,30 @@ # Post-checkout hook to configure and load centralized pipeline (runs after checkout) set -euo pipefail -echo "Configuring TEST_* variables and loading centralized pipeline..." +echo "Configuring central pipeline" -# Download the centralized pipeline (now with hardcoded conditions) +# Download the central pipeline template curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ - -o .buildkite/pipeline.yml + -o .buildkite/pipeline_base.yml -echo "Centralized pipeline loaded." \ No newline at end of file +# Create pipeline with merged env section +cat > .buildkite/pipeline.yml << EOF +env: + RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/centralized-bk/.buildkite/scripts/run_models.sh" + TEST_POSTGRES: true + TEST_SNOWFLAKE: false + TEST_BIGQUERY: false + TEST_REDSHIFT: false + TEST_DATABRICKS: false + TEST_DATABRICKS_SQL: false + TEST_SQLSERVER: false + +EOF + +# Append the steps section (skip the env section) +tail -n +4 .buildkite/pipeline_base.yml >> .buildkite/pipeline.yml + +# Clean up temp file +rm -f .buildkite/pipeline_base.yml + +echo "Central pipeline configured." \ No newline at end of file From 2a3c31ab166048c6566f033b791486e6daf66b06 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:07:25 -0500 Subject: [PATCH 14/42] test --- .buildkite/hooks/post-checkout | 39 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index f1d5d02..733e8f1 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -3,30 +3,19 @@ # Post-checkout hook to configure and load centralized pipeline (runs after checkout) set -euo pipefail -echo "Configuring central pipeline" - -# Download the central pipeline template +echo "Loading pipeline.yml..." + +# Set environment variables for warehouse testing using Buildkite agent +buildkite-agent env set TEST_POSTGRES true +buildkite-agent env set TEST_SNOWFLAKE false +buildkite-agent env set TEST_BIGQUERY false +buildkite-agent env set TEST_REDSHIFT false +buildkite-agent env set TEST_DATABRICKS false +buildkite-agent env set TEST_DATABRICKS_SQL false +buildkite-agent env set TEST_SQLSERVER false + +# Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ - -o .buildkite/pipeline_base.yml - -# Create pipeline with merged env section -cat > .buildkite/pipeline.yml << EOF -env: - RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/centralized-bk/.buildkite/scripts/run_models.sh" - TEST_POSTGRES: true - TEST_SNOWFLAKE: false - TEST_BIGQUERY: false - TEST_REDSHIFT: false - TEST_DATABRICKS: false - TEST_DATABRICKS_SQL: false - TEST_SQLSERVER: false - -EOF - -# Append the steps section (skip the env section) -tail -n +4 .buildkite/pipeline_base.yml >> .buildkite/pipeline.yml - -# Clean up temp file -rm -f .buildkite/pipeline_base.yml + -o .buildkite/pipeline.yml -echo "Central pipeline configured." \ No newline at end of file +echo "pipeline.yml loaded successfully." \ No newline at end of file From 4a37298946194f124e1733aa1147592a38950759 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:09:41 -0500 Subject: [PATCH 15/42] test --- .buildkite/hooks/post-checkout | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 733e8f1..edde2e6 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -6,13 +6,13 @@ set -euo pipefail echo "Loading pipeline.yml..." # Set environment variables for warehouse testing using Buildkite agent -buildkite-agent env set TEST_POSTGRES true -buildkite-agent env set TEST_SNOWFLAKE false -buildkite-agent env set TEST_BIGQUERY false -buildkite-agent env set TEST_REDSHIFT false -buildkite-agent env set TEST_DATABRICKS false -buildkite-agent env set TEST_DATABRICKS_SQL false -buildkite-agent env set TEST_SQLSERVER false +buildkite-agent env set TEST_POSTGRES=true +buildkite-agent env set TEST_SNOWFLAKE=false +buildkite-agent env set TEST_BIGQUERY=false +buildkite-agent env set TEST_REDSHIFT=false +buildkite-agent env set TEST_DATABRICKS=false +buildkite-agent env set TEST_DATABRICKS_SQL=false +buildkite-agent env set TEST_SQLSERVER=false # Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ From 5199b0b7a1874addfb77bdedf9c2fafc4265455c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:12:59 -0500 Subject: [PATCH 16/42] test --- .buildkite/hooks/post-checkout | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index edde2e6..c855ef1 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,14 +5,8 @@ set -euo pipefail echo "Loading pipeline.yml..." -# Set environment variables for warehouse testing using Buildkite agent -buildkite-agent env set TEST_POSTGRES=true -buildkite-agent env set TEST_SNOWFLAKE=false -buildkite-agent env set TEST_BIGQUERY=false -buildkite-agent env set TEST_REDSHIFT=false -buildkite-agent env set TEST_DATABRICKS=false -buildkite-agent env set TEST_DATABRICKS_SQL=false -buildkite-agent env set TEST_SQLSERVER=false +# Set enabled warehouses for this package +buildkite-agent env set ENABLED_WAREHOUSES=postgres,snowflake # Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ From a99ad5393f2d698cbf258228adf4f0a5ee256664 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:40:24 -0500 Subject: [PATCH 17/42] test --- .buildkite/hooks/post-checkout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index c855ef1..39e64ed 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -6,7 +6,7 @@ set -euo pipefail echo "Loading pipeline.yml..." # Set enabled warehouses for this package -buildkite-agent env set ENABLED_WAREHOUSES=postgres,snowflake +buildkite-agent env set ENABLED_WAREHOUSES="postgres" # Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ From d2c68cf1d24311fa7a501433a5be2d8e9fc0343a Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:48:29 -0500 Subject: [PATCH 18/42] test --- .buildkite/hooks/post-checkout | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 39e64ed..a3306a1 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,8 +5,8 @@ set -euo pipefail echo "Loading pipeline.yml..." -# Set enabled warehouses for this package -buildkite-agent env set ENABLED_WAREHOUSES="postgres" +# Set enabled warehouses for this package (as array) +buildkite-agent env set ENABLED_WAREHOUSES='["postgres", "snowflake"]' # Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ From f43c7ef23390bdf12edef900cca47d3c69952748 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:48:41 -0500 Subject: [PATCH 19/42] test --- .buildkite/hooks/post-checkout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index a3306a1..c0b52eb 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -6,7 +6,7 @@ set -euo pipefail echo "Loading pipeline.yml..." # Set enabled warehouses for this package (as array) -buildkite-agent env set ENABLED_WAREHOUSES='["postgres", "snowflake"]' +buildkite-agent env set ENABLED_WAREHOUSES=["postgres", "snowflake"] # Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ From ee4d5de42536aca6dba5ba2491738c5774bf7e61 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:49:50 -0500 Subject: [PATCH 20/42] test --- .buildkite/hooks/post-checkout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index c0b52eb..a3306a1 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -6,7 +6,7 @@ set -euo pipefail echo "Loading pipeline.yml..." # Set enabled warehouses for this package (as array) -buildkite-agent env set ENABLED_WAREHOUSES=["postgres", "snowflake"] +buildkite-agent env set ENABLED_WAREHOUSES='["postgres", "snowflake"]' # Download the central pipeline (use as-is) curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ From 6f8b38a22192b551e0eaaf4d178d6bff5871ef70 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:05:22 -0500 Subject: [PATCH 21/42] test --- .buildkite/hooks/post-checkout | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index a3306a1..b483e8e 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -6,10 +6,11 @@ set -euo pipefail echo "Loading pipeline.yml..." # Set enabled warehouses for this package (as array) -buildkite-agent env set ENABLED_WAREHOUSES='["postgres", "snowflake"]' +buildkite-agent env set ENABLED_WAREHOUSES="postgres snowflake" -# Download the central pipeline (use as-is) -curl -f -s https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/pipeline.yml \ - -o .buildkite/pipeline.yml +# Download the central pipeline generator +curl -fsSL \ + https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/bootstrap_pipeline.sh \ + | bash echo "pipeline.yml loaded successfully." \ No newline at end of file From e438e8aa5c8c3a5ecd13b6324b5a315da01fb7ad Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:08:33 -0500 Subject: [PATCH 22/42] test --- .buildkite/hooks/post-checkout | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index b483e8e..b2c4faa 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -3,14 +3,12 @@ # Post-checkout hook to configure and load centralized pipeline (runs after checkout) set -euo pipefail -echo "Loading pipeline.yml..." +echo "Generating centralized pipeline..." -# Set enabled warehouses for this package (as array) -buildkite-agent env set ENABLED_WAREHOUSES="postgres snowflake" +buildkite-agent env set ENABLED_WAREHOUSES "postgres snowflake" -# Download the central pipeline generator curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/bootstrap_pipeline.sh \ | bash -echo "pipeline.yml loaded successfully." \ No newline at end of file +echo "Centralized pipeline generated." \ No newline at end of file From c3f63e5ff384bb4b56064995b08921737a73159e Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:14:35 -0500 Subject: [PATCH 23/42] test --- .buildkite/hooks/post-checkout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index b2c4faa..e20c225 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,7 +5,7 @@ set -euo pipefail echo "Generating centralized pipeline..." -buildkite-agent env set ENABLED_WAREHOUSES "postgres snowflake" +buildkite-agent env set "ENABLED_WAREHOUSES='postgres snowflake'" curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/bootstrap_pipeline.sh \ From 3d99ad0c1c56c55d7d03976811f4adb6f0077e7b Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:16:43 -0500 Subject: [PATCH 24/42] test --- .buildkite/hooks/post-checkout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index e20c225..648603f 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,7 +5,7 @@ set -euo pipefail echo "Generating centralized pipeline..." -buildkite-agent env set "ENABLED_WAREHOUSES='postgres snowflake'" +buildkite-agent env set "ENABLED_WAREHOUSES=postgres snowflake" curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/bootstrap_pipeline.sh \ From 59f98f13f211a71d9e756f89c47a27ecd1dbae1c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:20:15 -0500 Subject: [PATCH 25/42] test --- .buildkite/hooks/post-checkout | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 648603f..8c16774 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,10 +5,8 @@ set -euo pipefail echo "Generating centralized pipeline..." -buildkite-agent env set "ENABLED_WAREHOUSES=postgres snowflake" - curl -fsSL \ - https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/bootstrap_pipeline.sh \ - | bash + https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/dynamic_pipeline.sh \ + | ENABLED_WAREHOUSES="postgres snowflake" bash echo "Centralized pipeline generated." \ No newline at end of file From 000a468fda889765a3f6cef6a1b3b433e62c9aa9 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:42:31 -0500 Subject: [PATCH 26/42] test --- .buildkite/hooks/post-checkout | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 8c16774..9bf9a31 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -5,8 +5,14 @@ set -euo pipefail echo "Generating centralized pipeline..." +# Download pre-command script for environment setup +curl -fsSL \ + https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/setup_vars.sh \ + -o .buildkite/hooks/pre-command + +# Available warehouse options: postgres, snowflake, bigquery, redshift, databricks, databricks_sql, sqlserver curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/dynamic_pipeline.sh \ - | ENABLED_WAREHOUSES="postgres snowflake" bash + | ENABLED_WAREHOUSES="postgres,snowflake" bash echo "Centralized pipeline generated." \ No newline at end of file From 1086718734c3a034a92bc929fdd73414449c68bf Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:52:30 -0500 Subject: [PATCH 27/42] test --- .buildkite/hooks/post-checkout | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 9bf9a31..314336d 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -10,9 +10,10 @@ curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/setup_vars.sh \ -o .buildkite/hooks/pre-command -# Available warehouse options: postgres, snowflake, bigquery, redshift, databricks, databricks_sql, sqlserver +# Always runs: postgres, snowflake, bigquery, redshift, databricks +# Optional flags: INCLUDE_DATABRICKS_SQL=true, INCLUDE_SQLSERVER=true curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/dynamic_pipeline.sh \ - | ENABLED_WAREHOUSES="postgres,snowflake" bash + | INCLUDE_DATABRICKS_SQL=true INCLUDE_SQLSERVER=false bash echo "Centralized pipeline generated." \ No newline at end of file From 30610c124613d72fa90164c2ed7a37a5fdb98143 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:56:15 -0500 Subject: [PATCH 28/42] test --- .buildkite/hooks/post-checkout | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 314336d..67901c9 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -3,7 +3,7 @@ # Post-checkout hook to configure and load centralized pipeline (runs after checkout) set -euo pipefail -echo "Generating centralized pipeline..." +echo "Generating pipeline..." # Download pre-command script for environment setup curl -fsSL \ @@ -16,4 +16,4 @@ curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/dynamic_pipeline.sh \ | INCLUDE_DATABRICKS_SQL=true INCLUDE_SQLSERVER=false bash -echo "Centralized pipeline generated." \ No newline at end of file +echo "Pipeline generated." \ No newline at end of file From a957e6037f9f91d9ef46b90daf586b844489d82e Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:09:24 -0500 Subject: [PATCH 29/42] Remove warehouse testing configuration Removed warehouse testing configuration from CI test scenarios. --- integration_tests/ci/test_scenarios.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 19b25ec..bcdff74 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -1,16 +1,6 @@ # Test scenarios configuration for Amazon Ads dbt package # This file defines the different test scenarios to run during CI/CD -# Warehouse testing configuration -warehouses: - postgres: true - snowflake: false - bigquery: false - redshift: false - databricks: false - databricks_sql: false - sqlserver: false - # The dbt variable name that controls which schema to use for testing schema_variable_name: "amazon_ads_schema" @@ -23,4 +13,4 @@ test_scenarios: - name: "portfolio history disabled" vars: amazon_ads__portfolio_history_enabled: false - include_incremental: false \ No newline at end of file + include_incremental: false From 07ddc78860c7f97b370a4fcd24fd556372ce93a7 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:10:54 -0500 Subject: [PATCH 30/42] remove sql warehouse --- .buildkite/hooks/post-checkout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 67901c9..bc8cf9c 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -14,6 +14,6 @@ curl -fsSL \ # Optional flags: INCLUDE_DATABRICKS_SQL=true, INCLUDE_SQLSERVER=true curl -fsSL \ https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/dynamic_pipeline.sh \ - | INCLUDE_DATABRICKS_SQL=true INCLUDE_SQLSERVER=false bash + | INCLUDE_DATABRICKS_SQL=false INCLUDE_SQLSERVER=false bash echo "Pipeline generated." \ No newline at end of file From 5b9fe5f2365c3c9b34177c76bb9e6d3893e76b93 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:36:55 -0500 Subject: [PATCH 31/42] yml update --- integration_tests/dbt_project.yml | 65 ++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4b069ae..d7c6c7a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: amazon_ads_integration_tests_03 + amazon_ads_schema: amazon_ads_integration_tests amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" @@ -41,14 +41,61 @@ seeds: show: False +quote_columns: "{{ true if target.type == 'redshift' else false }}" amazon_ads_integration_tests: - +column_types: - _fivetran_synced: "timestamp" - campaign_budget_amount: "float" - click_through_rate: "float" - keyword_bid: "float" - clicks: "float" - impressions: "float" - cost: "float" + ad_group_level_report_data: + +column_types: + _fivetran_synced: "timestamp" + clicks: "float" + impressions: "float" + cost: "float" + advertised_product_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + clicks: "float" + impressions: "float" + cost: "float" + campaign_level_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + clicks: "float" + impressions: "float" + cost: "float" + keyword_history_data: + +column_types: + _fivetran_synced: "timestamp" + portfolio_history_data: + +column_types: + _fivetran_synced: "timestamp" + product_ad_history_data: + +column_types: + _fivetran_synced: "timestamp" + profile_data: + +column_types: + _fivetran_synced: "timestamp" + search_term_ad_keyword_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + clicks: "float" + impressions: "float" + cost: "float" + keyword_bid: "float" + targeting_keyword_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + clicks: "float" + impressions: "float" + cost: "float" + keyword_bid: "float" + campaign_history_data: + +column_types: + _fivetran_synced: "timestamp" + ad_group_history_data: + +column_types: + _fivetran_synced: "timestamp" flags: send_anonymous_usage_stats: False use_colors: True + \ No newline at end of file From 46ee6ec43dae035f391b29c81020be438462318d Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:58:19 -0500 Subject: [PATCH 32/42] test --- .buildkite/hooks/post-checkout | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .buildkite/hooks/post-checkout diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout deleted file mode 100644 index bc8cf9c..0000000 --- a/.buildkite/hooks/post-checkout +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Post-checkout hook to configure and load centralized pipeline (runs after checkout) -set -euo pipefail - -echo "Generating pipeline..." - -# Download pre-command script for environment setup -curl -fsSL \ - https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/setup_vars.sh \ - -o .buildkite/hooks/pre-command - -# Always runs: postgres, snowflake, bigquery, redshift, databricks -# Optional flags: INCLUDE_DATABRICKS_SQL=true, INCLUDE_SQLSERVER=true -curl -fsSL \ - https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/dynamic_pipeline.sh \ - | INCLUDE_DATABRICKS_SQL=false INCLUDE_SQLSERVER=false bash - -echo "Pipeline generated." \ No newline at end of file From 4ec370d5a5a8dfd8a27b2acefd2becb6bbb2219e Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:58:29 -0500 Subject: [PATCH 33/42] test --- .buildkite/pipeline.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..bbdc97b --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,9 @@ +env: + INCLUDE_DATABRICKS_SQL: "true" + INCLUDE_SQLSERVER: "false" + +steps: + - label: ":rocket: Run CI Tests" + key: "super_run" + commands: | + curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/SUPER_RUN.sh | bash \ No newline at end of file From 305d51ee3bdb173656f97289c78f6bf26484f337 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:00:26 -0500 Subject: [PATCH 34/42] fix branch name --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index bbdc97b..b3dc9d5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,4 +6,4 @@ steps: - label: ":rocket: Run CI Tests" key: "super_run" commands: | - curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/feature/centralized-bk/.buildkite/scripts/SUPER_RUN.sh | bash \ No newline at end of file + curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/test/superscript/.buildkite/scripts/SUPER_RUN.sh | bash \ No newline at end of file From 80db2ad178deeccfe176f879b8e310f1e56840c1 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:26:39 -0500 Subject: [PATCH 35/42] move warehouse to test_Scenarios --- .buildkite/pipeline.yml | 15 ++++++++++----- integration_tests/ci/test_scenarios.yml | 3 +++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b3dc9d5..051ae08 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,9 +1,14 @@ -env: - INCLUDE_DATABRICKS_SQL: "true" - INCLUDE_SQLSERVER: "false" - steps: - label: ":rocket: Run CI Tests" key: "super_run" commands: | - curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/test/superscript/.buildkite/scripts/SUPER_RUN.sh | bash \ No newline at end of file + echo "🚀 Downloading and running SUPER_RUN.sh..." + if ! curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/test/superscript/.buildkite/scripts/SUPER_RUN.sh -o SUPER_RUN.sh; then + echo "❌ Failed to download SUPER_RUN.sh" + exit 1 + fi + if [[ ! -s SUPER_RUN.sh ]]; then + echo "❌ Downloaded SUPER_RUN.sh is empty" + exit 1 + fi + bash SUPER_RUN.sh \ No newline at end of file diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index bcdff74..53a25f8 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -4,6 +4,9 @@ # The dbt variable name that controls which schema to use for testing schema_variable_name: "amazon_ads_schema" +# Optional warehouse configuration (only specify if needed) +include_databricks_sql: true + # Whether the default scenario should include incremental testing default_include_incremental: false From 13c4ef172457d97844615716b1ec783371a1f4c6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:11:40 -0500 Subject: [PATCH 36/42] simplify --- .buildkite/pipeline.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 051ae08..ab71c47 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,13 +2,6 @@ steps: - label: ":rocket: Run CI Tests" key: "super_run" commands: | - echo "🚀 Downloading and running SUPER_RUN.sh..." - if ! curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/test/superscript/.buildkite/scripts/SUPER_RUN.sh -o SUPER_RUN.sh; then - echo "❌ Failed to download SUPER_RUN.sh" - exit 1 - fi - if [[ ! -s SUPER_RUN.sh ]]; then - echo "❌ Downloaded SUPER_RUN.sh is empty" - exit 1 - fi + set -e + curl -fsSL https://raw.githubusercontent.com/fivetran/dbt_package_automations/test/superscript/.buildkite/scripts/SUPER_RUN.sh -o SUPER_RUN.sh bash SUPER_RUN.sh \ No newline at end of file From 6ff3306f11f4b2dd4db75f5396f72ec2632d25af Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:20:42 -0500 Subject: [PATCH 37/42] simplify --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ab71c47..b67427d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ steps: - - label: ":rocket: Run CI Tests" + - label: ":rocket: Run CI Tests" key: "super_run" commands: | set -e From 61cd3fc7874d6b6303868cb1651428446382193c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:59:02 -0500 Subject: [PATCH 38/42] format --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b67427d..ab71c47 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ steps: - - label: ":rocket: Run CI Tests" + - label: ":rocket: Run CI Tests" key: "super_run" commands: | set -e From 6b174c9ee5fb3d7eec405aa524373e0486ccc674 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:51:40 -0500 Subject: [PATCH 39/42] formating --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ab71c47..0d876a9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ steps: - - label: ":rocket: Run CI Tests" + - label: ":building_construction: Setup CI Tests" key: "super_run" commands: | set -e From 09b9d56257c81790cb5444625f5cde4e0b7ee2d7 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 28 Apr 2026 13:47:38 -0500 Subject: [PATCH 40/42] test adapter version --- integration_tests/ci/test_scenarios.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 53a25f8..6f5b071 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -10,6 +10,14 @@ include_databricks_sql: true # Whether the default scenario should include incremental testing default_include_incremental: false +# Dbt adapter versions (optional - defaults will be used if not specified) +dbt_adapter_versions: + snowflake: ">=1.3.0,<1.6.0" + bigquery: ">=1.3.0,<2.0.0" + postgres: ">=1.3.0,<2.0.0" + redshift: ">=1.3.0,<2.0.0" + databricks: ">=1.3.0,<2.0.0" + # List of test scenarios to run test_scenarios: # Test with portfolio history disabled From 8e2f73cb1df41716d5c746333d1e86d04823b151 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:08:22 -0500 Subject: [PATCH 41/42] include default test scenario --- integration_tests/ci/test_scenarios.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 6f5b071..0a69d59 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -7,9 +7,6 @@ schema_variable_name: "amazon_ads_schema" # Optional warehouse configuration (only specify if needed) include_databricks_sql: true -# Whether the default scenario should include incremental testing -default_include_incremental: false - # Dbt adapter versions (optional - defaults will be used if not specified) dbt_adapter_versions: snowflake: ">=1.3.0,<1.6.0" @@ -20,7 +17,9 @@ dbt_adapter_versions: # List of test scenarios to run test_scenarios: - # Test with portfolio history disabled + - name: "default" + vars: {} + include_incremental: false - name: "portfolio history disabled" vars: amazon_ads__portfolio_history_enabled: false From 03601235c74496e1b746554795f244253909ba72 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:11:48 -0500 Subject: [PATCH 42/42] test default --- integration_tests/ci/test_scenarios.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 0a69d59..6a6d0bf 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -9,7 +9,7 @@ include_databricks_sql: true # Dbt adapter versions (optional - defaults will be used if not specified) dbt_adapter_versions: - snowflake: ">=1.3.0,<1.6.0" + snowflake: ">=1.3.0,<2.0.0" bigquery: ">=1.3.0,<2.0.0" postgres: ">=1.3.0,<2.0.0" redshift: ">=1.3.0,<2.0.0" @@ -19,7 +19,7 @@ dbt_adapter_versions: test_scenarios: - name: "default" vars: {} - include_incremental: false + include_incremental: true - name: "portfolio history disabled" vars: amazon_ads__portfolio_history_enabled: false