From c0115b88507c24b41e0a3fe4b38ad1f36a2f20bb Mon Sep 17 00:00:00 2001 From: Tomasz Date: Wed, 24 Jun 2026 21:49:23 +0200 Subject: [PATCH 1/3] Create singular test --- Makefile | 5 ++++- dbt/gittrends_dbt/tests/assert_no_future_commits.sql | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 dbt/gittrends_dbt/tests/assert_no_future_commits.sql diff --git a/Makefile b/Makefile index f722ff1..20e3caa 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,10 @@ dbt-debug: cd ./dbt/gittrends_dbt && dbt debug dbt-test: - cd ./dbt/gittrends_dbt && dbt test + cd ./dbt/gittrends_dbt && dbt test --select test_type:generic + +dbt-tests-singular: + cd ./dbt/gittrends_dbt && dbt test --select test_type:singular terraform-apply: cd ./terraform && terraform apply diff --git a/dbt/gittrends_dbt/tests/assert_no_future_commits.sql b/dbt/gittrends_dbt/tests/assert_no_future_commits.sql new file mode 100644 index 0000000..8179fe2 --- /dev/null +++ b/dbt/gittrends_dbt/tests/assert_no_future_commits.sql @@ -0,0 +1,6 @@ +SELECT + repo_name, + author, + event_date +FROM {{ ref('fact_daily_commits') }} +WHERE event_date > CURRENT_DATE \ No newline at end of file From 3af32e2beee468c1e98b0b8aed17383033c0544f Mon Sep 17 00:00:00 2001 From: Tomasz Date: Thu, 25 Jun 2026 14:00:15 +0200 Subject: [PATCH 2/3] Fix getenv in databricks --- src/gittrends/databricks/bronze_to_silver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/gittrends/databricks/bronze_to_silver.py b/src/gittrends/databricks/bronze_to_silver.py index c28c240..286a053 100644 --- a/src/gittrends/databricks/bronze_to_silver.py +++ b/src/gittrends/databricks/bronze_to_silver.py @@ -1,8 +1,14 @@ # Databricks notebook source +import os + +from dotenv import load_dotenv from pyspark.sql import DataFrame, SparkSession from pyspark.sql.functions import col, explode_outer, lit, to_timestamp from pyspark.sql.types import StructType +load_dotenv() +BUCKET_NAME = os.getenv("BUCKET_NAME") + def clean_and_flatten_data(df_bronze: DataFrame) -> DataFrame: payload_schema = df_bronze.schema["payload"].dataType @@ -48,7 +54,7 @@ def clean_and_flatten_data(df_bronze: DataFrame) -> DataFrame: def main(): spark = SparkSession.builder.appName("GitHubArchive-BronzeToSilver").getOrCreate() - BUCKET_NAME = "gittrends-data-lake" + BUCKET_NAME = "BUCKET_NAME" BRONZE_PATH = f"s3://{BUCKET_NAME}/bronze/*/*/*/*.json.gz" SILVER_PATH = f"s3://{BUCKET_NAME}/silver/github_events/" From f5703a79289900594af9405ebb88db4a73400cba Mon Sep 17 00:00:00 2001 From: Tomasz Date: Thu, 25 Jun 2026 14:06:11 +0200 Subject: [PATCH 3/3] Fix makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 20e3caa..bef23d2 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ install-dbt: upgrade-pip pip install dbt-core dbt-athena-community install-pyspark: upgrade-pip - pip install -e . pytest pyspark + pip install -e . pytest pyspark dotenv install-all: install-dbt install-pyspark