Singular tests

Makefile

@@ -16,7 +16,10 @@ dbt-debug:
 	cd ./dbt/gittrends_dbt && dbt debug
 
 dbt-test:
-	cd ./dbt/gittrends_dbt && dbt test
+	cd ./dbt/gittrends_dbt && dbt test --select test_type:generic
+
+dbt-tests-singular:
+	cd ./dbt/gittrends_dbt && dbt test --select test_type:singular
 
 terraform-apply:
 	cd ./terraform && terraform apply
@@ -28,7 +31,7 @@ install-dbt: upgrade-pip
 	pip install dbt-core dbt-athena-community
 
 install-pyspark: upgrade-pip
-	pip install -e . pytest pyspark
+	pip install -e . pytest pyspark dotenv
 
 install-all: install-dbt install-pyspark
 

dbt/gittrends_dbt/tests/assert_no_future_commits.sql

@@ -0,0 +1,6 @@
+SELECT 
+    repo_name,
+    author,
+    event_date
+FROM {{ ref('fact_daily_commits') }}
+WHERE event_date > CURRENT_DATE

src/gittrends/databricks/bronze_to_silver.py

@@ -1,8 +1,14 @@
 # Databricks notebook source
+import os
+
+from dotenv import load_dotenv
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import col, explode_outer, lit, to_timestamp
 from pyspark.sql.types import StructType
 
+load_dotenv()
+BUCKET_NAME = os.getenv("BUCKET_NAME")
+
 
 def clean_and_flatten_data(df_bronze: DataFrame) -> DataFrame:
     payload_schema = df_bronze.schema["payload"].dataType
@@ -48,7 +54,7 @@ def clean_and_flatten_data(df_bronze: DataFrame) -> DataFrame:
 def main():
     spark = SparkSession.builder.appName("GitHubArchive-BronzeToSilver").getOrCreate()
 
-    BUCKET_NAME = "gittrends-data-lake"
+    BUCKET_NAME = "BUCKET_NAME"
     BRONZE_PATH = f"s3://{BUCKET_NAME}/bronze/*/*/*/*.json.gz"
     SILVER_PATH = f"s3://{BUCKET_NAME}/silver/github_events/"