diff --git a/Makefile b/Makefile index f722ff1..bef23d2 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,10 @@ dbt-debug: cd ./dbt/gittrends_dbt && dbt debug dbt-test: - cd ./dbt/gittrends_dbt && dbt test + cd ./dbt/gittrends_dbt && dbt test --select test_type:generic + +dbt-tests-singular: + cd ./dbt/gittrends_dbt && dbt test --select test_type:singular terraform-apply: cd ./terraform && terraform apply @@ -28,7 +31,7 @@ install-dbt: upgrade-pip pip install dbt-core dbt-athena-community install-pyspark: upgrade-pip - pip install -e . pytest pyspark + pip install -e . pytest pyspark dotenv install-all: install-dbt install-pyspark diff --git a/dbt/gittrends_dbt/tests/assert_no_future_commits.sql b/dbt/gittrends_dbt/tests/assert_no_future_commits.sql new file mode 100644 index 0000000..8179fe2 --- /dev/null +++ b/dbt/gittrends_dbt/tests/assert_no_future_commits.sql @@ -0,0 +1,6 @@ +SELECT + repo_name, + author, + event_date +FROM {{ ref('fact_daily_commits') }} +WHERE event_date > CURRENT_DATE \ No newline at end of file diff --git a/src/gittrends/databricks/bronze_to_silver.py b/src/gittrends/databricks/bronze_to_silver.py index c28c240..286a053 100644 --- a/src/gittrends/databricks/bronze_to_silver.py +++ b/src/gittrends/databricks/bronze_to_silver.py @@ -1,8 +1,14 @@ # Databricks notebook source +import os + +from dotenv import load_dotenv from pyspark.sql import DataFrame, SparkSession from pyspark.sql.functions import col, explode_outer, lit, to_timestamp from pyspark.sql.types import StructType +load_dotenv() +BUCKET_NAME = os.getenv("BUCKET_NAME") + def clean_and_flatten_data(df_bronze: DataFrame) -> DataFrame: payload_schema = df_bronze.schema["payload"].dataType @@ -48,7 +54,7 @@ def clean_and_flatten_data(df_bronze: DataFrame) -> DataFrame: def main(): spark = SparkSession.builder.appName("GitHubArchive-BronzeToSilver").getOrCreate() - BUCKET_NAME = "gittrends-data-lake" + BUCKET_NAME = "BUCKET_NAME" BRONZE_PATH = f"s3://{BUCKET_NAME}/bronze/*/*/*/*.json.gz" SILVER_PATH = f"s3://{BUCKET_NAME}/silver/github_events/"