Configure unit tests

.github/workflows/dbt_tests.yml

@@ -22,7 +22,7 @@ jobs:
           python-version: '3.12'
 
       - name: Install dbt and dependencies
-        run: make install
+        run: make install-dbt
 
       - name: Configuration of AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/unit_tests.yml

@@ -0,0 +1,35 @@
+name: Python Unit Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Java 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: make install-pyspark
+
+      - name: Run Pytest
+        run: make run-tests

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install airflow-up airflow-down airflow-restart dbt-run dbt-debug dbt-test install
+.PHONY: install airflow-up airflow-down airflow-restart dbt-run dbt-debug dbt-test upgrade-pip install-dbt install-pyspark
 
 airflow-up:
 	docker compose -f ./airflow/docker-compose.yml up -d
@@ -21,6 +21,16 @@ dbt-test:
 terraform-apply:
 	cd ./terraform && terraform apply
 
-install:
-	pip install --upgrade pip
-	pip install dbt-core dbt-athena-community
+upgrade-pip:
+	python -m pip install --upgrade pip
+
+install-dbt: upgrade-pip
+	pip install dbt-core dbt-athena-community
+
+install-pyspark: upgrade-pip
+	pip install -e . pytest pyspark
+
+install-all: install-dbt install-pyspark
+
+run-tests:
+	PYTHONPATH=src pytest tests/ -v

pyproject.toml

@@ -8,6 +8,8 @@ dependencies = [
     "boto3>=1.43.26",
     "dbt-athena-community>=1.10.1",
     "dbt-core>=1.11.11",
+    "pyspark>=4.1.2",
+    "pytest>=9.1.1",
     "requests>=2.34.2",
     "ruff>=0.15.16",
 ]

tests/conftest.py

@@ -0,0 +1,17 @@
+import pytest
+from pyspark.sql import SparkSession
+
+
+@pytest.fixture(scope="session")
+def spark():
+
+    spark_session = (
+        SparkSession.builder.master("local[1]")
+        .appName("pytest-pyspark")
+        .config("spark.sql.shuffle.partitions", "1")
+        .getOrCreate()
+    )
+
+    yield spark_session
+
+    spark_session.stop()

tests/test_api.py

@@ -0,0 +1,29 @@
+from unittest.mock import patch
+
+from gittrends.ingestion.api import download_and_upload_to_s3, file_exists
+
+
+@patch("gittrends.ingestion.api.s3_client.head_object")
+def test_file_exists_when_file_is_present(mock_head_object):
+    mock_head_object.return_value = {}
+
+    result = file_exists("test-bucket", "bronze/file.json.gz")
+
+    assert result is True
+    mock_head_object.assert_called_once_with(
+        Bucket="test-bucket", Key="bronze/file.json.gz"
+    )
+
+
+@patch("gittrends.ingestion.api.file_exists")
+@patch("gittrends.ingestion.api.requests.get")
+def test_download_skipped_if_file_exists(mock_get, mock_file_exists):
+    mock_file_exists.return_value = True
+
+    result = download_and_upload_to_s3("2026", "06", "19", "12", "test-bucket")
+
+    assert (
+        result
+        == "s3://test-bucket/bronze/year=2026/month=06/day=19/2026-06-19-12.json.gz"
+    )
+    mock_get.assert_not_called()

tests/test_bronze_to_silver.py

@@ -0,0 +1,56 @@
+import json
+
+from gittrends.databricks.bronze_to_silver import clean_and_flatten_data
+
+
+def test_clean_and_flatten_data_explodes_commits_and_filters(spark):
+
+    mock_data = [
+        # EVENT 1: PushEvent with two commits
+        json.dumps(
+            {
+                "id": "1001",
+                "type": "PushEvent",
+                "actor": {"login": "octocat"},
+                "repo": {"name": "apache/spark"},
+                "created_at": "2026-06-19T10:00:00Z",
+                "payload": {
+                    "action": "created",
+                    "size": 2,
+                    "commits": [
+                        {"message": "Pierwszy commit", "author": {"name": "Alice"}},
+                        {"message": "Drugi commit", "author": {"name": "Bob"}},
+                    ],
+                },
+            }
+        ),
+        # EVENT 2: ID missing
+        json.dumps(
+            {
+                "id": None,
+                "type": "WatchEvent",
+                "actor": {"login": "janedoe"},
+                "repo": {"name": "aws/aws-cli"},
+                "created_at": "2026-06-19T11:00:00Z",
+                "payload": {},
+            }
+        ),
+    ]
+
+    rdd = spark.sparkContext.parallelize(mock_data)
+    df_bronze = spark.read.json(rdd)
+
+    df_silver = clean_and_flatten_data(df_bronze)
+    results = df_silver.collect()
+
+    assert len(results) == 2
+
+    assert results[0]["event_id"] == "1001"
+    assert results[0]["commit_author"] == "Alice"
+    assert results[0]["commit_message"] == "Pierwszy commit"
+
+    assert results[1]["event_id"] == "1001"
+    assert results[1]["commit_author"] == "Bob"
+    assert results[1]["commit_message"] == "Drugi commit"
+
+    assert results[0]["actor_login"] == "octocat"