From ed51ba214b3bdb270a6aba4852f96f42ee676b45 Mon Sep 17 00:00:00 2001 From: Tomasz Date: Fri, 19 Jun 2026 21:51:18 +0200 Subject: [PATCH] Configure unit tests --- .github/workflows/dbt_tests.yml | 2 +- .github/workflows/unit_tests.yml | 35 +++++++++++++++++ Makefile | 18 +++++++-- pyproject.toml | 2 + tests/conftest.py | 17 +++++++++ tests/test_api.py | 29 ++++++++++++++ tests/test_bronze_to_silver.py | 56 +++++++++++++++++++++++++++ uv.lock | 65 ++++++++++++++++++++++++++++++++ 8 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/unit_tests.yml create mode 100644 tests/conftest.py create mode 100644 tests/test_api.py create mode 100644 tests/test_bronze_to_silver.py diff --git a/.github/workflows/dbt_tests.yml b/.github/workflows/dbt_tests.yml index f4b9703..e7bca9d 100644 --- a/.github/workflows/dbt_tests.yml +++ b/.github/workflows/dbt_tests.yml @@ -22,7 +22,7 @@ jobs: python-version: '3.12' - name: Install dbt and dependencies - run: make install + run: make install-dbt - name: Configuration of AWS credentials uses: aws-actions/configure-aws-credentials@v4 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000..496b19f --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,35 @@ +name: Python Unit Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + unit-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install dependencies + run: make install-pyspark + + - name: Run Pytest + run: make run-tests \ No newline at end of file diff --git a/Makefile b/Makefile index 3a4d45a..f722ff1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install airflow-up airflow-down airflow-restart dbt-run dbt-debug dbt-test install +.PHONY: install airflow-up airflow-down airflow-restart dbt-run dbt-debug dbt-test upgrade-pip install-dbt install-pyspark airflow-up: docker compose -f ./airflow/docker-compose.yml up -d @@ -21,6 +21,16 @@ dbt-test: terraform-apply: cd ./terraform && terraform apply -install: - pip install --upgrade pip - pip install dbt-core dbt-athena-community \ No newline at end of file +upgrade-pip: + python -m pip install --upgrade pip + +install-dbt: upgrade-pip + pip install dbt-core dbt-athena-community + +install-pyspark: upgrade-pip + pip install -e . pytest pyspark + +install-all: install-dbt install-pyspark + +run-tests: + PYTHONPATH=src pytest tests/ -v \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5d6b550..f0a5605 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,8 @@ dependencies = [ "boto3>=1.43.26", "dbt-athena-community>=1.10.1", "dbt-core>=1.11.11", + "pyspark>=4.1.2", + "pytest>=9.1.1", "requests>=2.34.2", "ruff>=0.15.16", ] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3ccd47a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +import pytest +from pyspark.sql import SparkSession + + +@pytest.fixture(scope="session") +def spark(): + + spark_session = ( + SparkSession.builder.master("local[1]") + .appName("pytest-pyspark") + .config("spark.sql.shuffle.partitions", "1") + .getOrCreate() + ) + + yield spark_session + + spark_session.stop() diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..f8f2dcc --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,29 @@ +from unittest.mock import patch + +from gittrends.ingestion.api import download_and_upload_to_s3, file_exists + + +@patch("gittrends.ingestion.api.s3_client.head_object") +def test_file_exists_when_file_is_present(mock_head_object): + mock_head_object.return_value = {} + + result = file_exists("test-bucket", "bronze/file.json.gz") + + assert result is True + mock_head_object.assert_called_once_with( + Bucket="test-bucket", Key="bronze/file.json.gz" + ) + + +@patch("gittrends.ingestion.api.file_exists") +@patch("gittrends.ingestion.api.requests.get") +def test_download_skipped_if_file_exists(mock_get, mock_file_exists): + mock_file_exists.return_value = True + + result = download_and_upload_to_s3("2026", "06", "19", "12", "test-bucket") + + assert ( + result + == "s3://test-bucket/bronze/year=2026/month=06/day=19/2026-06-19-12.json.gz" + ) + mock_get.assert_not_called() diff --git a/tests/test_bronze_to_silver.py b/tests/test_bronze_to_silver.py new file mode 100644 index 0000000..6f9fd34 --- /dev/null +++ b/tests/test_bronze_to_silver.py @@ -0,0 +1,56 @@ +import json + +from gittrends.databricks.bronze_to_silver import clean_and_flatten_data + + +def test_clean_and_flatten_data_explodes_commits_and_filters(spark): + + mock_data = [ + # EVENT 1: PushEvent with two commits + json.dumps( + { + "id": "1001", + "type": "PushEvent", + "actor": {"login": "octocat"}, + "repo": {"name": "apache/spark"}, + "created_at": "2026-06-19T10:00:00Z", + "payload": { + "action": "created", + "size": 2, + "commits": [ + {"message": "Pierwszy commit", "author": {"name": "Alice"}}, + {"message": "Drugi commit", "author": {"name": "Bob"}}, + ], + }, + } + ), + # EVENT 2: ID missing + json.dumps( + { + "id": None, + "type": "WatchEvent", + "actor": {"login": "janedoe"}, + "repo": {"name": "aws/aws-cli"}, + "created_at": "2026-06-19T11:00:00Z", + "payload": {}, + } + ), + ] + + rdd = spark.sparkContext.parallelize(mock_data) + df_bronze = spark.read.json(rdd) + + df_silver = clean_and_flatten_data(df_bronze) + results = df_silver.collect() + + assert len(results) == 2 + + assert results[0]["event_id"] == "1001" + assert results[0]["commit_author"] == "Alice" + assert results[0]["commit_message"] == "Pierwszy commit" + + assert results[1]["event_id"] == "1001" + assert results[1]["commit_author"] == "Bob" + assert results[1]["commit_message"] == "Drugi commit" + + assert results[0]["actor_login"] == "octocat" diff --git a/uv.lock b/uv.lock index 7daac2f..0327ad5 100644 --- a/uv.lock +++ b/uv.lock @@ -417,6 +417,8 @@ dependencies = [ { name = "boto3" }, { name = "dbt-athena-community" }, { name = "dbt-core" }, + { name = "pyspark" }, + { name = "pytest" }, { name = "requests" }, { name = "ruff" }, ] @@ -426,6 +428,8 @@ requires-dist = [ { name = "boto3", specifier = ">=1.43.26" }, { name = "dbt-athena-community", specifier = ">=1.10.1" }, { name = "dbt-core", specifier = ">=1.11.11" }, + { name = "pyspark", specifier = ">=4.1.2" }, + { name = "pytest", specifier = ">=9.1.1" }, { name = "requests", specifier = ">=2.34.2" }, { name = "ruff", specifier = ">=0.15.16" }, ] @@ -451,6 +455,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "isodate" version = "0.7.2" @@ -763,6 +776,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "protobuf" version = "6.33.6" @@ -778,6 +800,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "py4j" +version = "0.10.9.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/db/ea0203e495be491c85af87b66e37acfd3bf756fd985f87e46fc5e3bf022c/py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533", size = 203008, upload-time = "2025-01-15T03:53:15.648Z" }, +] + [[package]] name = "pyathena" version = "3.32.0" @@ -884,6 +915,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, ] +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pyspark" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py4j" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/71/4dd20c69332a2a4bf7ece8a655c9da98e4bd9b6bcea235349c1a00399d57/pyspark-4.1.2.tar.gz", hash = "sha256:fa5d6159f700d0990a07f4f62df1b7449401dccee9cd7d5d6df8957530841602", size = 455428043, upload-time = "2026-05-21T14:49:21.785Z" } + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"