diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ea779b..60e6a72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,21 +5,33 @@ on: pull_request: jobs: - build: + test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - - name: Check syntax + - name: Lint with flake8 run: | - python -m compileall . \ No newline at end of file + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + run: | + pytest tests/ --cov=src --cov-report=xml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9f575e4..6aa9975 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ data/ *.wav *.m4a *.webm +*.ogg # ====================== # Logs @@ -76,3 +77,4 @@ Thumbs.db .coverage htmlcov/ .pytest_cache/ +coverage.xml diff --git a/app/app.py b/app/app.py index 6a5313f..9f88fa8 100644 --- a/app/app.py +++ b/app/app.py @@ -1,5 +1,8 @@ -import streamlit as st +import sys import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import streamlit as st from src.ingestion.youtube import fetch_youtube_transcript, download_audio, get_video_info from src.ingestion.transcribe import transcribe_audio from src.processing.summarize import summarize_text diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..a635c5c --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_chunking.py b/tests/test_chunking.py new file mode 100644 index 0000000..f288fed --- /dev/null +++ b/tests/test_chunking.py @@ -0,0 +1,26 @@ +import pytest +from src.processing.chunking import split_text + +def test_split_text_empty(): + assert split_text("") == [] + assert split_text(" ") == [] + +def test_split_text_single_short_sentence(): + text = "This is a short sentence." + chunks = split_text(text, max_words=10, overlap=2) + assert len(chunks) == 1 + assert chunks[0] == text + +def test_split_text_multiple_sentences(): + text = "Hello world. This is a test. We should split this well." + chunks = split_text(text, max_words=6, overlap=2) + assert len(chunks) >= 2 + # Ensure all words are present across chunks roughly + +def test_split_text_long_sentence_fallback(): + # Sentence without punctuation longer than max_words + text = "this is a very long sentence that has no punctuation and just keeps going on and on for many words without stopping so we need to fallback on word splitting" + chunks = split_text(text, max_words=10, overlap=2) + assert sum(len(c.split()) for c in chunks) >= len(text.split()) + for chunk in chunks: + assert len(chunk.split()) <= 10 diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py new file mode 100644 index 0000000..071ed0a --- /dev/null +++ b/tests/test_ingestion.py @@ -0,0 +1,35 @@ +import pytest +from unittest.mock import patch, MagicMock +from src.ingestion.youtube import fetch_youtube_transcript, get_video_info + +@patch('src.ingestion.youtube.yt_dlp.YoutubeDL') +def test_get_video_info(mock_ytdl): + mock_instance = MagicMock() + mock_ytdl.return_value.__enter__.return_value = mock_instance + mock_instance.extract_info.return_value = { + 'title': 'Test Video', + 'duration': 120, + 'channel': 'Test Channel', + 'subtitles': {'en': {}}, + 'automatic_captions': {} + } + + info = get_video_info('http://test.url') + assert info is not None + assert info['title'] == 'Test Video' + assert info['has_manual_subs'] == True + assert info['has_auto_subs'] == False + +@patch('src.ingestion.youtube.tempfile.TemporaryDirectory') +@patch('src.ingestion.youtube.yt_dlp.YoutubeDL') +def test_fetch_youtube_transcript_no_subs(mock_ytdl, mock_temp): + mock_instance = MagicMock() + mock_ytdl.return_value.__enter__.return_value = mock_instance + mock_instance.extract_info.return_value = { + 'subtitles': {}, + 'automatic_captions': {} + } + + transcript, source = fetch_youtube_transcript('http://test.url') + assert transcript is None + assert source is None