From da682994c01d9d011c13d1a3355f48c8bff079bb Mon Sep 17 00:00:00 2001 From: jorsincl Date: Wed, 12 Nov 2025 13:28:49 -0700 Subject: [PATCH] Project Setup and model development --- .env.example | 0 .github/workflows/ci.yml | 56 +++++++++++++++++ .gitignore | 4 ++ README.md | 89 ++++++++++++++++++++++++++- backend/Dockerfile | 0 backend/app/api/__init__.py | 0 backend/app/api/endpoints.py | 0 backend/app/core/config.py | 0 backend/app/core/database.py | 0 backend/app/main.py | 0 backend/app/services/model_service.py | 0 backend/requirements.txt | 0 backend/tests/test_api.py | 0 data/README.md | 18 ++++++ docker-compose.yml | 0 experiments/notebooks/eda.ipynb | 0 experiments/tracking/wandb.py | 0 experiments/training/preprocess.py | 0 experiments/training/train_model.py | 0 experiments/training/utils.py | 0 monitoring/Dockerfile | 0 monitoring/dashboard_app.py | 0 monitoring/requirements.txt | 0 requirements.txt | 0 tests/test_preprocess.py | 0 25 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 backend/Dockerfile create mode 100644 backend/app/api/__init__.py create mode 100644 backend/app/api/endpoints.py create mode 100644 backend/app/core/config.py create mode 100644 backend/app/core/database.py create mode 100644 backend/app/main.py create mode 100644 backend/app/services/model_service.py create mode 100644 backend/requirements.txt create mode 100644 backend/tests/test_api.py create mode 100644 data/README.md create mode 100644 docker-compose.yml create mode 100644 experiments/notebooks/eda.ipynb create mode 100644 experiments/tracking/wandb.py create mode 100644 experiments/training/preprocess.py create mode 100644 experiments/training/train_model.py create mode 100644 experiments/training/utils.py create mode 100644 monitoring/Dockerfile create mode 100644 monitoring/dashboard_app.py create mode 100644 monitoring/requirements.txt create mode 100644 requirements.txt create mode 100644 tests/test_preprocess.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ab53047 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: CI Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + run: | + pytest test.py -v + + - name: Test Streamlit app startup + run: | + timeout 10s streamlit run app.py --headless --server.port 8501 || true + echo "Streamlit app startup test completed" + + code-quality: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + + - name: Lint with flake8 + run: | + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # Treat all other issues as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics \ No newline at end of file diff --git a/.gitignore b/.gitignore index b7faf40..734a980 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,7 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# Data +raw/ +processed/ \ No newline at end of file diff --git a/README.md b/README.md index 4c91efc..5117f22 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,87 @@ -# readcrumbs -A system that recommends books given a user’s favorite titles, including an API, monitoring website, and app. +# ReadCrumbs + +This is an end-to-end Machine Learning Operations (MLOps) project designed to deliver personalized book recommendations in a production-ready environment. The system includes experiment tracking (W&B), a model registry, a FastAPI serving backend, persistent logging, and separate user and monitoring interfaces, all containerized and ready for deployment on AWS EC2. + +## Core System Components + +The architecture is split into three main containerized services: + +1. ML Model Backend: A Python FastAPI application that loads the "Production" Matrix Factorization model from the Model Registry (W&B), serves predictions via a /predict endpoint, and logs all requests to the persistent DynamoDB/RDS store. +2. Frontend Interface: A React application allowing users to input books and view real-time recommendations from the FastAPI backend. +3. Model Monitoring Dashboard: A dedicated Streamlit/Python dashboard that connects directly to the database to visualize live prediction latency, data drift, and model performance metrics. + + +## Local Setup and Installation + +Follow these steps to get the environment ready for development: + +1. Prerequisites +You should have Docker downloaded on your system, and follow the steps below to set up an environment. + +```bash +# Create virtual environment or conda environment. +# Conda: +conda create -n readcrumbs -y +conda activate readcrumbs +# -- or create a virtual environment -- +python -m venv venv +source venv/bin/activate + +# Install dependencies for whole project +pip install -r requirements.txt +``` + +2. Clone the Repository + +```bash +git clone https://github.com/smiley-maker/readcrumbs +cd readcrumbs +``` + +3. Environment Variables + +DO NOT COMMIT YOUR SECRETS TO GIT. + +Copy the structure from the example file to create your local secrets file: + +```bash +cp .env.example .env +``` + +Fill in the actual, sensitive values (API keys, passwords, etc.) into the new .env file. + +## Running the Project Locally + +The entire system is containerized and managed via docker-compose. This allows us to run the three main services (Backend API, Frontend, Monitoring) simultaneously. + +1. Build Containers + +Build the Docker images for all services defined in the docker-compose.yml file: + +```bash +docker compose build +``` + +2. Run All Services + +Start the entire MLOps system in detached mode: + +```bash +docker compose up -d +``` + +3. Accessing the Services + +Once running, you can access the three key components in your browser: + +- FastAPI Backend API (Health Check): http://localhost:8000/health +- Frontend Interface: http://localhost:8080/ +- Monitoring Dashboard: http://localhost:8081/ + +4. Shut Down + +To stop and remove the containers: + +```bash +docker compose down +``` \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/endpoints.py b/backend/app/api/endpoints.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/config.py b/backend/app/core/config.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/database.py b/backend/app/core/database.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/model_service.py b/backend/app/services/model_service.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py new file mode 100644 index 0000000..e69de29 diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..0f051c0 --- /dev/null +++ b/data/README.md @@ -0,0 +1,18 @@ +# Data Source and Processing Pipeline + +This directory documents the data source, processing steps, and artifact management for the Personalized Book Recommender. + +No large raw or processed data files are committed to Git. These files are either managed by local download or tracked as versioned W&B Artifacts. + +## Data Source Details + +The foundation of our recommendation model is the Amazon Review Data. + +| Attribute | Details | +| :------- | :------: | +| Dataset Name | Amazon Review Data — Books Subset | +| Original Source | Julian McAuley's Amazon Review Dataset | +| Dataset Components | Ratings, Books, and Users files (specific format depends on chosen subset) | +| Size | 10.3 million users, 4.4 million items, and 29.5 million ratings | +| License | Open access for non-commercial research purposes. | +| Link | https://amazon-reviews-2023.github.io/ | \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e69de29 diff --git a/experiments/notebooks/eda.ipynb b/experiments/notebooks/eda.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/experiments/tracking/wandb.py b/experiments/tracking/wandb.py new file mode 100644 index 0000000..e69de29 diff --git a/experiments/training/preprocess.py b/experiments/training/preprocess.py new file mode 100644 index 0000000..e69de29 diff --git a/experiments/training/train_model.py b/experiments/training/train_model.py new file mode 100644 index 0000000..e69de29 diff --git a/experiments/training/utils.py b/experiments/training/utils.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/Dockerfile b/monitoring/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/dashboard_app.py b/monitoring/dashboard_app.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/requirements.txt b/monitoring/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py new file mode 100644 index 0000000..e69de29