Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

# ----- PostgreSQL (Docker) -----
POSTGRES_USER=forum
POSTGRES_PASSWORD=forum_password
POSTGRES_PASSWORD=YOUR_DB_PASSWORD_HERE
POSTGRES_DB=the_forum
POSTGRES_PORT=5434

Expand All @@ -16,17 +16,17 @@ DATABASE_URL=postgresql://forum:forum_password@localhost:5434/the_forum
AUTH_SECRET=your-auth-secret-here
AUTH_AZURE_AD_CLIENT_ID=your-client-id
AUTH_AZURE_AD_CLIENT_SECRET=your-client-secret
AUTH_AZURE_AD_TENANT_ID=2ff60116-7431-425d-b5af-077d7791bda4
AUTH_AZURE_AD_TENANT_ID=YOUR_TENANT_ID_HERE

# ----- AWS S3 (image uploads) -----
# AWS_S3_BUCKET=the-forum-uploads
# AWS_REGION=us-east-1

# ----- Next.js public vars (prefix with NEXT_PUBLIC_) -----
# NEXT_PUBLIC_API_URL=http://localhost:8000
NEXT_PUBLIC_MAPBOX_TOKEN=pk.your-mapbox-public-token
NEXT_PUBLIC_CAMPUS_MAP_TOKEN=pk.campus-map-public-token
NEXT_PUBLIC_CAMPUS_MAP_STYLE=mapbox://styles/account/style-id
NEXT_PUBLIC_MAPBOX_TOKEN=YOUR_CAMPUS_MAPBOX_TOKEN_HERE
NEXT_PUBLIC_CAMPUS_MAP_TOKEN=YOUR_CAMPUS_MAP_TOKEN_HERE
NEXT_PUBLIC_CAMPUS_MAP_STYLE=YOUR_CAMPUS_MAP_STYLE_HERE

# ----- FastAPI -----
# FASTAPI_SECRET_KEY=changeme
Expand All @@ -44,4 +44,4 @@ NEXT_PUBLIC_CAMPUS_MAP_STYLE=mapbox://styles/account/style-id

# ----- Listserv Scraper -----
LISTSERV_EMAIL=tigerapp@princeton.edu
LISTSERV_PASSWORD=your-listserv-password
LISTSERV_PASSWORD=YOUR_LISTSERV_PASSWORD_HERE
41 changes: 41 additions & 0 deletions .github/workflows/scrape_listserv.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: Scrape WhitmanWire

on:
schedule:
# Runs at 8:00 AM UTC (4:00 AM Eastern Time) every day
- cron: '0 8 * * *'
workflow_dispatch: # Allows you to click a "Run workflow" button in GitHub's UI

jobs:
scrape-and-commit:
runs-on: ubuntu-latest
# This permission is required so the bot can push the JSON file back to your repo
permissions:
contents: write

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11' # The script uses standard libraries, so this is fine

- name: Run Scraper
env:
# Pulls securely from GitHub Secrets
LISTSERV_EMAIL: ${{ secrets.LISTSERV_EMAIL }}
LISTSERV_PASSWORD: ${{ secrets.LISTSERV_PASSWORD }}
run: |
python3 apps/listserv-scraper/src/scrape_listserv.py --list WHITMANWIRE --limit 100 --fetch-bodies

- name: Commit and push changes
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
# Stage the specific data folder where the script saves its output
git add apps/listserv-scraper/data/
# Commit the changes; if there are no new emails, it fails gracefully instead of crashing
git commit -m "chore: update WhitmanWire scraped data" || echo "No changes to commit"
git push
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ build/
!.env.example
!**/.env.example
!**/env.local.example
.forum_venv/

# ---- Database ----
drizzle/
# drizzle/

# ---- Python ----
__pycache__/
Expand Down
3 changes: 3 additions & 0 deletions apps/database/drizzle/0000_setup_vector_extension.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- Custom SQL migration file, put your code below! --

CREATE EXTENSION IF NOT EXISTS vector;
215 changes: 215 additions & 0 deletions apps/database/drizzle/0001_init_schema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
CREATE TYPE "public"."campus_region" AS ENUM('central', 'east', 'west', 'south', 'north', 'off-campus');--> statement-breakpoint
CREATE TYPE "public"."event_status" AS ENUM('draft', 'published');--> statement-breakpoint
CREATE TYPE "public"."event_tag" AS ENUM('free-food', 'workshop', 'performance', 'speaker', 'social', 'career', 'sports', 'music', 'art', 'academic', 'cultural', 'community-service', 'religious', 'political', 'tech', 'gaming', 'outdoor', 'wellness');--> statement-breakpoint
CREATE TYPE "public"."friendship_status" AS ENUM('pending', 'accepted', 'declined');--> statement-breakpoint
CREATE TYPE "public"."interaction_type" AS ENUM('view', 'click', 'rsvp', 'save', 'share', 'hide');--> statement-breakpoint
CREATE TYPE "public"."item_type" AS ENUM('event', 'organization');--> statement-breakpoint
CREATE TYPE "public"."location_category" AS ENUM('academic', 'residential', 'athletic', 'social', 'administrative', 'library', 'dining', 'other');--> statement-breakpoint
CREATE TYPE "public"."notification_type" AS ENUM('friend_request', 'event_reminder', 'org_new_event');--> statement-breakpoint
CREATE TYPE "public"."org_category" AS ENUM('career', 'affinity', 'performance', 'academic', 'athletic', 'social', 'cultural', 'religious', 'political', 'service');--> statement-breakpoint
CREATE TYPE "public"."org_role" AS ENUM('owner', 'officer', 'member');--> statement-breakpoint
CREATE TYPE "public"."pipeline_log_status" AS ENUM('success', 'skipped_not_event', 'duplicate', 'error');--> statement-breakpoint
CREATE TABLE "campus_locations" (
"id" varchar(100) PRIMARY KEY NOT NULL,
"name" varchar(255) NOT NULL,
"latitude" double precision NOT NULL,
"longitude" double precision NOT NULL,
"category" "location_category" NOT NULL
);
--> statement-breakpoint
CREATE TABLE "event_tags" (
"event_id" uuid NOT NULL,
"tag" "event_tag" NOT NULL,
CONSTRAINT "event_tags_event_id_tag_pk" PRIMARY KEY("event_id","tag")
);
--> statement-breakpoint
CREATE TABLE "events" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"title" varchar(200) NOT NULL,
"description" text NOT NULL,
"datetime" timestamp NOT NULL,
"end_datetime" timestamp,
"location_id" varchar(100) NOT NULL,
"org_id" uuid,
"creator_id" uuid NOT NULL,
"flyer_url" text,
"cover_preset" varchar(50),
"external_link" text,
"is_public" boolean DEFAULT true NOT NULL,
"status" "event_status" DEFAULT 'published' NOT NULL,
"source" varchar(20) DEFAULT 'manual' NOT NULL,
"source_message_id" varchar(255),
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "events_source_message_id_unique" UNIQUE("source_message_id")
);
--> statement-breakpoint
CREATE TABLE "friendships" (
"user_id" uuid NOT NULL,
"friend_id" uuid NOT NULL,
"status" "friendship_status" DEFAULT 'pending' NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "friendships_user_id_friend_id_pk" PRIMARY KEY("user_id","friend_id")
);
--> statement-breakpoint
CREATE TABLE "interactions" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"user_id" uuid NOT NULL,
"item_id" uuid NOT NULL,
"item_type" "item_type" DEFAULT 'event' NOT NULL,
"interaction_type" "interaction_type" NOT NULL,
"interaction_value" double precision NOT NULL,
"metadata" jsonb,
"created_at" timestamp DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "listserv_configs" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"address" varchar(255) NOT NULL,
"label" varchar(255) NOT NULL,
"org_id" uuid,
"gmail_label" varchar(255),
"enabled" boolean DEFAULT true NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "listserv_configs_address_unique" UNIQUE("address")
);
--> statement-breakpoint
CREATE TABLE "listserv_emails" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"message_id" varchar(255) NOT NULL,
"listserv" varchar(100) NOT NULL,
"subject" text NOT NULL,
"author_name" varchar(255),
"author_email" varchar(255),
"date" timestamp with time zone,
"body_text" text,
"body_html" text,
"is_hoagiemail" boolean DEFAULT false NOT NULL,
"hoagiemail_sender_name" varchar(255),
"hoagiemail_sender_email" varchar(255),
"links" jsonb DEFAULT '[]'::jsonb,
"images" jsonb DEFAULT '[]'::jsonb,
"attachments" jsonb DEFAULT '[]'::jsonb,
"listserv_url" text,
"created_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "listserv_emails_message_id_unique" UNIQUE("message_id")
);
--> statement-breakpoint
CREATE TABLE "notifications" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"user_id" uuid NOT NULL,
"type" "notification_type" NOT NULL,
"payload" jsonb NOT NULL,
"read" boolean DEFAULT false NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "org_followers" (
"org_id" uuid NOT NULL,
"user_id" uuid NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "org_followers_org_id_user_id_pk" PRIMARY KEY("org_id","user_id")
);
--> statement-breakpoint
CREATE TABLE "org_members" (
"org_id" uuid NOT NULL,
"user_id" uuid NOT NULL,
"role" "org_role" NOT NULL,
CONSTRAINT "org_members_org_id_user_id_pk" PRIMARY KEY("org_id","user_id")
);
--> statement-breakpoint
CREATE TABLE "organizations" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"name" varchar(255) NOT NULL,
"description" text,
"logo_url" text,
"category" "org_category" NOT NULL,
"creator_id" uuid NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "organizations_name_unique" UNIQUE("name")
);
--> statement-breakpoint
CREATE TABLE "pipeline_logs" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"message_id" varchar(255) NOT NULL,
"listserv_config_id" uuid,
"status" "pipeline_log_status" NOT NULL,
"error_text" text,
"extracted_event_id" uuid,
"created_at" timestamp DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "rsvps" (
"user_id" uuid NOT NULL,
"event_id" uuid NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "rsvps_user_id_event_id_pk" PRIMARY KEY("user_id","event_id")
);
--> statement-breakpoint
CREATE TABLE "saved_events" (
"user_id" uuid NOT NULL,
"event_id" uuid NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "saved_events_user_id_event_id_pk" PRIMARY KEY("user_id","event_id")
);
--> statement-breakpoint
CREATE TABLE "user_interests" (
"user_id" uuid NOT NULL,
"tag" "event_tag" NOT NULL,
CONSTRAINT "user_interests_user_id_tag_pk" PRIMARY KEY("user_id","tag")
);
--> statement-breakpoint
CREATE TABLE "user_preference_vectors" (
"user_id" uuid PRIMARY KEY NOT NULL,
"tag_weights" jsonb DEFAULT '{}'::jsonb NOT NULL,
"latent_vector" double precision[],
"interaction_count" double precision DEFAULT 0 NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "user_regions" (
"user_id" uuid NOT NULL,
"region" "campus_region" NOT NULL,
CONSTRAINT "user_regions_user_id_region_pk" PRIMARY KEY("user_id","region")
);
--> statement-breakpoint
CREATE TABLE "users" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"net_id" varchar(50) NOT NULL,
"email" varchar(255) NOT NULL,
"display_name" varchar(255) NOT NULL,
"class_year" varchar(10),
"major" varchar(255),
"avatar_url" text,
"is_org_leader" boolean DEFAULT false NOT NULL,
"onboarded" boolean DEFAULT false NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL,
CONSTRAINT "users_net_id_unique" UNIQUE("net_id"),
CONSTRAINT "users_email_unique" UNIQUE("email")
);
--> statement-breakpoint
ALTER TABLE "event_tags" ADD CONSTRAINT "event_tags_event_id_events_id_fk" FOREIGN KEY ("event_id") REFERENCES "public"."events"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "events" ADD CONSTRAINT "events_location_id_campus_locations_id_fk" FOREIGN KEY ("location_id") REFERENCES "public"."campus_locations"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "events" ADD CONSTRAINT "events_org_id_organizations_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."organizations"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "events" ADD CONSTRAINT "events_creator_id_users_id_fk" FOREIGN KEY ("creator_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "friendships" ADD CONSTRAINT "friendships_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "friendships" ADD CONSTRAINT "friendships_friend_id_users_id_fk" FOREIGN KEY ("friend_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "interactions" ADD CONSTRAINT "interactions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "listserv_configs" ADD CONSTRAINT "listserv_configs_org_id_organizations_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."organizations"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "notifications" ADD CONSTRAINT "notifications_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "org_followers" ADD CONSTRAINT "org_followers_org_id_organizations_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."organizations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "org_followers" ADD CONSTRAINT "org_followers_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "org_members" ADD CONSTRAINT "org_members_org_id_organizations_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."organizations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "org_members" ADD CONSTRAINT "org_members_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "organizations" ADD CONSTRAINT "organizations_creator_id_users_id_fk" FOREIGN KEY ("creator_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "pipeline_logs" ADD CONSTRAINT "pipeline_logs_listserv_config_id_listserv_configs_id_fk" FOREIGN KEY ("listserv_config_id") REFERENCES "public"."listserv_configs"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "pipeline_logs" ADD CONSTRAINT "pipeline_logs_extracted_event_id_events_id_fk" FOREIGN KEY ("extracted_event_id") REFERENCES "public"."events"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "rsvps" ADD CONSTRAINT "rsvps_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "rsvps" ADD CONSTRAINT "rsvps_event_id_events_id_fk" FOREIGN KEY ("event_id") REFERENCES "public"."events"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "saved_events" ADD CONSTRAINT "saved_events_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "saved_events" ADD CONSTRAINT "saved_events_event_id_events_id_fk" FOREIGN KEY ("event_id") REFERENCES "public"."events"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "user_interests" ADD CONSTRAINT "user_interests_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "user_preference_vectors" ADD CONSTRAINT "user_preference_vectors_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "user_regions" ADD CONSTRAINT "user_regions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
72 changes: 72 additions & 0 deletions apps/database/drizzle/0002_update_enums.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
ALTER TYPE "public"."pipeline_log_status" ADD VALUE 'needs_review';--> statement-breakpoint

CREATE TABLE "event_tag_embeddings" (
"tag_name" "event_tag" PRIMARY KEY NOT NULL,
"embedding" vector(1536) NOT NULL
);
--> statement-breakpoint

ALTER TABLE "event_tag_embeddings" ALTER COLUMN "tag_name" SET DATA TYPE text;--> statement-breakpoint
ALTER TABLE "event_tags" ALTER COLUMN "tag" SET DATA TYPE text;--> statement-breakpoint
ALTER TABLE "user_interests" ALTER COLUMN "tag" SET DATA TYPE text;--> statement-breakpoint

DROP TYPE "public"."event_tag";--> statement-breakpoint
CREATE TYPE "public"."event_tag" AS ENUM('free food', 'career', 'research', 'academics', 'tech', 'entrepreneurship', 'politics', 'visual arts', 'performing arts', 'literature', 'culture', 'music', 'gaming', 'athletics', 'religion', 'sustainability', 'outdoors', 'wellness', 'community service', 'speaker event', 'social event', 'stem');--> statement-breakpoint

-- event_tag_embeddings is a new table, so a direct cast is safe here
ALTER TABLE "event_tag_embeddings" ALTER COLUMN "tag_name" SET DATA TYPE "public"."event_tag" USING "tag_name"::"public"."event_tag";--> statement-breakpoint

-- Explicit mapping for event_tags
ALTER TABLE "event_tags" ALTER COLUMN "tag" SET DATA TYPE "public"."event_tag" USING (
CASE "tag"
WHEN 'free-food' THEN 'free food'
WHEN 'workshop' THEN 'academics'
WHEN 'performance' THEN 'performing arts'
WHEN 'speaker' THEN 'speaker event'
WHEN 'social' THEN 'social event'
WHEN 'sports' THEN 'athletics'
WHEN 'art' THEN 'visual arts'
WHEN 'academic' THEN 'academics'
WHEN 'cultural' THEN 'culture'
WHEN 'community-service' THEN 'community service'
WHEN 'religious' THEN 'religion'
WHEN 'political' THEN 'politics'
WHEN 'outdoor' THEN 'outdoors'
ELSE "tag"
END
)::"public"."event_tag";--> statement-breakpoint

-- Explicit mapping for user_interests
ALTER TABLE "user_interests" ALTER COLUMN "tag" SET DATA TYPE "public"."event_tag" USING (
CASE "tag"
WHEN 'free-food' THEN 'free food'
WHEN 'workshop' THEN 'academics'
WHEN 'performance' THEN 'performing arts'
WHEN 'speaker' THEN 'speaker event'
WHEN 'social' THEN 'social event'
WHEN 'sports' THEN 'athletics'
WHEN 'art' THEN 'visual arts'
WHEN 'academic' THEN 'academics'
WHEN 'cultural' THEN 'culture'
WHEN 'community-service' THEN 'community service'
WHEN 'religious' THEN 'religion'
WHEN 'political' THEN 'politics'
WHEN 'outdoor' THEN 'outdoors'
ELSE "tag"
END
)::"public"."event_tag";--> statement-breakpoint

ALTER TABLE "organizations" ALTER COLUMN "category" SET DATA TYPE text;--> statement-breakpoint
DROP TYPE "public"."org_category";--> statement-breakpoint
CREATE TYPE "public"."org_category" AS ENUM('career', 'affinity', 'performing arts', 'academics', 'athletics', 'social event', 'culture', 'religion', 'politics', 'community service');--> statement-breakpoint

-- Explicit mapping for organizations
ALTER TABLE "organizations" ALTER COLUMN "category" SET DATA TYPE "public"."org_category" USING (
CASE "category"
WHEN 'academic' THEN 'academics'
WHEN 'performance' THEN 'performing arts'
WHEN 'cultural' THEN 'culture'
WHEN 'athletic' THEN 'athletics'
ELSE "category"
END
)::"public"."org_category";
Loading
Loading