major major code refactor

.github/workflows/theseus-engine.yml

@@ -4,6 +4,21 @@ on:
   schedule:
     - cron: "0 0 1 * *"
   workflow_dispatch:
+    inputs:
+      reprocess:
+        description: 'Reprocess state. Type "all" to wipe and rebuild completely, or "last" to re-calculate the most recent period.'
+        required: false
+        type: choice
+        options:
+          - ""
+          - "last"
+          - "all"
+        default: ""
+      full_fossil_scan:
+        description: 'Run a full deep-scan for fossils (uncheck to only do a fast survivor update)'
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
   group: theseus-data-engine
@@ -43,12 +58,30 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction --no-root
 
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
+        with:
+          workspaces: "engine -> target"
+
+      - name: Build engine
+        working-directory: engine
+        run: cargo build --release
+
       - name: Run pipeline for ${{ matrix.repo }}
         id: pipeline
         continue-on-error: true
         env:
           REPO_NAME: ${{ matrix.repo }}
-        run: poetry run python -m scripts.run_pipeline --repo "$REPO_NAME" --update-survivor
+          REPROCESS_INPUT: ${{ inputs.reprocess }}
+        run: |
+          ARGS=()
+          if [ -n "$REPROCESS_INPUT" ]; then
+            ARGS+=(--reprocess "$REPROCESS_INPUT")
+          fi
+          if [ "${{ inputs.full_fossil_scan }}" != "true" ]; then
+            ARGS+=(--update-survivor)
+          fi
+          poetry run python -m scripts.run_pipeline --repo "$REPO_NAME" "${ARGS[@]}"
         timeout-minutes: 120
 
       - name: Save status
@@ -64,7 +97,9 @@ jobs:
         with:
           name: data-${{ matrix.repo }}
           path: |
-            data/raw/${{ matrix.repo }}_data.json
+            data/raw/${{ matrix.repo }}_history.jsonl
+            data/raw/${{ matrix.repo }}_fossils.json
+            data/state/${{ matrix.repo }}_state.json
             data/processed/${{ matrix.repo }}_graph.json
             data/.status/${{ matrix.repo }}.json
           retention-days: 1

.github/workflows/unit-tests.yml

@@ -28,9 +28,25 @@ jobs:
       - name: Install dependencies
         run: poetry install --with dev
 
-      - name: Run linter
+      - name: Run Python linter
         run: poetry run pylint scripts/ --output-format=colorized
         continue-on-error: true
 
-      - name: Run tests
+      - name: Run Python tests
         run: poetry run pytest tests/ -v --tb=short
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
+        with:
+          workspaces: "engine -> target"
+
+      - name: Run Rust Lints & Formatting
+        working-directory: engine
+        run: |
+          cargo fmt --check
+          cargo clippy -- -D warnings
+        continue-on-error: true
+
+      - name: Run Rust tests
+        working-directory: engine
+        run: cargo test

build_engine.bat

@@ -0,0 +1,19 @@
+@echo off
+set CARGO_CMD=cargo
+where cargo >nul 2>nul
+if %errorlevel% neq 0 (
+    set CARGO_CMD="%USERPROFILE%\.cargo\bin\cargo.exe"
+)
+
+echo Building Theseus Engine (Release mode)...
+pushd "%~dp0engine" || exit /b 1
+%CARGO_CMD% build --release
+set "EXIT_CODE=%errorlevel%"
+popd
+if %EXIT_CODE% neq 0 (
+    echo Build failed! Make sure Rust is installed and in your PATH.
+    pause
+    exit /b %EXIT_CODE%
+)
+echo Build successful! The binary is located at engine\target\release\engine.exe
+pause

data/raw/langchain_history.jsonl

@@ -0,0 +1,9 @@
+{"snapshot_date":"2022-12","commit_hash":"d95b39d37f9b15440cd87e97df7132699f69c8ab","composition":{"2022":33781}}
+{"snapshot_date":"2023-03","commit_hash":"4b59bb55c74449bdff0fe88bf0b98fd8052cea25","composition":{"2022":23173,"2023":99857}}
+{"snapshot_date":"2023-06","commit_hash":"b0859c9b185fe897f3c8e2699835a669b2a2ba61","composition":{"2023":673974,"2022":18477,"2026":4}}
+{"snapshot_date":"2023-09","commit_hash":"5e5039dbd284f7f25dd6d40d372c1334c654cf49","composition":{"2023":737974,"2022":15979,"2026":4}}
+{"snapshot_date":"2023-12","commit_hash":"26f84b74d0f7dc4d2211a1a62d47eec36cb1d726","composition":{"2023":1060355,"2022":11100,"2026":8}}
+{"snapshot_date":"2024-03","commit_hash":"003c98e5b440a420d8dbd5f9fdea08888a4a5a33","composition":{"2024":491964,"2023":889068,"2022":10561,"2026":8}}
+{"snapshot_date":"2024-06","commit_hash":"9604cb833b9cb9d04a0eb60754e68402ab2d4b3c","composition":{"2023":709506,"2024":566702,"2022":7565,"2026":8}}
+{"snapshot_date":"2024-09","commit_hash":"e317d457cfe8586e4006b5f41e2c4f1e18a4d58c","composition":{"2024":658281,"2023":662951,"2022":7142,"2026":8}}
+{"snapshot_date":"2024-12","commit_hash":"edbe7d5f5e0dcc771c1f53a49bb784a3960ce448","composition":{"2024":720724,"2023":544255,"2022":7122,"2026":8}}