Configure SQLMesh better

.github/workflows/tests.yaml

@@ -25,8 +25,5 @@ jobs:
       - name: 🔨 Install dependencies
         run: uv sync --no-default-groups
 
-      - name: 🤡 Create mock database
-        run: uv run python -m testing.setup
-
       - name: ✅ Run unit tests
         run: uv run sqlmesh -p billiam_database test

.gitignore

@@ -1,9 +1,8 @@
 ###  SQLMesh artifacts
 logs/
 
-### Raw data and sensitive seeds
+### Raw data
 billiam_database/models/raw/data/
-billiam_database/models/seeds/data/counterparty_exclusions.csv
 
 ###  Data objects
 *.db
@@ -143,6 +142,7 @@ celerybeat.pid
 
 ###  Environments
 .env
+.envrc
 .venv
 env/
 venv/

.pre-commit-config.yaml

@@ -2,7 +2,8 @@
 ---
 ci:
   autoupdate_schedule: quarterly
-  skip: ["identity", "uv-lock", "sqlmesh-test"]
+  skip: ["identity", "uv-lock"]
+#  skip: ["identity", "uv-lock", "sqlmesh-test"]
 
 default_install_hook_types: ["pre-commit", "pre-push"]
 default_stages: ["pre-commit"]
@@ -55,12 +56,13 @@ repos:
       - id: tidy-gitkeep
         name: Remove redundant .gitkeep files
 
-  - repo: local
-    hooks:
-      - id: sqlmesh-test
-        name: Run unit tests
-        entry: sqlmesh -p billiam_database test
-        stages: ["pre-push"]
-        language: system
-        pass_filenames: false
-        always_run: true
+  # This is trying to talk with MotherDuck and is generating a browser popup
+#  - repo: local
+#    hooks:
+#      - id: sqlmesh-test
+#        name: Run unit tests
+#        entry: sqlmesh -p billiam_database test
+#        stages: ["pre-push"]
+#        language: system
+#        pass_filenames: false
+#        always_run: true

README.md

@@ -1,4 +1,4 @@
-<div align="center">
+<span align="center">
 
 [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 [![Docker](https://img.shields.io/badge/Docker-28.4-blue.svg)](https://www.docker.com/)
@@ -7,10 +7,10 @@
 [![GitHub last commit](https://img.shields.io/github/last-commit/billwallis/billiam-database)](https://shields.io/badges/git-hub-last-commit)
 
 [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/billwallis/billiam-database/main.svg)](https://results.pre-commit.ci/latest/github/billwallis/billiam-database/main)
-[![DuckDB](https://img.shields.io/badge/DuckDB-1.4-teal.svg)](https://duckdb.org/)
+[![DuckDB](https://img.shields.io/badge/DuckDB-1.3-teal.svg)](https://duckdb.org/)
 [![Metabase](https://img.shields.io/badge/Metabase-0.47-teal.svg)](https://www.metabase.com/)
 
-</div>
+</span>
 
 ---
 

billiam_database/config.yaml

@@ -1,24 +1,29 @@
 ---
 project: billiam_database
 
-default_gateway: local
-gateways:
-  local:
-    connection:
-      type: duckdb
-      database: billiam_database/billiam.duckdb
-      # catalogs:
-      # dev: billiam_database/billiam-dev.duckdb
-      # prod: billiam_database/billiam.duckdb
-
+ignore_patterns: ["__*.sql"]
+default_target_environment: dev
 model_defaults:
   dialect: duckdb
   start: 2018-01-01
 
-ignore_patterns: ["__*.sql"]
-
 plan:
   enable_preview: true
-#environment_catalog_mapping:
-#  '^prod$': prod
-#  '^dev$': dev
+
+default_gateway: local
+gateways:
+  local:
+    connection:
+      type: duckdb
+      catalogs:
+        billiam: "billiam_database/billiam.duckdb"
+        ods:
+          type: motherduck
+          path: "md:ods?motherduck_token={{ env_var('MOTHERDUCK_TOKEN') }}"
+  motherduck:
+    connection:
+      type: motherduck
+      token: "{{ env_var('MOTHERDUCK_TOKEN') }}"
+      catalogs:
+        warehouse: warehouse
+        ods: ods

billiam_database/models/domain/career/working_hours.sql

@@ -23,7 +23,7 @@ model (
 with recursive
 
 weeks(week_starting) as (
-        select date_trunc('week', (select min(from_date) from seeds.work_hours))
+        select date_trunc('week', (select min(from_date) from raw.work_hours))
     union all
         select week_starting + interval '1 week'
         from weeks
@@ -43,14 +43,14 @@ weekly_quota as (
             + friday
             + saturday
         ) as weekly_quota
-    from seeds.work_hours
+    from raw.work_hours
 ),
 
 absence_hours as (
     select
         date_trunc('week', absence_date) as week_starting,
         sum(hours) as absence_hours
-    from seeds.work_absences
+    from raw.work_absences
     group by all
 ),
 

billiam_database/models/raw/counterparty_exclusions.sql

@@ -0,0 +1,13 @@
+model (
+    name raw.counterparty_exclusions,
+    kind full,
+    grain (counterparty),
+    tags (finances),
+    columns (
+      counterparty varchar,
+    ),
+);
+
+select *
+from ods.finances.counterparty_exclusions
+;

billiam_database/models/raw/finances.sql

@@ -4,8 +4,9 @@ model (
     grain (row_id),
     tags (finances),
     depends_on (
+        raw.amex_transactions,
         raw.monzo_transactions,
-        seeds.counterparty_exclusions,
+        raw.counterparty_exclusions,
     ),
     columns (
         row_id int,
@@ -115,7 +116,7 @@ monzo_txns as (
       and cost != 0
       and category not in ('Savings')
       and counterparty not in ('Transport for London')
-      and counterparty not in (from seeds.counterparty_exclusions)
+      and counterparty not in (from raw.counterparty_exclusions)
 ),
 
 joined as (

billiam_database/models/seeds/work_absences.sql → billiam_database/models/raw/work_absences.sql

@@ -1,8 +1,6 @@
 model (
-    name seeds.work_absences,
-    kind seed (
-      path 'data/work_absences.csv',
-    ),
+    name raw.work_absences,
+    kind full,
     grain (absence_date),
     tags (daily-tracker),
     columns (
@@ -11,3 +9,7 @@ model (
       hours decimal(4, 2),
     ),
 );
+
+select *
+from ods.daily_tracker.work_absences
+;

billiam_database/models/seeds/work_hours.sql → billiam_database/models/raw/work_hours.sql

@@ -1,8 +1,6 @@
 model (
-    name seeds.work_hours,
-    kind seed (
-      path 'data/work_hours.csv',
-    ),
+    name raw.work_hours,
+    kind full,
     grain (company, from_date),
     tags (daily-tracker),
     columns (
@@ -18,3 +16,7 @@ model (
       saturday decimal(4, 2),
     ),
 );
+
+select *
+from ods.daily_tracker.work_hours
+;

pyproject.toml

@@ -13,8 +13,8 @@ readme = "README.md"
 keywords = ["sql", "sqlmesh", "duckdb"]
 requires-python = ">=3.11"
 dependencies = [
-    "duckdb>=1.4.0",
-    "sqlmesh>=0.224.0",
+    "duckdb==1.3.2",
+    "sqlmesh[duckdb]>=0.224.0",
 ]
 
 [dependency-groups]
@@ -48,6 +48,8 @@ lint.shell = """
     SKIP=identity pre-commit run --all-files --hook-stage pre-commit
     echo '=== pre-push checks ==='
     pre-commit run --all-files --hook-stage pre-push
+    echo '=== sqlmesh checks ==='
+    sqlmesh -p billiam_database lint
 """
 update.shell = """
     uv sync --refresh --upgrade