From 856b9d0fd0308ad208a9d1b78525658836d09254 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Fri, 17 Jul 2026 18:33:40 +0300 Subject: [PATCH 01/28] feat: enhance webhooks --- .agents/skills/grill-with-docs/SKILL.md | 7 + .../skills/grill-with-docs/agents/openai.yaml | 5 + .env.example | 3 + .secrets.baseline | 274 +++++++++++++++++- docker-compose.dev.yml | 17 ++ docker-compose.prod.yml | 5 + docker-compose.yml | 13 + docs/docs/tutorial/webhooks.md | 78 +++++ docs/mkdocs.yml | 1 + fastid/admin/factory.py | 2 + fastid/admin/views/settings.py | 48 ++- fastid/auth/router.py | 13 +- fastid/auth/use_cases.py | 2 +- fastid/core/lifespan.py | 2 - fastid/database/models.py | 3 +- fastid/database/uow.py | 4 +- fastid/profile/router.py | 43 ++- fastid/profile/use_cases.py | 8 +- fastid/security/webhooks.py | 62 ++++ fastid/webhooks/config.py | 17 ++ fastid/webhooks/metrics.py | 21 ++ fastid/webhooks/models.py | 54 +++- fastid/webhooks/repositories.py | 8 +- fastid/webhooks/senders/dependencies.py | 13 +- fastid/webhooks/senders/httpx.py | 164 ++++++++++- fastid/webhooks/use_cases.py | 65 ++--- fastid/webhooks/worker.py | 246 ++++++++++++++++ ...7_17_1700-7f3c0d2f18a1_durable_webhooks.py | 154 ++++++++++ skills-lock.json | 11 + tests/api/webhooks/test_worker_delivery.py | 109 +++++++ tests/api/webhooks/test_wrong_url.py | 6 +- tests/security/test_webhooks.py | 30 +- tests/webhooks/__init__.py | 0 tests/webhooks/test_sender.py | 51 ++++ tests/webhooks/test_worker.py | 18 ++ 35 files changed, 1429 insertions(+), 128 deletions(-) create mode 100644 .agents/skills/grill-with-docs/SKILL.md create mode 100644 .agents/skills/grill-with-docs/agents/openai.yaml create mode 100644 docs/docs/tutorial/webhooks.md create mode 100644 fastid/webhooks/metrics.py create mode 100644 fastid/webhooks/worker.py create mode 100644 migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py create mode 100644 skills-lock.json create mode 100644 tests/api/webhooks/test_worker_delivery.py create mode 100644 tests/webhooks/__init__.py create mode 100644 tests/webhooks/test_sender.py create mode 100644 tests/webhooks/test_worker.py diff --git a/.agents/skills/grill-with-docs/SKILL.md b/.agents/skills/grill-with-docs/SKILL.md new file mode 100644 index 0000000..bed05d2 --- /dev/null +++ b/.agents/skills/grill-with-docs/SKILL.md @@ -0,0 +1,7 @@ +--- +name: grill-with-docs +description: A relentless interview to sharpen a plan or design, which also creates docs (ADR's and glossary) as we go. +disable-model-invocation: true +--- + +Run a `/grilling` session, using the `/domain-modeling` skill. diff --git a/.agents/skills/grill-with-docs/agents/openai.yaml b/.agents/skills/grill-with-docs/agents/openai.yaml new file mode 100644 index 0000000..5dbe278 --- /dev/null +++ b/.agents/skills/grill-with-docs/agents/openai.yaml @@ -0,0 +1,5 @@ +interface: + display_name: "Grill with Docs" + short_description: "Grill a design and write its docs" +policy: + allow_implicit_invocation: false diff --git a/.env.example b/.env.example index 2aa8dd5..9a3a3be 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,5 @@ POSTGRES_PASSWORD="changethis" REDIS_PASSWORD="changethis" +FASTID_WEBHOOK_ALLOW_INSECURE_URLS=false +FASTID_WEBHOOK_WORKER_CONCURRENCY=20 +FASTID_WEBHOOK_WORKER_METRICS_PORT=9101 diff --git a/.secrets.baseline b/.secrets.baseline index 943aa6a..f1c540e 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -90,10 +90,6 @@ { "path": "detect_secrets.filters.allowlist.is_line_allowlisted" }, - { - "path": "detect_secrets.filters.common.is_baseline_file", - "filename": ".secrets.baseline" - }, { "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 @@ -124,12 +120,6 @@ }, { "path": "detect_secrets.filters.heuristic.is_templated_secret" - }, - { - "path": "detect_secrets.filters.regex.should_exclude_file", - "pattern": [ - "migrations/*" - ] } ], "results": { @@ -151,15 +141,269 @@ "line_number": 62 } ], - "app\\db\\config.py": [ + "migrations\\env.py": [ { "type": "Basic Auth Credentials", - "filename": "app\\db\\config.py", - "hashed_secret": "cdb0e76c1a69873cbdcdbe0a142d56c023dc9f22", + "filename": "migrations\\env.py", + "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", + "is_verified": false, + "line_number": 29 + } + ], + "migrations\\versions\\2024_08_07_2001-590f147f3dca_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_08_07_2001-590f147f3dca_.py", + "hashed_secret": "187df19f5f6a46ee1ff0cae4361df15b4da5a2c5", + "is_verified": false, + "line_number": 15 + } + ], + "migrations\\versions\\2024_09_05_2030-5ee810fadd8e_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_05_2030-5ee810fadd8e_.py", + "hashed_secret": "187df19f5f6a46ee1ff0cae4361df15b4da5a2c5", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2024_09_09_1744-bd54ebc87d2a_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_09_1744-bd54ebc87d2a_.py", + "hashed_secret": "cd11eeeb82f9c3b9d253b3816401681e5b144942", + "is_verified": false, + "line_number": 15 + } + ], + "migrations\\versions\\2024_09_09_1755-5781ebeb542c_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_09_1755-5781ebeb542c_.py", + "hashed_secret": "ad246c736071283e7d474bfe6eca50e8ec7f46c4", + "is_verified": false, + "line_number": 16 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_09_1755-5781ebeb542c_.py", + "hashed_secret": "cd11eeeb82f9c3b9d253b3816401681e5b144942", + "is_verified": false, + "line_number": 17 + } + ], + "migrations\\versions\\2024_09_09_2224-064a5ee97841_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_09_2224-064a5ee97841_.py", + "hashed_secret": "d4373f80f50cc750a1b12c410d81a70767a66a13", + "is_verified": false, + "line_number": 15 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_09_2224-064a5ee97841_.py", + "hashed_secret": "ad246c736071283e7d474bfe6eca50e8ec7f46c4", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2024_09_14_1413-f2979d2bcb1c_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_14_1413-f2979d2bcb1c_.py", + "hashed_secret": "d4373f80f50cc750a1b12c410d81a70767a66a13", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2024_09_18_2148-fa7d22042c64_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_09_18_2148-fa7d22042c64_.py", + "hashed_secret": "6f207de350188de6c62f4b710e8c7dfd675cbe79", + "is_verified": false, + "line_number": 15 + } + ], + "migrations\\versions\\2024_10_17_1951-9d66ac80082e_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_10_17_1951-9d66ac80082e_.py", + "hashed_secret": "dca2e21ab81cc009c7111b790ae4f4b281c64c66", + "is_verified": false, + "line_number": 15 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_10_17_1951-9d66ac80082e_.py", + "hashed_secret": "6f207de350188de6c62f4b710e8c7dfd675cbe79", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2024_10_17_1955-55aea0333e65_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_10_17_1955-55aea0333e65_.py", + "hashed_secret": "dca2e21ab81cc009c7111b790ae4f4b281c64c66", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2024_10_27_1628-f02ee2d85935_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_10_27_1628-f02ee2d85935_.py", + "hashed_secret": "5a8f330e2372f167e8f0ab64325e6b137b3a6b02", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2024_11_13_0105-3282fde15451_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_11_13_0105-3282fde15451_.py", + "hashed_secret": "80b5d0233910f84862e7becac6cfbcf1ab47cf77", + "is_verified": false, + "line_number": 16 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_11_13_0105-3282fde15451_.py", + "hashed_secret": "5a8f330e2372f167e8f0ab64325e6b137b3a6b02", + "is_verified": false, + "line_number": 17 + } + ], + "migrations\\versions\\2024_11_17_1914-446313a37068_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2024_11_17_1914-446313a37068_.py", + "hashed_secret": "80b5d0233910f84862e7becac6cfbcf1ab47cf77", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2026_01_08_1507-81c4895dd132_add_templates.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_08_1507-81c4895dd132_add_templates.py", + "hashed_secret": "64f464ac769660cc979aa0a46bdaff5dd5019145", + "is_verified": false, + "line_number": 15 + } + ], + "migrations\\versions\\2026_01_08_1511-32845277651e_remove_name_in_templates.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_08_1511-32845277651e_remove_name_in_templates.py", + "hashed_secret": "82949d609c9c90e3170a72222fb2df047f78417b", + "is_verified": false, + "line_number": 15 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_08_1511-32845277651e_remove_name_in_templates.py", + "hashed_secret": "64f464ac769660cc979aa0a46bdaff5dd5019145", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2026_01_14_1153-0b32faa598cb_add_notifications.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_14_1153-0b32faa598cb_add_notifications.py", + "hashed_secret": "80f5293392b82dd85bd8e7c0314049eaa186a900", + "is_verified": false, + "line_number": 15 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_14_1153-0b32faa598cb_add_notifications.py", + "hashed_secret": "82949d609c9c90e3170a72222fb2df047f78417b", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2026_01_16_2014-18fabf10cb0a_add_versioning.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_16_2014-18fabf10cb0a_add_versioning.py", + "hashed_secret": "80f5293392b82dd85bd8e7c0314049eaa186a900", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2026_01_19_1430-8d17be37457c_.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_19_1430-8d17be37457c_.py", + "hashed_secret": "34cce76d51334697020bca2d804f0dadcf3aff77", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2026_01_19_1529-03244d107aec_update_wh_type.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_19_1529-03244d107aec_update_wh_type.py", + "hashed_secret": "9a1a141064fdc7bedb9bc3567cf47781f99f1d98", + "is_verified": false, + "line_number": 15 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_01_19_1529-03244d107aec_update_wh_type.py", + "hashed_secret": "34cce76d51334697020bca2d804f0dadcf3aff77", + "is_verified": false, + "line_number": 16 + } + ], + "migrations\\versions\\2026_06_06_2007-3dc1a72c7ad5_add_client_id_idx.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_06_06_2007-3dc1a72c7ad5_add_client_id_idx.py", + "hashed_secret": "9a1a141064fdc7bedb9bc3567cf47781f99f1d98", + "is_verified": false, + "line_number": 15 + } + ], + "migrations\\versions\\2026_07_16_1933-bfa61bae871d_add_oauth_providers.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_07_16_1933-bfa61bae871d_add_oauth_providers.py", + "hashed_secret": "e07b058c212cb62219f3bc38c883997b989e1c1f", + "is_verified": false, + "line_number": 15 + } + ], + "migrations\\versions\\2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py": [ + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py", + "hashed_secret": "02700820903b32d3be49ae353c977dc39e91f9d2", + "is_verified": false, + "line_number": 14 + }, + { + "type": "Hex High Entropy String", + "filename": "migrations\\versions\\2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py", + "hashed_secret": "e07b058c212cb62219f3bc38c883997b989e1c1f", + "is_verified": false, + "line_number": 15 + } + ], + "skills-lock.json": [ + { + "type": "Hex High Entropy String", + "filename": "skills-lock.json", + "hashed_secret": "b90567df9a56aa46f4a821253aab48a9eebe4e26", "is_verified": false, - "line_number": 7 + "line_number": 8 } ] }, - "generated_at": "2026-05-29T10:11:37Z" + "generated_at": "2026-07-17T15:32:48Z" } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8cc7dd1..2c95ca7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -80,5 +80,22 @@ services: retries: 3 start_period: 40s + fastid-webhook-worker: + build: + context: . + dockerfile: docker/Dockerfile + target: dev + env_file: .env + entrypoint: ["poetry", "run", "python", "-m", "fastid.webhooks.worker"] + depends_on: + fastid-app: + condition: service_healthy + environment: + FASTID_DB_URL: ${FASTID_DB_URL:-postgresql+asyncpg://${POSTGRES_USER:-fastid}:${POSTGRES_PASSWORD:?database password required}@postgres:5432/${POSTGRES_DB:-fastid}} + FASTID_WEBHOOK_ALLOW_INSECURE_URLS: ${FASTID_WEBHOOK_ALLOW_INSECURE_URLS:-true} + volumes: + - "./fastid:/opt/fastid/fastid" + restart: unless-stopped + volumes: pg_data: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c437f2a..f30dfb3 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -4,3 +4,8 @@ services: context: . dockerfile: docker/Dockerfile target: prod + fastid-webhook-worker: + build: + context: . + dockerfile: docker/Dockerfile + target: prod diff --git a/docker-compose.yml b/docker-compose.yml index f59ffe3..45616c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,5 +67,18 @@ services: retries: 3 start_period: 40s + fastid-webhook-worker: + env_file: + - .env + image: everysoftware/fastid:latest + entrypoint: ["python", "-m", "fastid.webhooks.worker"] + depends_on: + fastid-app: + condition: service_healthy + environment: + FASTID_DB_URL: ${FASTID_DB_URL:-postgresql+asyncpg://${POSTGRES_USER:-fastid}:${POSTGRES_PASSWORD:?database password required}@postgres:5432/${POSTGRES_DB:-fastid}} + FASTID_WEBHOOK_ALLOW_INSECURE_URLS: ${FASTID_WEBHOOK_ALLOW_INSECURE_URLS:-false} + restart: unless-stopped + volumes: pg_data: diff --git a/docs/docs/tutorial/webhooks.md b/docs/docs/tutorial/webhooks.md new file mode 100644 index 0000000..09013a7 --- /dev/null +++ b/docs/docs/tutorial/webhooks.md @@ -0,0 +1,78 @@ +# Webhooks + +FastID sends user lifecycle events to every active endpoint subscribed to the event type. Delivery is asynchronous, +durable, and at least once: consumers must treat `webhook-id` as an idempotency key because a request can be repeated +after a timeout or worker crash. Delivery order is not guaranteed. + +## Request format + +Each request is a JSON `POST`. The existing `X-Webhook-*` headers remain available, and new integrations should verify +the Standard Webhooks headers: + +- `webhook-id`: logical event UUID, unchanged for retries. +- `webhook-timestamp`: Unix timestamp for the delivery attempt. +- `webhook-signature`: `v1,` signature. + +The signed value is the exact byte sequence `webhook-id.webhook-timestamp.raw_body`. Verify the raw request body before +parsing JSON; parsing and serializing it again can change the bytes. + +```python +from fastapi import FastAPI, Header, Request + +from fastid.security.webhooks import verify_standard_headers + +app = FastAPI() + + +@app.post("/fastid-webhooks") +async def receive(request: Request) -> dict[str, bool]: + body = await request.body() + if not verify_standard_headers(body, request.headers, "whsec_..."): + return {"accepted": False} + event = await request.json() + # Atomically record event["event"]["event_id"] before applying side effects. + return {"accepted": True} +``` + +The payload remains backward compatible: + +```json +{ + "event": { + "event_type": "user_registration", + "event_id": "019b...", + "timestamp": 1784293200 + }, + "data": { + "user": { + "id": "019b...", + "email": "person@example.com" + } + } +} +``` + +User events that historically exposed user fields directly in `data` continue to include those fields during the +compatibility period; use `data.user` in new consumers. + +## Delivery behavior + +A `2xx` response completes delivery. Other responses and network failures are retried with jittered exponential +backoff for roughly three days. FastID respects `Retry-After` up to 24 hours, never follows redirects, disables an +endpoint immediately after `410 Gone`, and disables it after all attempts are exhausted. + +The API transaction stores delivery records; a separate worker sends them: + +```console +python -m fastid.webhooks.worker +``` + +Docker Compose starts this worker automatically. Its Prometheus metrics are exposed on port `9101` inside the Compose +network. Delivery and attempt history is also visible in the FastID admin application. + +## Endpoint security + +Production endpoints must use HTTPS, contain no URL credentials, and resolve only to public IP addresses. FastID pins +the validated address for the connection and does not follow redirects. Keep the worker in an egress-restricted network +as defense in depth. Local HTTP and private-address endpoints require the explicit +`FASTID_WEBHOOK_ALLOW_INSECURE_URLS=true` development setting. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index dd32e45..ac4fc50 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -126,6 +126,7 @@ nav: - tutorial/index.md - tutorial/get_started.md - tutorial/notifications.md + - tutorial/webhooks.md - tutorial/social.md - tutorial/observability.md - tutorial/api.md diff --git a/fastid/admin/factory.py b/fastid/admin/factory.py index 13064ca..76acf76 100644 --- a/fastid/admin/factory.py +++ b/fastid/admin/factory.py @@ -13,6 +13,7 @@ OAuthProviderAdmin, TelegramTemplateAdmin, WebhookAdmin, + WebhookAttemptAdmin, WebhookEventAdmin, ) from fastid.admin.views.versioning import ( @@ -56,6 +57,7 @@ def create(self) -> FastAPI: admin.add_view(OAuthProviderAdmin) admin.add_view(WebhookAdmin) admin.add_view(WebhookEventAdmin) + admin.add_view(WebhookAttemptAdmin) admin.add_view(EmailTemplateAdmin) admin.add_view(TelegramTemplateAdmin) # Versioning diff --git a/fastid/admin/views/settings.py b/fastid/admin/views/settings.py index 9187cff..443532f 100644 --- a/fastid/admin/views/settings.py +++ b/fastid/admin/views/settings.py @@ -2,7 +2,15 @@ from fastid.admin.views.base import BaseView from fastid.admin.views.utils import json_format -from fastid.database.models import App, EmailTemplate, OAuthProvider, TelegramTemplate, Webhook, WebhookEvent +from fastid.database.models import ( + App, + EmailTemplate, + OAuthProvider, + TelegramTemplate, + Webhook, + WebhookAttempt, + WebhookEvent, +) class AppAdmin(BaseView, model=App): @@ -99,6 +107,8 @@ class WebhookAdmin(BaseView, model=Webhook): Webhook.id, Webhook.app, Webhook.url, + Webhook.is_active, + Webhook.disabled_reason, Webhook.created_at, Webhook.updated_at, ] @@ -106,6 +116,7 @@ class WebhookAdmin(BaseView, model=Webhook): OperationColumnFilter(Webhook.app_id), AllUniqueStringValuesFilter(Webhook.type), OperationColumnFilter(Webhook.url), + BooleanFilter(Webhook.is_active), ] @@ -114,23 +125,50 @@ class WebhookEventAdmin(BaseView, model=WebhookEvent): can_edit = False can_delete = False - name = "Webhook Event" - name_plural = "Webhook Events" + name = "Webhook Delivery" + name_plural = "Webhook Deliveries" icon = "fa-solid fa-book-atlas" category = "Settings" column_list = [ WebhookEvent.id, + WebhookEvent.event_id, WebhookEvent.webhook, "webhook.type", + WebhookEvent.status, + WebhookEvent.attempt_count, WebhookEvent.status_code, - WebhookEvent.request, WebhookEvent.response, + WebhookEvent.next_attempt_at, + WebhookEvent.completed_at, WebhookEvent.created_at, - WebhookEvent.updated_at, ] column_filters = [ OperationColumnFilter(WebhookEvent.webhook_id), OperationColumnFilter(WebhookEvent.status_code), + AllUniqueStringValuesFilter(WebhookEvent.status), ] column_formatters = {WebhookEvent.response: json_format, WebhookEvent.request: json_format} + + +class WebhookAttemptAdmin(BaseView, model=WebhookAttempt): + can_create = False + can_edit = False + can_delete = False + + name = "Webhook Attempt" + name_plural = "Webhook Attempts" + icon = "fa-solid fa-clock-rotate-left" + category = "Settings" + + column_list = [ + WebhookAttempt.id, + WebhookAttempt.delivery, + WebhookAttempt.attempt_number, + WebhookAttempt.status_code, + WebhookAttempt.error, + WebhookAttempt.duration_ms, + WebhookAttempt.created_at, + ] + column_filters = [OperationColumnFilter(WebhookAttempt.delivery_id)] + column_formatters = {WebhookAttempt.response: json_format, WebhookAttempt.request: json_format} diff --git a/fastid/auth/router.py b/fastid/auth/router.py index 8bc9409..9721a9b 100644 --- a/fastid/auth/router.py +++ b/fastid/auth/router.py @@ -43,10 +43,9 @@ async def register( user = await service.register(dto) notification = PushNotificationRequest(template="welcome") background.add_task(notify.push, user, notification) # pragma: nocover - webhook = SendWebhookRequest( - type=WebhookType.user_registration, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_registration, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user # pragma: nocover @@ -59,7 +58,6 @@ async def login( form: Annotated[OAuth2PasswordRequest, Form()], password_grant: Annotated[PasswordGrant, Depends()], webhooks: WebhooksDep, - background: BackgroundTasks, ) -> Any: match form.grant_type: case OAuth2Grant.password: @@ -68,7 +66,7 @@ async def login( raise NotSupportedGrantError assert auth_data.user is not None webhook = SendWebhookRequest(type=WebhookType.user_login, payload={"user": auth_data.user.model_dump(mode="json")}) - background.add_task(webhooks.send, webhook) # pragma: nocover + await webhooks.enqueue(webhook) return cookie_transport.get_login_response(auth_data.token) @@ -84,7 +82,6 @@ async def authorize( # noqa: PLR0913 refresh_token_grant: Annotated[RefreshTokenGrant, Depends()], client_credentials_grant: Annotated[ClientCredentialsGrant, Depends()], webhooks: WebhooksDep, - background: BackgroundTasks, ) -> Any: match form.grant_type: case OAuth2Grant.password: @@ -101,7 +98,7 @@ async def authorize( # noqa: PLR0913 webhook = SendWebhookRequest( type=WebhookType.user_login, payload={"user": auth_data.user.model_dump(mode="json")} ) - background.add_task(webhooks.send, webhook) # pragma: nocover + await webhooks.enqueue(webhook) return auth_data.token diff --git a/fastid/auth/use_cases.py b/fastid/auth/use_cases.py index 25628dc..3e178de 100644 --- a/fastid/auth/use_cases.py +++ b/fastid/auth/use_cases.py @@ -25,7 +25,7 @@ async def register(self, dto: UserCreate) -> User: raise UserAlreadyExistsError user = User.from_create(dto) user = await self.uow.users.add(user) - await self.uow.commit() + await self.uow.flush() return user async def get_userinfo(self, token: str, *, token_type: str = "access") -> User: # noqa: S107 diff --git a/fastid/core/lifespan.py b/fastid/core/lifespan.py index a21811d..72c9725 100644 --- a/fastid/core/lifespan.py +++ b/fastid/core/lifespan.py @@ -13,7 +13,6 @@ from fastid.notify.utils import collect_email_templates, collect_telegram_templates from fastid.oauth.models import OAUTH_PROVIDER_NAMES, OAuthProvider from fastid.oauth.repositories import OAuthProviderNameSpecification -from fastid.webhooks.senders.dependencies import client as webhooks_client class LifespanTasks: @@ -69,7 +68,6 @@ async def create_oauth_providers(self) -> None: async def on_shutdown(self) -> None: await self.cache.client.aclose() - await webhooks_client.aclose() async def __aenter__(self) -> Self: await self.uow.__aenter__() diff --git a/fastid/database/models.py b/fastid/database/models.py index e977e0a..e53317b 100644 --- a/fastid/database/models.py +++ b/fastid/database/models.py @@ -6,7 +6,7 @@ from fastid.database.base import BaseOrm from fastid.notify.models import EmailTemplate, Notification, TelegramTemplate from fastid.oauth.models import OAuthAccount, OAuthProvider -from fastid.webhooks.models import Webhook, WebhookEvent +from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookEvent configure_mappers() @@ -38,6 +38,7 @@ "Transaction", "WebhookVersion", "Webhook", + "WebhookAttempt", "WebhookEvent", "OAuthAccountVersion", "OAuthProviderVersion", diff --git a/fastid/database/uow.py b/fastid/database/uow.py index 3df0eb7..48d57a2 100644 --- a/fastid/database/uow.py +++ b/fastid/database/uow.py @@ -9,7 +9,7 @@ from fastid.auth.repositories import UserRepository from fastid.notify.repositories import EmailTemplateRepository, NotificationRepository, TelegramTemplateRepository from fastid.oauth.repositories import OAuthAccountRepository, OAuthProviderRepository -from fastid.webhooks.repositories import WebhookEventRepository, WebhookRepository +from fastid.webhooks.repositories import WebhookAttemptRepository, WebhookEventRepository, WebhookRepository if TYPE_CHECKING: from types import TracebackType @@ -30,6 +30,7 @@ class SQLAlchemyUOW: notifications: NotificationRepository webhooks: WebhookRepository webhook_events: WebhookEventRepository + webhook_attempts: WebhookAttemptRepository session_factory: async_sessionmaker[AsyncSession] session: AsyncSession @@ -48,6 +49,7 @@ async def begin(self) -> None: self.notifications = NotificationRepository(self.session) self.webhooks = WebhookRepository(self.session) self.webhook_events = WebhookEventRepository(self.session) + self.webhook_attempts = WebhookAttemptRepository(self.session) @property def is_active(self) -> bool: diff --git a/fastid/profile/router.py b/fastid/profile/router.py index 8ad2eff..7e5825a 100644 --- a/fastid/profile/router.py +++ b/fastid/profile/router.py @@ -1,6 +1,6 @@ from typing import Any -from fastapi import APIRouter, BackgroundTasks, status +from fastapi import APIRouter, status from fastid.auth.dependencies import UserDep, UserVTDep from fastid.auth.schemas import ( @@ -19,14 +19,11 @@ @router.patch("/users/me/profile", response_model=UserDTO, status_code=status.HTTP_200_OK) -async def patch( - service: ProfilesDep, user: UserDep, dto: UserUpdate, webhooks: WebhooksDep, background: BackgroundTasks -) -> Any: +async def patch(service: ProfilesDep, user: UserDep, dto: UserUpdate, webhooks: WebhooksDep) -> Any: user = await service.update_profile(user, dto) - webhook = SendWebhookRequest( - type=WebhookType.user_update_profile, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_update_profile, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user @@ -35,20 +32,18 @@ async def patch( response_model=UserDTO, status_code=status.HTTP_200_OK, ) -async def change_email( # noqa: PLR0913 +async def change_email( service: ProfilesDep, notify_service: NotifyDep, user: UserVTDep, dto: UserChangeEmail, webhooks: WebhooksDep, - background: BackgroundTasks, ) -> Any: await notify_service.validate_otp(user, dto.code) user = await service.change_email(user, dto) - webhook = SendWebhookRequest( - type=WebhookType.user_change_email, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_change_email, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user @@ -57,14 +52,11 @@ async def change_email( # noqa: PLR0913 response_model=UserDTO, status_code=status.HTTP_200_OK, ) -async def change_password( - service: ProfilesDep, user: UserVTDep, dto: UserChangePassword, webhooks: WebhooksDep, background: BackgroundTasks -) -> Any: +async def change_password(service: ProfilesDep, user: UserVTDep, dto: UserChangePassword, webhooks: WebhooksDep) -> Any: user = await service.change_password(user, dto) - webhook = SendWebhookRequest( - type=WebhookType.user_change_password, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_change_password, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user @@ -73,10 +65,9 @@ async def change_password( response_model=UserDTO, status_code=status.HTTP_200_OK, ) -async def delete(service: ProfilesDep, user: UserVTDep, webhooks: WebhooksDep, background: BackgroundTasks) -> Any: +async def delete(service: ProfilesDep, user: UserVTDep, webhooks: WebhooksDep) -> Any: user = await service.delete_account(user) - webhook = SendWebhookRequest( - type=WebhookType.user_delete, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_delete, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user diff --git a/fastid/profile/use_cases.py b/fastid/profile/use_cases.py index 9517fc2..e504ba7 100644 --- a/fastid/profile/use_cases.py +++ b/fastid/profile/use_cases.py @@ -32,7 +32,7 @@ async def update_profile( dto: UserUpdate, ) -> User: user.merge_model(dto) - await self.uow.commit() + await self.uow.flush() return user async def change_email(self, user: User, dto: UserChangeEmail) -> User: @@ -43,17 +43,17 @@ async def change_email(self, user: User, dto: UserChangeEmail) -> User: else: raise UserAlreadyExistsError user.change_email(dto.new_email) - await self.uow.commit() + await self.uow.flush() return user async def change_password(self, user: User, dto: UserChangePassword) -> User: user.set_password(dto.password) - await self.uow.commit() + await self.uow.flush() return user async def delete_account(self, user: User) -> User: user = await self.uow.users.delete(user) - await self.uow.commit() + await self.uow.flush() return user async def get_many(self, pagination: Pagination, sorting: Sorting) -> Page[User]: diff --git a/fastid/security/webhooks.py b/fastid/security/webhooks.py index 06de90c..798acad 100644 --- a/fastid/security/webhooks.py +++ b/fastid/security/webhooks.py @@ -1,11 +1,14 @@ +import base64 import hashlib import hmac import json import time +from collections.abc import Mapping from typing import Any from fastid.database.utils import UUIDv7, uuid from fastid.webhooks.config import webhook_settings +from fastid.webhooks.models import generate_webhook_secret from fastid.webhooks.schemas import SignatureAlgorithm HASH_FUNCTIONS = { @@ -14,6 +17,46 @@ SignatureAlgorithm.sha1: hashlib.sha1, } +STANDARD_ID_HEADER = "webhook-id" +STANDARD_TIMESTAMP_HEADER = "webhook-timestamp" +STANDARD_SIGNATURE_HEADER = "webhook-signature" + + +def serialize_payload(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def generate_secret() -> str: + return generate_webhook_secret() + + +def _secret_bytes(secret_key: str) -> bytes: + if not secret_key.startswith("whsec_"): + return secret_key.encode() + try: + return base64.b64decode(secret_key.removeprefix("whsec_"), validate=True) + except ValueError as exc: + msg = "Invalid whsec_ webhook secret" + raise ValueError(msg) from exc + + +def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + digest = hmac.new(_secret_bytes(secret_key), signed, hashlib.sha256).digest() + return f"v1,{base64.b64encode(digest).decode()}" + + +def generate_delivery_headers( # noqa: PLR0913 + payload: dict[str, Any], body: bytes, event_id: str, delivery_id: str, timestamp: int, secret_key: str +) -> dict[str, str]: + return generate_headers(payload, timestamp, delivery_id, secret_key) | { + STANDARD_ID_HEADER: event_id, + STANDARD_TIMESTAMP_HEADER: str(timestamp), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, event_id, timestamp, secret_key), + "Content-Type": "application/json", + "User-Agent": webhook_settings.user_agent, + } + def generate_headers( payload: dict[str, Any], @@ -69,6 +112,25 @@ def verify_headers( return hmac.compare_digest(received_signature, expected_signature) +def verify_standard_headers( + body: bytes, + headers: Mapping[str, str], + secret_key: str, + tolerance_seconds: int = webhook_settings.tolerance_seconds, +) -> bool: + normalized = {key.lower(): value for key, value in headers.items()} + try: + timestamp = int(normalized[STANDARD_TIMESTAMP_HEADER]) + webhook_id = normalized[STANDARD_ID_HEADER] + signatures = normalized[STANDARD_SIGNATURE_HEADER].split() + except (KeyError, ValueError): + return False + if not webhook_id or not signatures or not is_timestamp_valid(timestamp, tolerance_seconds): + return False + expected = generate_standard_signature(body, webhook_id, timestamp, secret_key) + return any(hmac.compare_digest(signature, expected) for signature in signatures) + + def get_event_id() -> UUIDv7: return uuid() diff --git a/fastid/webhooks/config.py b/fastid/webhooks/config.py index c49fa7e..85f8210 100644 --- a/fastid/webhooks/config.py +++ b/fastid/webhooks/config.py @@ -11,6 +11,23 @@ class WebhookSettings(BaseSettings): id_header: str = "X-Webhook-Id" tolerance_seconds: int = 300 page_expires_in_seconds: int = 60 + retry_delays_seconds: tuple[int, ...] = (0, 5, 300, 1800, 7200, 18000, 36000, 50400, 72000, 86400) + retry_jitter_ratio: float = 0.2 + retry_after_max_seconds: int = 86400 + worker_batch_size: int = 100 + worker_concurrency: int = 20 + worker_poll_seconds: float = 1.0 + worker_lease_seconds: int = 60 + worker_metrics_port: int = 9101 + connect_timeout_seconds: float = 5.0 + read_timeout_seconds: float = 20.0 + write_timeout_seconds: float = 5.0 + pool_timeout_seconds: float = 5.0 + max_connections: int = 100 + max_keepalive_connections: int = 20 + max_response_bytes: int = 65536 + allow_insecure_urls: bool = False + user_agent: str = "FastID-Webhooks/0.1" model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX}webhook_") diff --git a/fastid/webhooks/metrics.py b/fastid/webhooks/metrics.py new file mode 100644 index 0000000..e972b6c --- /dev/null +++ b/fastid/webhooks/metrics.py @@ -0,0 +1,21 @@ +from prometheus_client import Counter, Gauge, Histogram + +ATTEMPTS = Counter( + "fastid_webhook_attempts_total", + "Webhook delivery attempts by event type and outcome.", + ["event_type", "outcome"], +) +ATTEMPT_DURATION = Histogram( + "fastid_webhook_attempt_duration_seconds", + "Webhook delivery attempt duration.", + ["event_type"], +) +DUE_DELIVERIES = Gauge( + "fastid_webhook_due_deliveries", + "Webhook deliveries currently due for processing.", +) +DISABLED_ENDPOINTS = Counter( + "fastid_webhook_disabled_endpoints_total", + "Webhook endpoints disabled by reason.", + ["reason"], +) diff --git a/fastid/webhooks/models.py b/fastid/webhooks/models.py index a6f5185..15a5a06 100644 --- a/fastid/webhooks/models.py +++ b/fastid/webhooks/models.py @@ -1,10 +1,13 @@ from __future__ import annotations +import base64 +import datetime # noqa: TCH003 - SQLAlchemy resolves mapped annotations at runtime +import secrets from enum import auto from typing import TYPE_CHECKING, Any from uuid import UUID # noqa: TCH003 -from sqlalchemy import ForeignKey +from sqlalchemy import ForeignKey, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from fastid.core.schemas import BaseEnum @@ -14,6 +17,10 @@ from fastid.apps.models import App +def generate_webhook_secret() -> str: + return f"whsec_{base64.b64encode(secrets.token_bytes(32)).decode()}" + + class WebhookType(BaseEnum): user_registration = auto() user_login = auto() @@ -23,13 +30,24 @@ class WebhookType(BaseEnum): user_delete = auto() +class WebhookDeliveryStatus(BaseEnum): + pending = auto() + processing = auto() + succeeded = auto() + exhausted = auto() + cancelled = auto() + + class Webhook(VersionedEntity): __tablename__ = "webhooks" app_id: Mapped[UUID] = mapped_column(ForeignKey("apps.id"), index=True) type: Mapped[WebhookType] - secret: Mapped[str] + secret: Mapped[str] = mapped_column(default=generate_webhook_secret) url: Mapped[str] + is_active: Mapped[bool] = mapped_column(default=True) + disabled_at: Mapped[datetime.datetime | None] + disabled_reason: Mapped[str | None] app: Mapped[App] = relationship(back_populates="webhooks") webhook_events: Mapped[list[WebhookEvent]] = relationship(back_populates="webhook", cascade="delete") @@ -39,8 +57,34 @@ class WebhookEvent(Entity): __tablename__ = "webhook_events" webhook_id: Mapped[UUID] = mapped_column(ForeignKey("webhooks.id"), index=True) - request: Mapped[dict[str, Any]] - status_code: Mapped[int] - response: Mapped[dict[str, Any]] + event_id: Mapped[UUID] = mapped_column(index=True) + event_type: Mapped[WebhookType] + payload: Mapped[dict[str, Any]] + status: Mapped[WebhookDeliveryStatus] = mapped_column(default=WebhookDeliveryStatus.pending, index=True) + attempt_count: Mapped[int] = mapped_column(default=0) + next_attempt_at: Mapped[datetime.datetime] = mapped_column(index=True) + leased_until: Mapped[datetime.datetime | None] = mapped_column(index=True) + completed_at: Mapped[datetime.datetime | None] + request: Mapped[dict[str, Any] | None] + status_code: Mapped[int | None] + response: Mapped[dict[str, Any] | None] + error: Mapped[str | None] webhook: Mapped[Webhook] = relationship(back_populates="webhook_events") + attempts: Mapped[list[WebhookAttempt]] = relationship(back_populates="delivery", cascade="delete") + + +class WebhookAttempt(Entity): + __tablename__ = "webhook_attempts" + __table_args__ = (UniqueConstraint("delivery_id", "attempt_number"),) + + delivery_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_events.id"), index=True) + attempt_number: Mapped[int] + timestamp: Mapped[int] + request: Mapped[dict[str, Any]] + status_code: Mapped[int | None] + response: Mapped[dict[str, Any] | None] + error: Mapped[str | None] + duration_ms: Mapped[int] + + delivery: Mapped[WebhookEvent] = relationship(back_populates="attempts") diff --git a/fastid/webhooks/repositories.py b/fastid/webhooks/repositories.py index 2986275..a9b2dd6 100644 --- a/fastid/webhooks/repositories.py +++ b/fastid/webhooks/repositories.py @@ -3,7 +3,7 @@ from fastid.database.repository import SQLAlchemyRepository from fastid.database.specification import Specification from fastid.database.utils import UUIDv7 -from fastid.webhooks.models import Webhook, WebhookEvent, WebhookType +from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookEvent, WebhookType class WebhookRepository(SQLAlchemyRepository[Webhook]): @@ -15,13 +15,17 @@ def __init__(self, webhook_type: WebhookType) -> None: self.type = webhook_type def apply(self, stmt: Any) -> Any: - return stmt.where(Webhook.type == self.type) + return stmt.where(Webhook.type == self.type, Webhook.is_active.is_(True)) class WebhookEventRepository(SQLAlchemyRepository[WebhookEvent]): model_type = WebhookEvent +class WebhookAttemptRepository(SQLAlchemyRepository[WebhookAttempt]): + model_type = WebhookAttempt + + class WebhookEventWebhookIDSpecification(Specification): def __init__(self, webhook_id: UUIDv7) -> None: self.webhook_id = webhook_id diff --git a/fastid/webhooks/senders/dependencies.py b/fastid/webhooks/senders/dependencies.py index 964a445..342c109 100644 --- a/fastid/webhooks/senders/dependencies.py +++ b/fastid/webhooks/senders/dependencies.py @@ -3,9 +3,20 @@ import httpx from fastapi import Depends +from fastid.webhooks.config import webhook_settings from fastid.webhooks.senders.httpx import WebhookSender -client = httpx.AsyncClient() +timeout = httpx.Timeout( + webhook_settings.read_timeout_seconds, + connect=webhook_settings.connect_timeout_seconds, + write=webhook_settings.write_timeout_seconds, + pool=webhook_settings.pool_timeout_seconds, +) +limits = httpx.Limits( + max_connections=webhook_settings.max_connections, + max_keepalive_connections=webhook_settings.max_keepalive_connections, +) +client = httpx.AsyncClient(timeout=timeout, limits=limits, follow_redirects=False, trust_env=False) def get_sender() -> WebhookSender: diff --git a/fastid/webhooks/senders/httpx.py b/fastid/webhooks/senders/httpx.py index 5afb33a..6d0e30c 100644 --- a/fastid/webhooks/senders/httpx.py +++ b/fastid/webhooks/senders/httpx.py @@ -1,19 +1,163 @@ -from typing import Any +import asyncio +import ipaddress +import json +import socket +import time +from dataclasses import dataclass +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime import httpx +from fastid.webhooks.config import webhook_settings + +HTTPS_PORT = 443 + + +class UnsafeWebhookURLError(ValueError): + pass + + +@dataclass(frozen=True) +class ResolvedTarget: + url: httpx.URL + host_header: str + sni_hostname: str | None + + +@dataclass(frozen=True) +class WebhookResponse: + status_code: int + content: dict[str, object] + error: str | None + retry_after_seconds: int | None + duration_ms: int + + +async def resolve_targets(value: str) -> list[ResolvedTarget]: + url = httpx.URL(value) + if not url.host: + msg = "Webhook URL must include a host" + raise UnsafeWebhookURLError(msg) + if url.username or url.password: + msg = "Webhook URL must not include credentials" + raise UnsafeWebhookURLError(msg) + if webhook_settings.allow_insecure_urls: + return [ResolvedTarget(url=url, host_header=_host_header(url), sni_hostname=url.host)] + if url.scheme != "https": + msg = "Webhook URL must use HTTPS" + raise UnsafeWebhookURLError(msg) + + port = url.port or HTTPS_PORT + loop = asyncio.get_running_loop() + try: + records = await loop.getaddrinfo(url.host, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + msg = f"Webhook host could not be resolved: {exc}" + raise UnsafeWebhookURLError(msg) from exc + addresses = list(dict.fromkeys(record[4][0].split("%", 1)[0] for record in records)) + if not addresses: + msg = "Webhook host did not resolve to an address" + raise UnsafeWebhookURLError(msg) + parsed = [ipaddress.ip_address(address) for address in addresses] + if any(not address.is_global for address in parsed): + msg = "Webhook host resolves to a non-public address" + raise UnsafeWebhookURLError(msg) + return [ + ResolvedTarget(url=url.copy_with(host=str(address)), host_header=_host_header(url), sni_hostname=url.host) + for address in parsed + ] + + +def _host_header(url: httpx.URL) -> str: + host = url.host + if ":" in host and not host.startswith("["): + host = f"[{host}]" + if url.port and url.port != HTTPS_PORT: + return f"{host}:{url.port}" + return host + + +def _retry_after(value: str | None) -> int | None: + if value is None: + return None + try: + return max(0, int(value)) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=UTC) + return max(0, int((retry_at - datetime.now(UTC)).total_seconds())) + + +def _content(raw: bytes, *, truncated: bool) -> dict[str, object]: + text = raw.decode(errors="replace") + try: + value = json.loads(text) if text else None + except json.JSONDecodeError: + value = text + content: dict[str, object] = value if isinstance(value, dict) else {"value": value} + if truncated: + content["truncated"] = True + return content + class WebhookSender: def __init__(self, client: httpx.AsyncClient) -> None: self.client = client - async def send(self, url: str, payload: dict[str, Any], headers: dict[str, str]) -> dict[str, Any]: - try: - response = await self.client.post(url, json=payload, headers=headers) - except httpx.RequestError as e: - return {"status_code": 0, "content": {"error": str(e)}} + async def send(self, url: str, body: bytes, headers: dict[str, str]) -> WebhookResponse: + started = time.monotonic() try: - content = response.json() - except Exception: # noqa: BLE001 - content = {"text": response.text} - return {"status_code": response.status_code, "content": content} + targets = await resolve_targets(url) + except UnsafeWebhookURLError as exc: + return self._error(exc, started) + + last_error: httpx.RequestError | None = None + for target in targets: + request_headers = headers | {"Host": target.host_header} + extensions = {"sni_hostname": target.sni_hostname} if target.sni_hostname else None + request = self.client.build_request( + "POST", target.url, content=body, headers=request_headers, extensions=extensions + ) + try: + response = await self.client.send(request, stream=True, follow_redirects=False) + except httpx.RequestError as exc: + last_error = exc + continue + try: + raw = bytearray() + truncated = False + async for chunk in response.aiter_bytes(): + remaining = webhook_settings.max_response_bytes - len(raw) + if remaining <= 0: + truncated = True + break + raw.extend(chunk[:remaining]) + if len(chunk) > remaining: + truncated = True + break + finally: + await response.aclose() + return WebhookResponse( + status_code=response.status_code, + content=_content(bytes(raw), truncated=truncated), + error=None, + retry_after_seconds=_retry_after(response.headers.get("Retry-After")), + duration_ms=int((time.monotonic() - started) * 1000), + ) + assert last_error is not None + return self._error(last_error, started) + + @staticmethod + def _error(exc: Exception, started: float) -> WebhookResponse: + return WebhookResponse( + status_code=0, + content={"error": str(exc)}, + error=str(exc), + retry_after_seconds=None, + duration_ms=int((time.monotonic() - started) * 1000), + ) diff --git a/fastid/webhooks/use_cases.py b/fastid/webhooks/use_cases.py index 5a77f4d..d136139 100644 --- a/fastid/webhooks/use_cases.py +++ b/fastid/webhooks/use_cases.py @@ -1,56 +1,35 @@ -from fastid.cache.dependencies import CacheDep -from fastid.cache.exceptions import KeyNotFoundError from fastid.core.base import UseCase -from fastid.database.dependencies import UOWRawDep, transactional -from fastid.security.webhooks import ( - generate_headers, - get_event_id, - get_timestamp, - get_webhook_id, -) -from fastid.webhooks.config import webhook_settings -from fastid.webhooks.models import Webhook, WebhookEvent +from fastid.database.dependencies import UOWDep +from fastid.database.utils import naive_utc +from fastid.security.webhooks import get_event_id, get_timestamp, get_webhook_id +from fastid.webhooks.models import WebhookDeliveryStatus, WebhookEvent from fastid.webhooks.repositories import WebhookTypeSpecification from fastid.webhooks.schemas import Event, SendWebhookRequest, WebhookPayload -from fastid.webhooks.senders.dependencies import SenderDep class WebhookUseCases(UseCase): - def __init__(self, uow: UOWRawDep, cache: CacheDep, sender: SenderDep) -> None: - self.uow = uow # Due to background nature of notification use cases, use raw dependency - self.cache = cache - self.sender = sender + def __init__(self, uow: UOWDep) -> None: + self.uow = uow - @transactional - async def send(self, dto: SendWebhookRequest) -> None: - cache_key = f"webhooks:{dto.type}" - try: - cached = await self.cache.get(cache_key) - except KeyNotFoundError: - webhook_page = await self.uow.webhooks.get_many(WebhookTypeSpecification(dto.type)) - webhooks = webhook_page.items - await self.cache.set( - cache_key, - [{"id": str(w.id), "secret": w.secret, "url": w.url} for w in webhooks], - expire=webhook_settings.page_expires_in_seconds, - ) - else: - webhooks = [Webhook(**w) for w in cached] + async def enqueue(self, dto: SendWebhookRequest) -> None: + webhook_page = await self.uow.webhooks.get_many(WebhookTypeSpecification(dto.type)) event_id = get_event_id() event_timestamp = get_timestamp() event_dto = Event(event_type=dto.type, event_id=event_id, timestamp=event_timestamp) payload = WebhookPayload(event=event_dto, data=dto.payload).model_dump(mode="json") - for webhook in webhooks: - webhook_id = get_webhook_id() - timestamp = get_timestamp() - headers = generate_headers(payload, timestamp, str(webhook_id), webhook.secret) - request = {"headers": headers, "body": payload} - data = await self.sender.send(webhook.url, payload, headers) - event = WebhookEvent( - id=webhook_id, + now = naive_utc() + for webhook in webhook_page.items: + delivery = WebhookEvent( + id=get_webhook_id(), webhook_id=webhook.id, - request=request, - status_code=data["status_code"], - response=data["content"], + event_id=event_id, + event_type=dto.type, + payload=payload, + status=WebhookDeliveryStatus.pending, + next_attempt_at=now, ) - await self.uow.webhook_events.add(event) + await self.uow.webhook_events.add(delivery) + + async def send(self, dto: SendWebhookRequest) -> None: + """Backward-compatible alias for enqueueing a durable delivery.""" + await self.enqueue(dto) diff --git a/fastid/webhooks/worker.py b/fastid/webhooks/worker.py new file mode 100644 index 0000000..f35941a --- /dev/null +++ b/fastid/webhooks/worker.py @@ -0,0 +1,246 @@ +import asyncio +import logging +import secrets +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timedelta +from uuid import UUID + +from prometheus_client import start_http_server +from sqlalchemy import func, or_, select, update +from sqlalchemy.orm import joinedload + +from fastid.database.dependencies import get_uow_raw +from fastid.database.uow import SQLAlchemyUOW +from fastid.database.utils import naive_utc +from fastid.security.webhooks import generate_delivery_headers, get_timestamp, serialize_payload +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.metrics import ATTEMPT_DURATION, ATTEMPTS, DISABLED_ENDPOINTS, DUE_DELIVERIES +from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDeliveryStatus, WebhookEvent +from fastid.webhooks.senders.dependencies import client +from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender + +log = logging.getLogger(__name__) +HTTP_OK = 200 +HTTP_MULTIPLE_CHOICES = 300 +HTTP_GONE = 410 +RANDOM = secrets.SystemRandom() + + +def get_retry_delay(attempt_number: int, retry_after_seconds: int | None, *, jitter: float | None = None) -> int: + delay = webhook_settings.retry_delays_seconds[attempt_number] + if jitter is None: + jitter = RANDOM.uniform(1 - webhook_settings.retry_jitter_ratio, 1 + webhook_settings.retry_jitter_ratio) + delay = max(0, int(delay * jitter)) + if retry_after_seconds is not None: + retry_after = min(retry_after_seconds, webhook_settings.retry_after_max_seconds) + delay = max(delay, retry_after) + return delay + + +@dataclass(frozen=True) +class ClaimedDelivery: + id: UUID + event_id: UUID + event_type: str + payload: dict[str, object] + endpoint_url: str + endpoint_secret: str + + +class WebhookWorker: + def __init__(self, sender: WebhookSender | None = None) -> None: + self.sender = sender or WebhookSender(client) + self.stopping = asyncio.Event() + self.semaphore = asyncio.Semaphore(webhook_settings.worker_concurrency) + + async def run(self) -> None: + log.info("Webhook worker started") + while not self.stopping.is_set(): + count = await self.run_once() + if count == 0: + with suppress(TimeoutError): + await asyncio.wait_for(self.stopping.wait(), timeout=webhook_settings.worker_poll_seconds) + log.info("Webhook worker stopped") + + async def run_once(self) -> int: + deliveries = await self._claim() + await asyncio.gather(*(self._process_with_limit(delivery) for delivery in deliveries)) + return len(deliveries) + + async def _claim(self) -> list[ClaimedDelivery]: + now = naive_utc() + lease = now + timedelta(seconds=webhook_settings.worker_lease_seconds) + uow = get_uow_raw() + async with uow: + stmt = ( + select(WebhookEvent) + .options(joinedload(WebhookEvent.webhook)) + .where( + WebhookEvent.next_attempt_at <= now, + or_( + WebhookEvent.status == WebhookDeliveryStatus.pending, + (WebhookEvent.status == WebhookDeliveryStatus.processing) & (WebhookEvent.leased_until <= now), + ), + ) + .order_by(WebhookEvent.next_attempt_at, WebhookEvent.created_at) + .limit(webhook_settings.worker_batch_size) + .with_for_update(skip_locked=True, of=WebhookEvent) + ) + rows = list((await uow.session.scalars(stmt)).all()) + due_count = await uow.session.scalar( + select(func.count()) + .select_from(WebhookEvent) + .where( + WebhookEvent.next_attempt_at <= now, + or_( + WebhookEvent.status == WebhookDeliveryStatus.pending, + (WebhookEvent.status == WebhookDeliveryStatus.processing) & (WebhookEvent.leased_until <= now), + ), + ) + ) + DUE_DELIVERIES.set(due_count or 0) + claimed: list[ClaimedDelivery] = [] + for delivery in rows: + if not delivery.webhook.is_active: + delivery.status = WebhookDeliveryStatus.cancelled + delivery.completed_at = now + delivery.error = "endpoint disabled" + continue + delivery.status = WebhookDeliveryStatus.processing + delivery.leased_until = lease + claimed.append( + ClaimedDelivery( + id=delivery.id, + event_id=delivery.event_id, + event_type=str(delivery.event_type), + payload=delivery.payload, + endpoint_url=delivery.webhook.url, + endpoint_secret=delivery.webhook.secret, + ) + ) + return claimed + return [] # pragma: no cover - the unit of work context always enters + + async def _process_with_limit(self, delivery: ClaimedDelivery) -> None: + async with self.semaphore: + await self._process(delivery) + + async def _process(self, delivery: ClaimedDelivery) -> None: + timestamp = get_timestamp() + body = serialize_payload(delivery.payload) + headers = generate_delivery_headers( + delivery.payload, + body, + str(delivery.event_id), + str(delivery.id), + timestamp, + delivery.endpoint_secret, + ) + response = await self.sender.send(delivery.endpoint_url, body, headers) + await self._record(delivery.id, delivery.event_type, timestamp, headers, response) + + async def _record( + self, + delivery_id: UUID, + event_type: str, + timestamp: int, + headers: dict[str, str], + response: WebhookResponse, + ) -> None: + now = naive_utc() + uow = get_uow_raw() + async with uow: + delivery = await uow.webhook_events.get(delivery_id) + endpoint = await uow.webhooks.get(delivery.webhook_id) + attempt_number = delivery.attempt_count + 1 + stored_headers = { + key: "[redacted]" if "signature" in key.lower() else value for key, value in headers.items() + } + request = {"headers": stored_headers, "body": delivery.payload} + attempt = WebhookAttempt( + delivery_id=delivery.id, + attempt_number=attempt_number, + timestamp=timestamp, + request=request, + status_code=response.status_code, + response=response.content, + error=response.error, + duration_ms=response.duration_ms, + ) + await uow.webhook_attempts.add(attempt) + delivery.attempt_count = attempt_number + delivery.request = request + delivery.status_code = response.status_code + delivery.response = response.content + delivery.error = response.error + delivery.leased_until = None + + if HTTP_OK <= response.status_code < HTTP_MULTIPLE_CHOICES: + delivery.status = WebhookDeliveryStatus.succeeded + delivery.completed_at = now + outcome = "succeeded" + elif response.status_code == HTTP_GONE: + await self._disable(uow, endpoint, delivery, now, "endpoint returned 410 Gone") + outcome = "gone" + DISABLED_ENDPOINTS.labels(reason="gone").inc() + elif attempt_number >= len(webhook_settings.retry_delays_seconds): + await self._disable(uow, endpoint, delivery, now, "delivery retries exhausted") + outcome = "exhausted" + DISABLED_ENDPOINTS.labels(reason="exhausted").inc() + else: + delay = get_retry_delay(attempt_number, response.retry_after_seconds) + delivery.status = WebhookDeliveryStatus.pending + delivery.next_attempt_at = now + timedelta(seconds=delay) + outcome = "retry" + ATTEMPTS.labels(event_type=event_type, outcome=outcome).inc() + ATTEMPT_DURATION.labels(event_type=event_type).observe(response.duration_ms / 1000) + log.info( + "Webhook attempt completed: delivery_id=%s event_id=%s attempt=%d status_code=%d state=%s duration_ms=%d", + delivery.id, + delivery.event_id, + attempt_number, + response.status_code, + delivery.status, + response.duration_ms, + ) + + @staticmethod + async def _disable( + uow: SQLAlchemyUOW, endpoint: Webhook, delivery: WebhookEvent, now: datetime, reason: str + ) -> None: + delivery.status = ( + WebhookDeliveryStatus.exhausted if delivery.status_code != HTTP_GONE else WebhookDeliveryStatus.cancelled + ) + delivery.completed_at = now + endpoint.is_active = False + endpoint.disabled_at = now + endpoint.disabled_reason = reason + await uow.session.execute( + update(WebhookEvent) + .where( + WebhookEvent.webhook_id == endpoint.id, + WebhookEvent.id != delivery.id, + WebhookEvent.status.in_((WebhookDeliveryStatus.pending, WebhookDeliveryStatus.processing)), + ) + .values(status=WebhookDeliveryStatus.cancelled, completed_at=now, error="endpoint disabled") + ) + + +async def run_worker() -> None: + if webhook_settings.worker_metrics_port > 0: + start_http_server(webhook_settings.worker_metrics_port) + worker = WebhookWorker() + try: + await worker.run() + finally: + await client.aclose() + + +def main() -> None: + with suppress(KeyboardInterrupt): + asyncio.run(run_worker()) + + +if __name__ == "__main__": + main() diff --git a/migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py b/migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py new file mode 100644 index 0000000..bd30671 --- /dev/null +++ b/migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py @@ -0,0 +1,154 @@ +"""durable webhooks + +Revision ID: 7f3c0d2f18a1 +Revises: bfa61bae871d +Create Date: 2026-07-17 17:00:00 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "7f3c0d2f18a1" +down_revision: str | None = "bfa61bae871d" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +webhook_type = sa.Enum( + "user_registration", + "user_login", + "user_update_profile", + "user_change_email", + "user_change_password", + "user_delete", + name="webhooktype", + native_enum=False, +) +delivery_status = sa.Enum( + "pending", + "processing", + "succeeded", + "exhausted", + "cancelled", + name="webhookdeliverystatus", + native_enum=False, +) + + +def upgrade() -> None: + op.add_column("webhooks", sa.Column("is_active", sa.Boolean(), server_default=sa.true(), nullable=False)) + op.add_column("webhooks", sa.Column("disabled_at", sa.DateTime(), nullable=True)) + op.add_column("webhooks", sa.Column("disabled_reason", sa.String(), nullable=True)) + op.add_column("webhooks_version", sa.Column("is_active", sa.Boolean(), nullable=True)) + op.add_column("webhooks_version", sa.Column("disabled_at", sa.DateTime(), nullable=True)) + op.add_column("webhooks_version", sa.Column("disabled_reason", sa.String(), nullable=True)) + + op.add_column("webhook_events", sa.Column("event_id", sa.Uuid(), nullable=True)) + op.add_column("webhook_events", sa.Column("event_type", webhook_type, nullable=True)) + op.add_column("webhook_events", sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column("webhook_events", sa.Column("status", delivery_status, nullable=True)) + op.add_column("webhook_events", sa.Column("attempt_count", sa.BigInteger(), server_default="0", nullable=False)) + op.add_column("webhook_events", sa.Column("next_attempt_at", sa.DateTime(), nullable=True)) + op.add_column("webhook_events", sa.Column("leased_until", sa.DateTime(), nullable=True)) + op.add_column("webhook_events", sa.Column("completed_at", sa.DateTime(), nullable=True)) + op.add_column("webhook_events", sa.Column("error", sa.String(), nullable=True)) + op.alter_column("webhook_events", "request", existing_type=postgresql.JSONB(), nullable=True) + op.alter_column("webhook_events", "status_code", existing_type=sa.BigInteger(), nullable=True) + op.alter_column("webhook_events", "response", existing_type=postgresql.JSONB(), nullable=True) + + op.execute( + """ + UPDATE webhook_events AS delivery + SET event_id = delivery.id, + event_type = endpoint.type, + payload = COALESCE(delivery.request -> 'body', '{}'::jsonb), + status = CASE + WHEN delivery.status_code BETWEEN 200 AND 299 THEN 'succeeded' + ELSE 'exhausted' + END, + attempt_count = 1, + next_attempt_at = delivery.created_at, + completed_at = delivery.updated_at, + error = CASE WHEN delivery.status_code = 0 THEN delivery.response ->> 'error' ELSE NULL END + FROM webhooks AS endpoint + WHERE endpoint.id = delivery.webhook_id + """ + ) + op.alter_column("webhook_events", "event_id", nullable=False) + op.alter_column("webhook_events", "event_type", nullable=False) + op.alter_column("webhook_events", "payload", nullable=False) + op.alter_column("webhook_events", "status", nullable=False) + op.alter_column("webhook_events", "next_attempt_at", nullable=False) + + op.create_index(op.f("webhook_events_event_id_idx"), "webhook_events", ["event_id"]) + op.create_index(op.f("webhook_events_status_idx"), "webhook_events", ["status"]) + op.create_index(op.f("webhook_events_next_attempt_at_idx"), "webhook_events", ["next_attempt_at"]) + op.create_index(op.f("webhook_events_leased_until_idx"), "webhook_events", ["leased_until"]) + + op.create_table( + "webhook_attempts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("delivery_id", sa.Uuid(), nullable=False), + sa.Column("attempt_number", sa.BigInteger(), nullable=False), + sa.Column("timestamp", sa.BigInteger(), nullable=False), + sa.Column("request", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("status_code", sa.BigInteger(), nullable=True), + sa.Column("response", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("error", sa.String(), nullable=True), + sa.Column("duration_ms", sa.BigInteger(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["delivery_id"], ["webhook_events.id"], name=op.f("webhook_attempts_delivery_id_fkey")), + sa.PrimaryKeyConstraint("id", name=op.f("webhook_attempts_pkey")), + sa.UniqueConstraint("delivery_id", "attempt_number", name=op.f("webhook_attempts_delivery_id_key")), + ) + op.create_index(op.f("webhook_attempts_delivery_id_idx"), "webhook_attempts", ["delivery_id"]) + op.execute( + """ + INSERT INTO webhook_attempts ( + id, delivery_id, attempt_number, timestamp, request, status_code, + response, error, duration_ms, created_at, updated_at + ) + SELECT id, id, 1, EXTRACT(EPOCH FROM created_at)::bigint, request, + status_code, response, error, 0, created_at, updated_at + FROM webhook_events + """ + ) + + +def downgrade() -> None: + op.drop_index(op.f("webhook_attempts_delivery_id_idx"), table_name="webhook_attempts") + op.drop_table("webhook_attempts") + op.execute( + """ + UPDATE webhook_events + SET request = COALESCE(request, jsonb_build_object('headers', '{}'::jsonb, 'body', payload)), + status_code = COALESCE(status_code, 0), + response = COALESCE(response, jsonb_build_object('error', COALESCE(error, 'not delivered'))) + """ + ) + op.alter_column("webhook_events", "response", existing_type=postgresql.JSONB(), nullable=False) + op.alter_column("webhook_events", "status_code", existing_type=sa.BigInteger(), nullable=False) + op.alter_column("webhook_events", "request", existing_type=postgresql.JSONB(), nullable=False) + op.drop_index(op.f("webhook_events_leased_until_idx"), table_name="webhook_events") + op.drop_index(op.f("webhook_events_next_attempt_at_idx"), table_name="webhook_events") + op.drop_index(op.f("webhook_events_status_idx"), table_name="webhook_events") + op.drop_index(op.f("webhook_events_event_id_idx"), table_name="webhook_events") + for column in ( + "error", + "completed_at", + "leased_until", + "next_attempt_at", + "attempt_count", + "status", + "payload", + "event_type", + "event_id", + ): + op.drop_column("webhook_events", column) + for table in ("webhooks_version", "webhooks"): + op.drop_column(table, "disabled_reason") + op.drop_column(table, "disabled_at") + op.drop_column(table, "is_active") diff --git a/skills-lock.json b/skills-lock.json new file mode 100644 index 0000000..b18d465 --- /dev/null +++ b/skills-lock.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "skills": { + "grill-with-docs": { + "source": "mattpocock/skills", + "sourceType": "github", + "skillPath": "skills/engineering/grill-with-docs/SKILL.md", + "computedHash": "96718e0a89e48c7d76cc9759c7094a82ff34eb89ddc46a30388063fa9812356f" + } + } +} diff --git a/tests/api/webhooks/test_worker_delivery.py b/tests/api/webhooks/test_worker_delivery.py new file mode 100644 index 0000000..11855b4 --- /dev/null +++ b/tests/api/webhooks/test_worker_delivery.py @@ -0,0 +1,109 @@ +import asyncio +from datetime import timedelta + +import pytest +from httpx import AsyncClient +from starlette import status + +import fastid.webhooks.worker as worker_module +from fastid.database.uow import SQLAlchemyUOW +from fastid.database.utils import naive_utc +from fastid.security.webhooks import verify_standard_headers +from fastid.webhooks.models import Webhook, WebhookDeliveryStatus +from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender +from fastid.webhooks.worker import WebhookWorker +from tests.dependencies import get_test_uow +from tests.mocks import USER_CREATE + +HTTP_MULTIPLE_CHOICES = 300 + + +class StubSender(WebhookSender): + def __init__(self, status_code: int) -> None: + self.status_code = status_code + self.calls: list[tuple[bytes, dict[str, str]]] = [] + + async def send(self, url: str, body: bytes, headers: dict[str, str]) -> WebhookResponse: + assert url + self.calls.append((body, headers)) + return WebhookResponse( + status_code=self.status_code, + content={"accepted": self.status_code < HTTP_MULTIPLE_CHOICES}, + error=None, + retry_after_seconds=None, + duration_ms=2, + ) + + +@pytest.mark.parametrize( + ("status_code", "expected_status", "active"), + [ + (204, WebhookDeliveryStatus.succeeded, True), + (410, WebhookDeliveryStatus.cancelled, False), + (500, WebhookDeliveryStatus.pending, True), + ], +) +async def test_worker_records_delivery_outcome( # noqa: PLR0913 + client: AsyncClient, + webhook_registration: Webhook, + uow: SQLAlchemyUOW, + monkeypatch: pytest.MonkeyPatch, + status_code: int, + expected_status: WebhookDeliveryStatus, + *, + active: bool, +) -> None: + response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) + assert response.status_code == status.HTTP_201_CREATED + monkeypatch.setattr(worker_module, "get_uow_raw", get_test_uow) + sender = StubSender(status_code) + + assert await WebhookWorker(sender=sender).run_once() == 1 + + delivery = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_registration.id)) + assert delivery.status == expected_status + assert delivery.attempt_count == 1 + assert delivery.status_code == status_code + attempts = (await uow.webhook_attempts.get_many()).items + assert len(attempts) == 1 + body, headers = sender.calls[0] + assert verify_standard_headers(body, headers, webhook_registration.secret) + await uow.session.refresh(webhook_registration) + assert webhook_registration.is_active is active + + +async def test_workers_do_not_claim_the_same_delivery( + client: AsyncClient, + webhook_registration: Webhook, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) + assert response.status_code == status.HTTP_201_CREATED + monkeypatch.setattr(worker_module, "get_uow_raw", get_test_uow) + senders = [StubSender(204), StubSender(204)] + + counts = await asyncio.gather(*(WebhookWorker(sender=sender).run_once() for sender in senders)) + + assert sum(counts) == 1 + assert sum(len(sender.calls) for sender in senders) == 1 + + +async def test_worker_recovers_an_expired_lease( + client: AsyncClient, + webhook_registration: Webhook, + uow: SQLAlchemyUOW, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) + assert response.status_code == status.HTTP_201_CREATED + delivery = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_registration.id)) + delivery.status = WebhookDeliveryStatus.processing + delivery.leased_until = naive_utc() - timedelta(seconds=1) + await uow.commit() + monkeypatch.setattr(worker_module, "get_uow_raw", get_test_uow) + + assert await WebhookWorker(sender=StubSender(204)).run_once() == 1 + + await uow.session.refresh(delivery) + assert delivery.status == WebhookDeliveryStatus.succeeded diff --git a/tests/api/webhooks/test_wrong_url.py b/tests/api/webhooks/test_wrong_url.py index a712634..f14d82e 100644 --- a/tests/api/webhooks/test_wrong_url.py +++ b/tests/api/webhooks/test_wrong_url.py @@ -4,7 +4,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import Webhook, WebhookDeliveryStatus from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification from tests.mocks import USER_CREATE @@ -15,6 +15,8 @@ async def test_wrong_url(client: AsyncClient, webhook_wrong_url: Webhook, uow: S try: event = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_wrong_url.id)) - assert event.status_code == 0 + assert event.status == WebhookDeliveryStatus.pending + assert event.status_code is None + assert event.attempt_count == 0 except NoResultFoundError: pytest.fail("No webhook event created") diff --git a/tests/security/test_webhooks.py b/tests/security/test_webhooks.py index 6579ce4..b070019 100644 --- a/tests/security/test_webhooks.py +++ b/tests/security/test_webhooks.py @@ -1,7 +1,14 @@ import pytest -from fastid.security.webhooks import generate_headers, verify_headers -from tests.mocks import WEBHOOK_PAYLOAD, WEBHOOK_TEST_DATA +from fastid.security.webhooks import ( + generate_delivery_headers, + generate_headers, + generate_secret, + serialize_payload, + verify_headers, + verify_standard_headers, +) +from tests.mocks import WEBHOOK_ID, WEBHOOK_PAYLOAD, WEBHOOK_TEST_DATA, WEBHOOK_TIMESTAMP @pytest.mark.parametrize( @@ -12,3 +19,22 @@ def test_verify_headers( ) -> None: headers = generate_headers(WEBHOOK_PAYLOAD, timestamp, webhook_id, generate_secret_key) assert verify_headers(WEBHOOK_PAYLOAD, headers, verify_secret_key) == expected + + +@pytest.mark.parametrize("secret", ["legacy-secret", generate_secret()]) +def test_verify_standard_headers(secret: str) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(WEBHOOK_PAYLOAD, body, WEBHOOK_ID, "delivery-id", WEBHOOK_TIMESTAMP, secret) + + assert verify_standard_headers(body, headers, secret) + assert not verify_standard_headers(body + b" ", headers, secret) + assert not verify_standard_headers(body, headers, "wrong-secret") + + +def test_standard_signature_uses_exact_utf8_body() -> None: + payload = {"message": "Привет"} + body = serialize_payload(payload) + headers = generate_delivery_headers(payload, body, WEBHOOK_ID, "delivery-id", WEBHOOK_TIMESTAMP, "secret") + + assert b"\\u" not in body + assert verify_standard_headers(body, {key.upper(): value for key, value in headers.items()}, "secret") diff --git a/tests/webhooks/__init__.py b/tests/webhooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/webhooks/test_sender.py b/tests/webhooks/test_sender.py new file mode 100644 index 0000000..cf9b43d --- /dev/null +++ b/tests/webhooks/test_sender.py @@ -0,0 +1,51 @@ +import httpx +import pytest +from starlette import status + +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.senders.httpx import WebhookSender, resolve_targets +from tests.mocks import faker + + +async def test_sender_does_not_follow_redirects(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", True) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.content == b'{"ok":true}' + return httpx.Response(302, headers={"Location": "http://127.0.0.1/private"}, text="redirect") + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + result = await WebhookSender(client).send("http://example.test/hook", b'{"ok":true}', {}) + + assert result.status_code == status.HTTP_302_FOUND + assert result.content == {"value": "redirect"} + + +async def test_sender_bounds_response(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", True) + monkeypatch.setattr(webhook_settings, "max_response_bytes", 4) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _request: httpx.Response(status.HTTP_200_OK, text="123456")) + ) as client: + result = await WebhookSender(client).send("http://example.test/hook", b"{}", {}) + + assert result.content == {"value": 1234, "truncated": True} + + +@pytest.mark.parametrize( + ("url", "message"), + [("http://example.com/hook", "HTTPS"), (f"https://user:{faker.password()}@example.com/hook", "credentials")], +) +async def test_resolve_targets_rejects_unsafe_url(monkeypatch: pytest.MonkeyPatch, url: str, message: str) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", False) + + with pytest.raises(ValueError, match=message): + await resolve_targets(url) + + +async def test_resolve_targets_rejects_loopback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", False) + + with pytest.raises(ValueError, match="non-public"): + await resolve_targets("https://127.0.0.1/hook") diff --git a/tests/webhooks/test_worker.py b/tests/webhooks/test_worker.py new file mode 100644 index 0000000..fe7e8a8 --- /dev/null +++ b/tests/webhooks/test_worker.py @@ -0,0 +1,18 @@ +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.worker import get_retry_delay + +FIVE_SECONDS = 5 +FIVE_MINUTES = 300 +ONE_MINUTE = 60 +ONE_DAY = 86400 + + +def test_retry_schedule_without_jitter() -> None: + assert get_retry_delay(1, None, jitter=1) == FIVE_SECONDS + assert get_retry_delay(2, None, jitter=1) == FIVE_MINUTES + assert get_retry_delay(9, None, jitter=1) == ONE_DAY + + +def test_retry_after_overrides_schedule_and_is_capped() -> None: + assert get_retry_delay(1, ONE_MINUTE, jitter=1) == ONE_MINUTE + assert get_retry_delay(1, webhook_settings.retry_after_max_seconds * 2, jitter=1) == ONE_DAY From 16dd6058a42cb65b1ea793c676fc8ea7546079bd Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Fri, 17 Jul 2026 19:42:07 +0300 Subject: [PATCH 02/28] feat: rename webhook events to deliveries and update related references --- .agents/skills/grill-with-docs/SKILL.md | 7 ---- .../skills/grill-with-docs/agents/openai.yaml | 5 --- .codegraph/.gitignore | 5 +++ .pre-commit-config.yaml | 2 + fastid/admin/factory.py | 4 +- fastid/admin/views/settings.py | 32 +++++++-------- fastid/database/models.py | 4 +- fastid/database/uow.py | 6 +-- fastid/webhooks/models.py | 12 +++--- fastid/webhooks/repositories.py | 10 ++--- fastid/webhooks/use_cases.py | 6 +-- fastid/webhooks/worker.py | 38 +++++++++--------- ...5f6_rename_webhook_events_to_deliveries.py | 39 +++++++++++++++++++ skills-lock.json | 11 ------ .../api/auth/test_authorize_password_grant.py | 6 +-- tests/api/auth/test_register.py | 6 +-- tests/api/profile/test_delete_user.py | 6 +-- tests/api/profile/test_update_user_email.py | 6 +-- .../api/profile/test_update_user_password.py | 6 +-- tests/api/profile/test_update_user_profile.py | 6 +-- tests/api/webhooks/test_worker_delivery.py | 6 +-- tests/api/webhooks/test_wrong_url.py | 12 +++--- 22 files changed, 130 insertions(+), 105 deletions(-) delete mode 100644 .agents/skills/grill-with-docs/SKILL.md delete mode 100644 .agents/skills/grill-with-docs/agents/openai.yaml create mode 100644 .codegraph/.gitignore create mode 100644 migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py delete mode 100644 skills-lock.json diff --git a/.agents/skills/grill-with-docs/SKILL.md b/.agents/skills/grill-with-docs/SKILL.md deleted file mode 100644 index bed05d2..0000000 --- a/.agents/skills/grill-with-docs/SKILL.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: grill-with-docs -description: A relentless interview to sharpen a plan or design, which also creates docs (ADR's and glossary) as we go. -disable-model-invocation: true ---- - -Run a `/grilling` session, using the `/domain-modeling` skill. diff --git a/.agents/skills/grill-with-docs/agents/openai.yaml b/.agents/skills/grill-with-docs/agents/openai.yaml deleted file mode 100644 index 5dbe278..0000000 --- a/.agents/skills/grill-with-docs/agents/openai.yaml +++ /dev/null @@ -1,5 +0,0 @@ -interface: - display_name: "Grill with Docs" - short_description: "Grill a design and write its docs" -policy: - allow_implicit_invocation: false diff --git a/.codegraph/.gitignore b/.codegraph/.gitignore new file mode 100644 index 0000000..d20c0fe --- /dev/null +++ b/.codegraph/.gitignore @@ -0,0 +1,5 @@ +# CodeGraph data files — local to each machine, not for committing. +# Ignore everything in .codegraph/ except this file itself, so transient +# files (the database, daemon.pid, sockets, logs) never show up in git. +* +!.gitignore diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d6a98c..d866fa4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,6 +59,8 @@ repos: hooks: - id: detect-secrets args: [ + '--exclude-files', 'docs', + '--exclude-files', 'migrations', '--baseline', '.secrets.baseline', ] exclude: package.lock.json diff --git a/fastid/admin/factory.py b/fastid/admin/factory.py index 76acf76..0f03c54 100644 --- a/fastid/admin/factory.py +++ b/fastid/admin/factory.py @@ -14,7 +14,7 @@ TelegramTemplateAdmin, WebhookAdmin, WebhookAttemptAdmin, - WebhookEventAdmin, + WebhookDeliveryAdmin, ) from fastid.admin.views.versioning import ( AppVersionAdmin, @@ -56,7 +56,7 @@ def create(self) -> FastAPI: admin.add_view(AppAdmin) admin.add_view(OAuthProviderAdmin) admin.add_view(WebhookAdmin) - admin.add_view(WebhookEventAdmin) + admin.add_view(WebhookDeliveryAdmin) admin.add_view(WebhookAttemptAdmin) admin.add_view(EmailTemplateAdmin) admin.add_view(TelegramTemplateAdmin) diff --git a/fastid/admin/views/settings.py b/fastid/admin/views/settings.py index 443532f..a2fe175 100644 --- a/fastid/admin/views/settings.py +++ b/fastid/admin/views/settings.py @@ -9,7 +9,7 @@ TelegramTemplate, Webhook, WebhookAttempt, - WebhookEvent, + WebhookDelivery, ) @@ -120,7 +120,7 @@ class WebhookAdmin(BaseView, model=Webhook): ] -class WebhookEventAdmin(BaseView, model=WebhookEvent): +class WebhookDeliveryAdmin(BaseView, model=WebhookDelivery): can_create = False can_edit = False can_delete = False @@ -131,24 +131,24 @@ class WebhookEventAdmin(BaseView, model=WebhookEvent): category = "Settings" column_list = [ - WebhookEvent.id, - WebhookEvent.event_id, - WebhookEvent.webhook, + WebhookDelivery.id, + WebhookDelivery.event_id, + WebhookDelivery.webhook, "webhook.type", - WebhookEvent.status, - WebhookEvent.attempt_count, - WebhookEvent.status_code, - WebhookEvent.response, - WebhookEvent.next_attempt_at, - WebhookEvent.completed_at, - WebhookEvent.created_at, + WebhookDelivery.status, + WebhookDelivery.attempt_count, + WebhookDelivery.status_code, + WebhookDelivery.response, + WebhookDelivery.next_attempt_at, + WebhookDelivery.completed_at, + WebhookDelivery.created_at, ] column_filters = [ - OperationColumnFilter(WebhookEvent.webhook_id), - OperationColumnFilter(WebhookEvent.status_code), - AllUniqueStringValuesFilter(WebhookEvent.status), + OperationColumnFilter(WebhookDelivery.webhook_id), + OperationColumnFilter(WebhookDelivery.status_code), + AllUniqueStringValuesFilter(WebhookDelivery.status), ] - column_formatters = {WebhookEvent.response: json_format, WebhookEvent.request: json_format} + column_formatters = {WebhookDelivery.response: json_format, WebhookDelivery.request: json_format} class WebhookAttemptAdmin(BaseView, model=WebhookAttempt): diff --git a/fastid/database/models.py b/fastid/database/models.py index e53317b..c40919e 100644 --- a/fastid/database/models.py +++ b/fastid/database/models.py @@ -6,7 +6,7 @@ from fastid.database.base import BaseOrm from fastid.notify.models import EmailTemplate, Notification, TelegramTemplate from fastid.oauth.models import OAuthAccount, OAuthProvider -from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookEvent +from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDelivery configure_mappers() @@ -39,7 +39,7 @@ "WebhookVersion", "Webhook", "WebhookAttempt", - "WebhookEvent", + "WebhookDelivery", "OAuthAccountVersion", "OAuthProviderVersion", ] diff --git a/fastid/database/uow.py b/fastid/database/uow.py index 48d57a2..504604d 100644 --- a/fastid/database/uow.py +++ b/fastid/database/uow.py @@ -9,7 +9,7 @@ from fastid.auth.repositories import UserRepository from fastid.notify.repositories import EmailTemplateRepository, NotificationRepository, TelegramTemplateRepository from fastid.oauth.repositories import OAuthAccountRepository, OAuthProviderRepository -from fastid.webhooks.repositories import WebhookAttemptRepository, WebhookEventRepository, WebhookRepository +from fastid.webhooks.repositories import WebhookAttemptRepository, WebhookDeliveryRepository, WebhookRepository if TYPE_CHECKING: from types import TracebackType @@ -29,7 +29,7 @@ class SQLAlchemyUOW: telegram_templates: TelegramTemplateRepository notifications: NotificationRepository webhooks: WebhookRepository - webhook_events: WebhookEventRepository + webhook_deliveries: WebhookDeliveryRepository webhook_attempts: WebhookAttemptRepository session_factory: async_sessionmaker[AsyncSession] @@ -48,7 +48,7 @@ async def begin(self) -> None: self.telegram_templates = TelegramTemplateRepository(self.session) self.notifications = NotificationRepository(self.session) self.webhooks = WebhookRepository(self.session) - self.webhook_events = WebhookEventRepository(self.session) + self.webhook_deliveries = WebhookDeliveryRepository(self.session) self.webhook_attempts = WebhookAttemptRepository(self.session) @property diff --git a/fastid/webhooks/models.py b/fastid/webhooks/models.py index 15a5a06..18ceb62 100644 --- a/fastid/webhooks/models.py +++ b/fastid/webhooks/models.py @@ -50,11 +50,11 @@ class Webhook(VersionedEntity): disabled_reason: Mapped[str | None] app: Mapped[App] = relationship(back_populates="webhooks") - webhook_events: Mapped[list[WebhookEvent]] = relationship(back_populates="webhook", cascade="delete") + deliveries: Mapped[list[WebhookDelivery]] = relationship(back_populates="webhook", cascade="delete") -class WebhookEvent(Entity): - __tablename__ = "webhook_events" +class WebhookDelivery(Entity): + __tablename__ = "webhook_deliveries" webhook_id: Mapped[UUID] = mapped_column(ForeignKey("webhooks.id"), index=True) event_id: Mapped[UUID] = mapped_column(index=True) @@ -70,7 +70,7 @@ class WebhookEvent(Entity): response: Mapped[dict[str, Any] | None] error: Mapped[str | None] - webhook: Mapped[Webhook] = relationship(back_populates="webhook_events") + webhook: Mapped[Webhook] = relationship(back_populates="deliveries") attempts: Mapped[list[WebhookAttempt]] = relationship(back_populates="delivery", cascade="delete") @@ -78,7 +78,7 @@ class WebhookAttempt(Entity): __tablename__ = "webhook_attempts" __table_args__ = (UniqueConstraint("delivery_id", "attempt_number"),) - delivery_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_events.id"), index=True) + delivery_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_deliveries.id"), index=True) attempt_number: Mapped[int] timestamp: Mapped[int] request: Mapped[dict[str, Any]] @@ -87,4 +87,4 @@ class WebhookAttempt(Entity): error: Mapped[str | None] duration_ms: Mapped[int] - delivery: Mapped[WebhookEvent] = relationship(back_populates="attempts") + delivery: Mapped[WebhookDelivery] = relationship(back_populates="attempts") diff --git a/fastid/webhooks/repositories.py b/fastid/webhooks/repositories.py index a9b2dd6..ebc6144 100644 --- a/fastid/webhooks/repositories.py +++ b/fastid/webhooks/repositories.py @@ -3,7 +3,7 @@ from fastid.database.repository import SQLAlchemyRepository from fastid.database.specification import Specification from fastid.database.utils import UUIDv7 -from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookEvent, WebhookType +from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDelivery, WebhookType class WebhookRepository(SQLAlchemyRepository[Webhook]): @@ -18,17 +18,17 @@ def apply(self, stmt: Any) -> Any: return stmt.where(Webhook.type == self.type, Webhook.is_active.is_(True)) -class WebhookEventRepository(SQLAlchemyRepository[WebhookEvent]): - model_type = WebhookEvent +class WebhookDeliveryRepository(SQLAlchemyRepository[WebhookDelivery]): + model_type = WebhookDelivery class WebhookAttemptRepository(SQLAlchemyRepository[WebhookAttempt]): model_type = WebhookAttempt -class WebhookEventWebhookIDSpecification(Specification): +class WebhookDeliveryWebhookIDSpecification(Specification): def __init__(self, webhook_id: UUIDv7) -> None: self.webhook_id = webhook_id def apply(self, stmt: Any) -> Any: - return stmt.where(WebhookEvent.webhook_id == self.webhook_id) + return stmt.where(WebhookDelivery.webhook_id == self.webhook_id) diff --git a/fastid/webhooks/use_cases.py b/fastid/webhooks/use_cases.py index d136139..2a978e7 100644 --- a/fastid/webhooks/use_cases.py +++ b/fastid/webhooks/use_cases.py @@ -2,7 +2,7 @@ from fastid.database.dependencies import UOWDep from fastid.database.utils import naive_utc from fastid.security.webhooks import get_event_id, get_timestamp, get_webhook_id -from fastid.webhooks.models import WebhookDeliveryStatus, WebhookEvent +from fastid.webhooks.models import WebhookDelivery, WebhookDeliveryStatus from fastid.webhooks.repositories import WebhookTypeSpecification from fastid.webhooks.schemas import Event, SendWebhookRequest, WebhookPayload @@ -19,7 +19,7 @@ async def enqueue(self, dto: SendWebhookRequest) -> None: payload = WebhookPayload(event=event_dto, data=dto.payload).model_dump(mode="json") now = naive_utc() for webhook in webhook_page.items: - delivery = WebhookEvent( + delivery = WebhookDelivery( id=get_webhook_id(), webhook_id=webhook.id, event_id=event_id, @@ -28,7 +28,7 @@ async def enqueue(self, dto: SendWebhookRequest) -> None: status=WebhookDeliveryStatus.pending, next_attempt_at=now, ) - await self.uow.webhook_events.add(delivery) + await self.uow.webhook_deliveries.add(delivery) async def send(self, dto: SendWebhookRequest) -> None: """Backward-compatible alias for enqueueing a durable delivery.""" diff --git a/fastid/webhooks/worker.py b/fastid/webhooks/worker.py index f35941a..2b33ab3 100644 --- a/fastid/webhooks/worker.py +++ b/fastid/webhooks/worker.py @@ -16,7 +16,7 @@ from fastid.security.webhooks import generate_delivery_headers, get_timestamp, serialize_payload from fastid.webhooks.config import webhook_settings from fastid.webhooks.metrics import ATTEMPT_DURATION, ATTEMPTS, DISABLED_ENDPOINTS, DUE_DELIVERIES -from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDeliveryStatus, WebhookEvent +from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDelivery, WebhookDeliveryStatus from fastid.webhooks.senders.dependencies import client from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender @@ -74,28 +74,30 @@ async def _claim(self) -> list[ClaimedDelivery]: uow = get_uow_raw() async with uow: stmt = ( - select(WebhookEvent) - .options(joinedload(WebhookEvent.webhook)) + select(WebhookDelivery) + .options(joinedload(WebhookDelivery.webhook)) .where( - WebhookEvent.next_attempt_at <= now, + WebhookDelivery.next_attempt_at <= now, or_( - WebhookEvent.status == WebhookDeliveryStatus.pending, - (WebhookEvent.status == WebhookDeliveryStatus.processing) & (WebhookEvent.leased_until <= now), + WebhookDelivery.status == WebhookDeliveryStatus.pending, + (WebhookDelivery.status == WebhookDeliveryStatus.processing) + & (WebhookDelivery.leased_until <= now), ), ) - .order_by(WebhookEvent.next_attempt_at, WebhookEvent.created_at) + .order_by(WebhookDelivery.next_attempt_at, WebhookDelivery.created_at) .limit(webhook_settings.worker_batch_size) - .with_for_update(skip_locked=True, of=WebhookEvent) + .with_for_update(skip_locked=True, of=WebhookDelivery) ) rows = list((await uow.session.scalars(stmt)).all()) due_count = await uow.session.scalar( select(func.count()) - .select_from(WebhookEvent) + .select_from(WebhookDelivery) .where( - WebhookEvent.next_attempt_at <= now, + WebhookDelivery.next_attempt_at <= now, or_( - WebhookEvent.status == WebhookDeliveryStatus.pending, - (WebhookEvent.status == WebhookDeliveryStatus.processing) & (WebhookEvent.leased_until <= now), + WebhookDelivery.status == WebhookDeliveryStatus.pending, + (WebhookDelivery.status == WebhookDeliveryStatus.processing) + & (WebhookDelivery.leased_until <= now), ), ) ) @@ -151,7 +153,7 @@ async def _record( now = naive_utc() uow = get_uow_raw() async with uow: - delivery = await uow.webhook_events.get(delivery_id) + delivery = await uow.webhook_deliveries.get(delivery_id) endpoint = await uow.webhooks.get(delivery.webhook_id) attempt_number = delivery.attempt_count + 1 stored_headers = { @@ -207,7 +209,7 @@ async def _record( @staticmethod async def _disable( - uow: SQLAlchemyUOW, endpoint: Webhook, delivery: WebhookEvent, now: datetime, reason: str + uow: SQLAlchemyUOW, endpoint: Webhook, delivery: WebhookDelivery, now: datetime, reason: str ) -> None: delivery.status = ( WebhookDeliveryStatus.exhausted if delivery.status_code != HTTP_GONE else WebhookDeliveryStatus.cancelled @@ -217,11 +219,11 @@ async def _disable( endpoint.disabled_at = now endpoint.disabled_reason = reason await uow.session.execute( - update(WebhookEvent) + update(WebhookDelivery) .where( - WebhookEvent.webhook_id == endpoint.id, - WebhookEvent.id != delivery.id, - WebhookEvent.status.in_((WebhookDeliveryStatus.pending, WebhookDeliveryStatus.processing)), + WebhookDelivery.webhook_id == endpoint.id, + WebhookDelivery.id != delivery.id, + WebhookDelivery.status.in_((WebhookDeliveryStatus.pending, WebhookDeliveryStatus.processing)), ) .values(status=WebhookDeliveryStatus.cancelled, completed_at=now, error="endpoint disabled") ) diff --git a/migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py b/migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py new file mode 100644 index 0000000..af940e9 --- /dev/null +++ b/migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py @@ -0,0 +1,39 @@ +"""rename webhook events to deliveries + +Revision ID: a4e1d2c3b5f6 +Revises: 7f3c0d2f18a1 +Create Date: 2026-07-17 18:00:00 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "a4e1d2c3b5f6" +down_revision: str | None = "7f3c0d2f18a1" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +OLD_TABLE = "webhook_events" +NEW_TABLE = "webhook_deliveries" +INDEX_COLUMNS = ("webhook_id", "event_id", "status", "next_attempt_at", "leased_until") + + +def upgrade() -> None: + op.rename_table(OLD_TABLE, NEW_TABLE) + op.execute(f"ALTER TABLE {NEW_TABLE} RENAME CONSTRAINT {OLD_TABLE}_pkey TO {NEW_TABLE}_pkey") + op.execute( + f"ALTER TABLE {NEW_TABLE} " f"RENAME CONSTRAINT {OLD_TABLE}_webhook_id_fkey TO {NEW_TABLE}_webhook_id_fkey" + ) + for column in INDEX_COLUMNS: + op.execute(f"ALTER INDEX {OLD_TABLE}_{column}_idx RENAME TO {NEW_TABLE}_{column}_idx") + + +def downgrade() -> None: + op.rename_table(NEW_TABLE, OLD_TABLE) + op.execute(f"ALTER TABLE {OLD_TABLE} RENAME CONSTRAINT {NEW_TABLE}_pkey TO {OLD_TABLE}_pkey") + op.execute( + f"ALTER TABLE {OLD_TABLE} " f"RENAME CONSTRAINT {NEW_TABLE}_webhook_id_fkey TO {OLD_TABLE}_webhook_id_fkey" + ) + for column in INDEX_COLUMNS: + op.execute(f"ALTER INDEX {NEW_TABLE}_{column}_idx RENAME TO {OLD_TABLE}_{column}_idx") diff --git a/skills-lock.json b/skills-lock.json deleted file mode 100644 index b18d465..0000000 --- a/skills-lock.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": 1, - "skills": { - "grill-with-docs": { - "source": "mattpocock/skills", - "sourceType": "github", - "skillPath": "skills/engineering/grill-with-docs/SKILL.md", - "computedHash": "96718e0a89e48c7d76cc9759c7094a82ff34eb89ddc46a30388063fa9812356f" - } - } -} diff --git a/tests/api/auth/test_authorize_password_grant.py b/tests/api/auth/test_authorize_password_grant.py index 4a2b4dc..79bd820 100644 --- a/tests/api/auth/test_authorize_password_grant.py +++ b/tests/api/auth/test_authorize_password_grant.py @@ -8,7 +8,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from tests import mocks from tests.mocks import faker from tests.utils.auth import ( @@ -25,9 +25,9 @@ async def test_authorize_password_grant( assert token.refresh_token is not None try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_login.id)) + await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_login.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_dynamic_server_url_is_used_as_jwt_issuer( diff --git a/tests/api/auth/test_register.py b/tests/api/auth/test_register.py index af047cc..bc0d8ce 100644 --- a/tests/api/auth/test_register.py +++ b/tests/api/auth/test_register.py @@ -6,7 +6,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from tests.mocks import USER_CREATE @@ -20,9 +20,9 @@ async def test_register(client: AsyncClient, webhook_registration: Webhook, uow: assert user.email == USER_CREATE.email try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_registration.id)) + await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_registration.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_register_existent(client: AsyncClient, user: UserDTO) -> None: diff --git a/tests/api/profile/test_delete_user.py b/tests/api/profile/test_delete_user.py index 00367fa..1e53ba6 100644 --- a/tests/api/profile/test_delete_user.py +++ b/tests/api/profile/test_delete_user.py @@ -7,7 +7,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification async def test_delete_user( # noqa: PLR0913 @@ -28,9 +28,9 @@ async def test_delete_user( # noqa: PLR0913 UserDTO.model_validate_json(response.content) try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_delete.id)) + await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_delete.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_delete_user_not_verified(client: AsyncClient, user: UserDTO, user_token: TokenResponse) -> None: diff --git a/tests/api/profile/test_update_user_email.py b/tests/api/profile/test_update_user_email.py index 64d1645..0b2e4a5 100644 --- a/tests/api/profile/test_update_user_email.py +++ b/tests/api/profile/test_update_user_email.py @@ -9,7 +9,7 @@ from fastid.database.uow import SQLAlchemyUOW from fastid.security.crypto import generate_otp from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from tests.mocks import faker @@ -38,9 +38,9 @@ async def test_update_user_email( # noqa: PLR0913 assert user.email == new_email try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_change_email.id)) + await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_change_email.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_update_user_email_already_exists( # noqa: PLR0913 diff --git a/tests/api/profile/test_update_user_password.py b/tests/api/profile/test_update_user_password.py index 922770a..8284100 100644 --- a/tests/api/profile/test_update_user_password.py +++ b/tests/api/profile/test_update_user_password.py @@ -7,7 +7,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from tests.mocks import faker @@ -31,9 +31,9 @@ async def test_update_user_password( # noqa: PLR0913 UserDTO.model_validate_json(response.content) try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_change_password.id)) + await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_change_password.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_update_user_password_not_verified(client: AsyncClient, user: UserDTO, user_token: TokenResponse) -> None: diff --git a/tests/api/profile/test_update_user_profile.py b/tests/api/profile/test_update_user_profile.py index 47fa3e2..84ffd22 100644 --- a/tests/api/profile/test_update_user_profile.py +++ b/tests/api/profile/test_update_user_profile.py @@ -6,7 +6,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from tests import mocks @@ -24,6 +24,6 @@ async def test_update_user_profile( assert user.last_name == mocks.USER_UPDATE.last_name try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_profile_update.id)) + await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_profile_update.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") diff --git a/tests/api/webhooks/test_worker_delivery.py b/tests/api/webhooks/test_worker_delivery.py index 11855b4..e9e4012 100644 --- a/tests/api/webhooks/test_worker_delivery.py +++ b/tests/api/webhooks/test_worker_delivery.py @@ -10,7 +10,7 @@ from fastid.database.utils import naive_utc from fastid.security.webhooks import verify_standard_headers from fastid.webhooks.models import Webhook, WebhookDeliveryStatus -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender from fastid.webhooks.worker import WebhookWorker from tests.dependencies import get_test_uow @@ -61,7 +61,7 @@ async def test_worker_records_delivery_outcome( # noqa: PLR0913 assert await WebhookWorker(sender=sender).run_once() == 1 - delivery = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_registration.id)) + delivery = await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_registration.id)) assert delivery.status == expected_status assert delivery.attempt_count == 1 assert delivery.status_code == status_code @@ -97,7 +97,7 @@ async def test_worker_recovers_an_expired_lease( ) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) assert response.status_code == status.HTTP_201_CREATED - delivery = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_registration.id)) + delivery = await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_registration.id)) delivery.status = WebhookDeliveryStatus.processing delivery.leased_until = naive_utc() - timedelta(seconds=1) await uow.commit() diff --git a/tests/api/webhooks/test_wrong_url.py b/tests/api/webhooks/test_wrong_url.py index f14d82e..73874b0 100644 --- a/tests/api/webhooks/test_wrong_url.py +++ b/tests/api/webhooks/test_wrong_url.py @@ -5,7 +5,7 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.webhooks.models import Webhook, WebhookDeliveryStatus -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification from tests.mocks import USER_CREATE @@ -14,9 +14,9 @@ async def test_wrong_url(client: AsyncClient, webhook_wrong_url: Webhook, uow: S assert response.status_code == status.HTTP_201_CREATED try: - event = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_wrong_url.id)) - assert event.status == WebhookDeliveryStatus.pending - assert event.status_code is None - assert event.attempt_count == 0 + delivery = await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_wrong_url.id)) + assert delivery.status == WebhookDeliveryStatus.pending + assert delivery.status_code is None + assert delivery.attempt_count == 0 except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") From 00b94e779cfa7e8fc9f27124b02d47ef6d4d0450 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Fri, 17 Jul 2026 19:51:57 +0300 Subject: [PATCH 03/28] feat: remove .env.example --- .env.example | 5 ----- .pre-commit-config.yaml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 .env.example diff --git a/.env.example b/.env.example deleted file mode 100644 index 9a3a3be..0000000 --- a/.env.example +++ /dev/null @@ -1,5 +0,0 @@ -POSTGRES_PASSWORD="changethis" -REDIS_PASSWORD="changethis" -FASTID_WEBHOOK_ALLOW_INSECURE_URLS=false -FASTID_WEBHOOK_WORKER_CONCURRENCY=20 -FASTID_WEBHOOK_WORKER_METRICS_PORT=9101 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d866fa4..9e7b2a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: args: [ '--exclude-files', 'docs', '--exclude-files', 'migrations', - '--baseline', '.secrets.baseline', + '--exclude-files', 'alembic.ini', ] exclude: package.lock.json stages: [ pre-commit, pre-merge-commit, manual ] From 2f644c9ff85565d8bc243f02892f40758c1e3b15 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Fri, 17 Jul 2026 19:52:17 +0300 Subject: [PATCH 04/28] feat: remove .secrets.baseline --- .secrets.baseline | 409 ---------------------------------------------- 1 file changed, 409 deletions(-) delete mode 100644 .secrets.baseline diff --git a/.secrets.baseline b/.secrets.baseline deleted file mode 100644 index f1c540e..0000000 --- a/.secrets.baseline +++ /dev/null @@ -1,409 +0,0 @@ -{ - "version": "1.5.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "DiscordBotTokenDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "GitLabTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "IPPublicDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "OpenAIDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "PypiTokenDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TelegramBotTokenDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - } - ], - "results": { - ".env.example": [ - { - "type": "Secret Keyword", - "filename": ".env.example", - "hashed_secret": "cdb0e76c1a69873cbdcdbe0a142d56c023dc9f22", - "is_verified": false, - "line_number": 1 - } - ], - "alembic.ini": [ - { - "type": "Basic Auth Credentials", - "filename": "alembic.ini", - "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", - "is_verified": false, - "line_number": 62 - } - ], - "migrations\\env.py": [ - { - "type": "Basic Auth Credentials", - "filename": "migrations\\env.py", - "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", - "is_verified": false, - "line_number": 29 - } - ], - "migrations\\versions\\2024_08_07_2001-590f147f3dca_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_08_07_2001-590f147f3dca_.py", - "hashed_secret": "187df19f5f6a46ee1ff0cae4361df15b4da5a2c5", - "is_verified": false, - "line_number": 15 - } - ], - "migrations\\versions\\2024_09_05_2030-5ee810fadd8e_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_05_2030-5ee810fadd8e_.py", - "hashed_secret": "187df19f5f6a46ee1ff0cae4361df15b4da5a2c5", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2024_09_09_1744-bd54ebc87d2a_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_09_1744-bd54ebc87d2a_.py", - "hashed_secret": "cd11eeeb82f9c3b9d253b3816401681e5b144942", - "is_verified": false, - "line_number": 15 - } - ], - "migrations\\versions\\2024_09_09_1755-5781ebeb542c_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_09_1755-5781ebeb542c_.py", - "hashed_secret": "ad246c736071283e7d474bfe6eca50e8ec7f46c4", - "is_verified": false, - "line_number": 16 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_09_1755-5781ebeb542c_.py", - "hashed_secret": "cd11eeeb82f9c3b9d253b3816401681e5b144942", - "is_verified": false, - "line_number": 17 - } - ], - "migrations\\versions\\2024_09_09_2224-064a5ee97841_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_09_2224-064a5ee97841_.py", - "hashed_secret": "d4373f80f50cc750a1b12c410d81a70767a66a13", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_09_2224-064a5ee97841_.py", - "hashed_secret": "ad246c736071283e7d474bfe6eca50e8ec7f46c4", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2024_09_14_1413-f2979d2bcb1c_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_14_1413-f2979d2bcb1c_.py", - "hashed_secret": "d4373f80f50cc750a1b12c410d81a70767a66a13", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2024_09_18_2148-fa7d22042c64_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_09_18_2148-fa7d22042c64_.py", - "hashed_secret": "6f207de350188de6c62f4b710e8c7dfd675cbe79", - "is_verified": false, - "line_number": 15 - } - ], - "migrations\\versions\\2024_10_17_1951-9d66ac80082e_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_10_17_1951-9d66ac80082e_.py", - "hashed_secret": "dca2e21ab81cc009c7111b790ae4f4b281c64c66", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_10_17_1951-9d66ac80082e_.py", - "hashed_secret": "6f207de350188de6c62f4b710e8c7dfd675cbe79", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2024_10_17_1955-55aea0333e65_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_10_17_1955-55aea0333e65_.py", - "hashed_secret": "dca2e21ab81cc009c7111b790ae4f4b281c64c66", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2024_10_27_1628-f02ee2d85935_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_10_27_1628-f02ee2d85935_.py", - "hashed_secret": "5a8f330e2372f167e8f0ab64325e6b137b3a6b02", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2024_11_13_0105-3282fde15451_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_11_13_0105-3282fde15451_.py", - "hashed_secret": "80b5d0233910f84862e7becac6cfbcf1ab47cf77", - "is_verified": false, - "line_number": 16 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_11_13_0105-3282fde15451_.py", - "hashed_secret": "5a8f330e2372f167e8f0ab64325e6b137b3a6b02", - "is_verified": false, - "line_number": 17 - } - ], - "migrations\\versions\\2024_11_17_1914-446313a37068_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2024_11_17_1914-446313a37068_.py", - "hashed_secret": "80b5d0233910f84862e7becac6cfbcf1ab47cf77", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2026_01_08_1507-81c4895dd132_add_templates.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_08_1507-81c4895dd132_add_templates.py", - "hashed_secret": "64f464ac769660cc979aa0a46bdaff5dd5019145", - "is_verified": false, - "line_number": 15 - } - ], - "migrations\\versions\\2026_01_08_1511-32845277651e_remove_name_in_templates.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_08_1511-32845277651e_remove_name_in_templates.py", - "hashed_secret": "82949d609c9c90e3170a72222fb2df047f78417b", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_08_1511-32845277651e_remove_name_in_templates.py", - "hashed_secret": "64f464ac769660cc979aa0a46bdaff5dd5019145", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2026_01_14_1153-0b32faa598cb_add_notifications.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_14_1153-0b32faa598cb_add_notifications.py", - "hashed_secret": "80f5293392b82dd85bd8e7c0314049eaa186a900", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_14_1153-0b32faa598cb_add_notifications.py", - "hashed_secret": "82949d609c9c90e3170a72222fb2df047f78417b", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2026_01_16_2014-18fabf10cb0a_add_versioning.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_16_2014-18fabf10cb0a_add_versioning.py", - "hashed_secret": "80f5293392b82dd85bd8e7c0314049eaa186a900", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2026_01_19_1430-8d17be37457c_.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_19_1430-8d17be37457c_.py", - "hashed_secret": "34cce76d51334697020bca2d804f0dadcf3aff77", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2026_01_19_1529-03244d107aec_update_wh_type.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_19_1529-03244d107aec_update_wh_type.py", - "hashed_secret": "9a1a141064fdc7bedb9bc3567cf47781f99f1d98", - "is_verified": false, - "line_number": 15 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_01_19_1529-03244d107aec_update_wh_type.py", - "hashed_secret": "34cce76d51334697020bca2d804f0dadcf3aff77", - "is_verified": false, - "line_number": 16 - } - ], - "migrations\\versions\\2026_06_06_2007-3dc1a72c7ad5_add_client_id_idx.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_06_06_2007-3dc1a72c7ad5_add_client_id_idx.py", - "hashed_secret": "9a1a141064fdc7bedb9bc3567cf47781f99f1d98", - "is_verified": false, - "line_number": 15 - } - ], - "migrations\\versions\\2026_07_16_1933-bfa61bae871d_add_oauth_providers.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_07_16_1933-bfa61bae871d_add_oauth_providers.py", - "hashed_secret": "e07b058c212cb62219f3bc38c883997b989e1c1f", - "is_verified": false, - "line_number": 15 - } - ], - "migrations\\versions\\2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py": [ - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py", - "hashed_secret": "02700820903b32d3be49ae353c977dc39e91f9d2", - "is_verified": false, - "line_number": 14 - }, - { - "type": "Hex High Entropy String", - "filename": "migrations\\versions\\2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py", - "hashed_secret": "e07b058c212cb62219f3bc38c883997b989e1c1f", - "is_verified": false, - "line_number": 15 - } - ], - "skills-lock.json": [ - { - "type": "Hex High Entropy String", - "filename": "skills-lock.json", - "hashed_secret": "b90567df9a56aa46f4a821253aab48a9eebe4e26", - "is_verified": false, - "line_number": 8 - } - ] - }, - "generated_at": "2026-07-17T15:32:48Z" -} From 2f31b19de77bbd30a7db9a11ad5b3467e0204f59 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 13:20:25 +0300 Subject: [PATCH 05/28] docs: standardize webhook headers --- docs/adr/0001-use-standard-webhook-headers.md | 26 ++++++ docs/glossary.md | 32 +++++++ ...6-07-20-standard-webhook-headers-design.md | 89 +++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 docs/adr/0001-use-standard-webhook-headers.md create mode 100644 docs/glossary.md create mode 100644 docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md diff --git a/docs/adr/0001-use-standard-webhook-headers.md b/docs/adr/0001-use-standard-webhook-headers.md new file mode 100644 index 0000000..f6c2e7b --- /dev/null +++ b/docs/adr/0001-use-standard-webhook-headers.md @@ -0,0 +1,26 @@ +# ADR 0001: Use Only Standard Webhook Headers + +- Status: Accepted +- Date: 2026-07-20 + +## Context + +FastID sends both a legacy configurable `X-Webhook-*` signature protocol and the fixed Standard Webhooks protocol. The +protocols use different identifiers, signed representations, and signature encodings. Consumers therefore have two +ways to authenticate the same delivery, only one of which protects the exact transmitted bytes. + +## Decision + +FastID will send and support only `webhook-id`, `webhook-timestamp`, and `webhook-signature`. The signature will remain a +versioned base64 HMAC-SHA256 over `webhook-id.webhook-timestamp.raw_body`. + +The legacy signing functions, verification function, configurable header names, and configurable signature algorithm +will be removed. + +## Consequences + +- New examples and integrations have one authentication contract. +- The stable event ID is unambiguously the idempotency key. +- Verification authenticates the exact transmitted body. +- Existing consumers of `X-Webhook-*` must migrate to Standard Webhooks headers. +- Supporting a future signing change will require a new version in `webhook-signature`, not a second header family. diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000..e708eb9 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,32 @@ +# Glossary + +## Webhooks + +**Delivery attempt** +: One HTTP request made for a webhook event. Retries are separate attempts with a new timestamp and signature. + +**Endpoint** +: A consumer-owned URL registered to receive one FastID webhook event type. + +**Event** +: A logical user-lifecycle occurrence emitted by FastID. One event can have multiple delivery attempts. + +**Event ID** +: The stable UUID sent in `webhook-id` and in `event.event_id`. Consumers use it as the idempotency key. + +**Idempotency key** +: A stable identifier atomically recorded by a consumer to prevent repeated side effects when an event is delivered +more than once. + +**Raw body** +: The exact request bytes received before JSON parsing or re-serialization. These bytes are part of the signed content. + +**Standard Webhooks headers** +: The `webhook-id`, `webhook-timestamp`, and `webhook-signature` headers used to identify and authenticate a delivery. + +**Webhook secret** +: Endpoint-specific HMAC key material represented as `whsec_`. It must be stored and transmitted as a secret. + +**Webhook signature** +: A versioned, base64-encoded HMAC-SHA256 authenticating +`webhook-id.webhook-timestamp.raw_body`. FastID currently emits version `v1`. diff --git a/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md b/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md new file mode 100644 index 0000000..1364d3d --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md @@ -0,0 +1,89 @@ +# Standard Webhook Headers Design + +## Context + +FastID currently sends two independent webhook authentication protocols on every delivery: + +- legacy configurable `X-Webhook-*` headers, whose signature covers a re-serialized JSON payload; +- fixed Standard Webhooks headers, whose signature covers the exact raw request body. + +The legacy `X-Webhook-Id` identifies a delivery, while the standard `webhook-id` identifies the logical event and remains +stable across retries. Keeping both protocols makes the consumer contract ambiguous and complicates the webhook +examples. + +The repository has no production caller of the legacy signing or verification functions. The legacy protocol remains +only in the delivery-header composition, configuration, tests, and a compatibility note in the webhook tutorial. There +are no tagged releases establishing a published compatibility requirement. + +## Decision + +FastID will expose only the Standard Webhooks header family: + +- `webhook-id`: stable logical event ID; +- `webhook-timestamp`: Unix timestamp for the delivery attempt; +- `webhook-signature`: one or more space-separated `v1,` signatures. + +The signed content remains the exact byte sequence +`webhook-id.webhook-timestamp.raw_body`. Endpoint secrets retain the `whsec_` prefix and base64-encoded key material. + +## Code changes + +`generate_delivery_headers()` will directly construct the standard headers plus `Content-Type` and `User-Agent`. It +will no longer accept or use a separately generated delivery ID for authentication. + +The following legacy API will be removed: + +- `generate_headers()`; +- `generate_signature()`; +- `verify_headers()`; +- the `SignatureAlgorithm` enum and algorithm lookup table; +- configurable legacy ID, timestamp, and signature header settings. + +`generate_standard_signature()`, `verify_standard_headers()`, payload serialization, timestamp validation, and secret +generation remain the supported implementation. + +All callers, tests, and documentation will be updated to use the single protocol. The delivery database ID remains an +internal identifier and is not sent as an authentication header. + +## Request flow + +For each delivery attempt, the worker serializes the payload once, generates a timestamp, and signs the stable event ID, +timestamp, and exact serialized bytes. The sender transmits those same bytes and the generated standard headers. + +Consumers read the raw body, validate timestamp freshness, calculate the expected HMAC over the raw bytes, compare the +signature in constant time, and only then parse JSON. Consumers use `webhook-id` as the idempotency key. + +## Compatibility + +This is an intentional breaking change for consumers that verify `X-Webhook-*`. Because the repository has no tagged +release and documentation already directs new integrations to Standard Webhooks, FastID will remove the legacy path +without a deprecation period. + +No payload fields, delivery retry behavior, endpoint secrets, or persisted delivery records change. + +## Error handling + +The sender continues treating header generation errors as programming or configuration failures. Consumer-side +verification continues returning `False` for missing headers, malformed timestamps, stale timestamps, and signature +mismatches. An invalid `whsec_` value continues raising `ValueError` so configuration errors are distinguishable from +untrusted requests. + +## Verification + +Tests will establish that: + +- generated deliveries contain only the standard authentication headers; +- signatures authenticate the exact UTF-8 body; +- header lookup is case-insensitive; +- body changes, wrong secrets, stale timestamps, and malformed headers fail verification; +- the worker passes valid standard headers to the sender; +- no legacy header settings, helpers, or documentation remain. + +The focused webhook security and worker tests will run first, followed by linting and the broader test suite where +practical. + +## Follow-up + +After this change is verified, FastID will add three standalone webhook receiver examples: a quick-start receiver, an +advanced receiver with an idempotency-store abstraction, and a SQLAlchemy-backed receiver. Their design and +implementation remain separate from this prerequisite cleanup. From e3d33c0f664bc444a9de67339d1fce40a083e903 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 13:24:36 +0300 Subject: [PATCH 06/28] docs: plan standard webhook header cleanup --- .../2026-07-20-standard-webhook-headers.md | 447 ++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-20-standard-webhook-headers.md diff --git a/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md b/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md new file mode 100644 index 0000000..e157edf --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md @@ -0,0 +1,447 @@ +# Standard Webhook Headers Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove FastID's legacy `X-Webhook-*` authentication protocol so every delivery uses only Standard Webhooks headers. + +**Architecture:** Keep one signing path in `fastid.security.webhooks`: serialize the payload once, sign the exact bytes with the stable event ID, and send only the three standard authentication headers. Remove the legacy configurable protocol from settings and schemas, then update the worker and tutorial to the smaller contract. + +**Tech Stack:** Python 3.12, FastAPI, Pydantic Settings, pytest, Ruff, mypy + +## Global Constraints + +- Preserve `webhook-id` as the stable logical event ID and consumer idempotency key. +- Sign the exact bytes `webhook-id.webhook-timestamp.raw_body` with HMAC-SHA256. +- Keep the `v1,` signature representation and `whsec_` secrets. +- Do not change payload fields, retry behavior, persisted deliveries, or endpoint secret generation. +- Do not modify the user's existing `docker/Dockerfile` changes. +- Prefix every shell command with `rtk`. + +## File structure + +- `fastid/security/webhooks.py`: the sole standard signing and verification implementation. +- `fastid/webhooks/config.py`: delivery policy settings; no consumer-controlled authentication protocol settings. +- `fastid/webhooks/schemas.py`: webhook payload schemas; no obsolete signing algorithm enum. +- `fastid/webhooks/worker.py`: signs a claimed delivery with its stable event ID and exact body. +- `tests/security/test_webhooks.py`: standard header generation and verification behavior. +- `tests/mocks.py`: shared webhook payload, timestamp, and event ID fixtures only. +- `tests/api/webhooks/test_worker_delivery.py`: existing end-to-end worker/header contract test. +- `docs/docs/tutorial/webhooks.md`: documents the single supported header family. + +--- + +### Task 1: Make Standard Webhooks the only generated and verified protocol + +**Files:** +- Modify: `tests/security/test_webhooks.py` +- Modify: `tests/mocks.py:8-10,141-153` +- Modify: `fastid/security/webhooks.py` + +**Interfaces:** +- Produces: `generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str) -> dict[str, str]` +- Preserves: `verify_standard_headers(body: bytes, headers: Mapping[str, str], secret_key: str, tolerance_seconds: int = 300) -> bool` +- Preserves: `serialize_payload(payload: dict[str, Any]) -> bytes` + +- [ ] **Step 1: Replace legacy-oriented security tests with the single-protocol contract** + +Replace `tests/security/test_webhooks.py` with: + +```python +import pytest + +import fastid.security.webhooks as webhook_security +from fastid.security.webhooks import ( + generate_delivery_headers, + generate_secret, + serialize_payload, + verify_standard_headers, +) +from tests.mocks import WEBHOOK_ID, WEBHOOK_PAYLOAD, WEBHOOK_TIMESTAMP + + +@pytest.mark.parametrize("secret", ["plain-secret", generate_secret()]) +def test_verify_standard_headers(secret: str) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, secret) + + assert set(headers) == { + "webhook-id", + "webhook-timestamp", + "webhook-signature", + "Content-Type", + "User-Agent", + } + assert verify_standard_headers(body, headers, secret) + assert not verify_standard_headers(body + b" ", headers, secret) + assert not verify_standard_headers(body, headers, "wrong-secret") + + +def test_standard_signature_uses_exact_utf8_body() -> None: + payload = {"message": "Привет"} + body = serialize_payload(payload) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + + assert b"\\u" not in body + assert verify_standard_headers(body, {key.upper(): value for key, value in headers.items()}, "secret") + + +@pytest.mark.parametrize( + "headers", + [ + {}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": "invalid", "webhook-signature": "v1,value"}, + {"webhook-id": "", "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": "v1,value"}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": ""}, + ], +) +def test_verify_standard_headers_rejects_malformed_headers(headers: dict[str, str]) -> None: + assert not verify_standard_headers(b"{}", headers, "secret") + + +def test_verify_standard_headers_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + monkeypatch.setattr(webhook_security.time, "time", lambda: WEBHOOK_TIMESTAMP + 301) + + assert not verify_standard_headers(body, headers, "secret") + + +def test_invalid_prefixed_secret_is_a_configuration_error() -> None: + with pytest.raises(ValueError, match="Invalid whsec_ webhook secret"): + generate_delivery_headers(b"{}", WEBHOOK_ID, WEBHOOK_TIMESTAMP, "whsec_not-base64") +``` + +In `tests/mocks.py`, remove the `webhook_settings` import and remove `WEBHOOK_SECRET_KEY`, +`WEBHOOK_EXPIRED_TIMESTAMP`, `WEBHOOK_WRONG_SECRET_KEY`, and `WEBHOOK_TEST_DATA`. Retain exactly these shared values: + +```python +WEBHOOK_PAYLOAD = {"test": {"test1": 1, "test2": "hello", "hello3": True}} +WEBHOOK_TIMESTAMP = get_timestamp() +WEBHOOK_ID = str(get_webhook_id()) +``` + +- [ ] **Step 2: Run the rewritten tests and verify the old API fails** + +Run: + +```powershell +rtk pytest tests/security/test_webhooks.py -q +``` + +Expected: FAIL because the existing `generate_delivery_headers()` still requires `payload` and `delivery_id`, and generated deliveries still contain legacy headers. + +- [ ] **Step 3: Replace the security module with the standard-only implementation** + +Replace `fastid/security/webhooks.py` with: + +```python +import base64 +import hashlib +import hmac +import json +import time +from collections.abc import Mapping +from typing import Any + +from fastid.database.utils import UUIDv7, uuid +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.models import generate_webhook_secret + +STANDARD_ID_HEADER = "webhook-id" +STANDARD_TIMESTAMP_HEADER = "webhook-timestamp" +STANDARD_SIGNATURE_HEADER = "webhook-signature" + + +def serialize_payload(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def generate_secret() -> str: + return generate_webhook_secret() + + +def _secret_bytes(secret_key: str) -> bytes: + if not secret_key.startswith("whsec_"): + return secret_key.encode() + try: + return base64.b64decode(secret_key.removeprefix("whsec_"), validate=True) + except ValueError as exc: + msg = "Invalid whsec_ webhook secret" + raise ValueError(msg) from exc + + +def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + digest = hmac.new(_secret_bytes(secret_key), signed, hashlib.sha256).digest() + return f"v1,{base64.b64encode(digest).decode()}" + + +def generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str) -> dict[str, str]: + return { + STANDARD_ID_HEADER: event_id, + STANDARD_TIMESTAMP_HEADER: str(timestamp), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, event_id, timestamp, secret_key), + "Content-Type": "application/json", + "User-Agent": webhook_settings.user_agent, + } + + +def verify_standard_headers( + body: bytes, + headers: Mapping[str, str], + secret_key: str, + tolerance_seconds: int = webhook_settings.tolerance_seconds, +) -> bool: + normalized = {key.lower(): value for key, value in headers.items()} + try: + timestamp = int(normalized[STANDARD_TIMESTAMP_HEADER]) + webhook_id = normalized[STANDARD_ID_HEADER] + signatures = normalized[STANDARD_SIGNATURE_HEADER].split() + except (KeyError, ValueError): + return False + if not webhook_id or not signatures or not is_timestamp_valid(timestamp, tolerance_seconds): + return False + expected = generate_standard_signature(body, webhook_id, timestamp, secret_key) + return any(hmac.compare_digest(signature, expected) for signature in signatures) + + +def get_event_id() -> UUIDv7: + return uuid() + + +def get_webhook_id() -> UUIDv7: + return uuid() + + +def get_timestamp() -> int: + return int(time.time()) + + +def is_timestamp_valid(timestamp: int, tolerance_seconds: int) -> bool: + current_time = int(time.time()) + return abs(current_time - timestamp) <= tolerance_seconds +``` + +- [ ] **Step 4: Run focused security checks** + +Run: + +```powershell +rtk pytest tests/security/test_webhooks.py -q +rtk ruff check fastid/security/webhooks.py tests/security/test_webhooks.py tests/mocks.py +rtk mypy fastid/security/webhooks.py tests/security/test_webhooks.py +``` + +Expected: all tests pass and both static checks exit successfully. + +- [ ] **Step 5: Commit the standard-only security contract** + +```powershell +rtk git add fastid/security/webhooks.py tests/security/test_webhooks.py tests/mocks.py +rtk git commit -m "refactor: use standard webhook signatures only" +``` + +Expected: commit succeeds without staging `docker/Dockerfile`. + +--- + +### Task 2: Remove the obsolete configuration and schema surface + +**Files:** +- Modify: `fastid/webhooks/config.py:1-12` +- Modify: `fastid/webhooks/schemas.py:1-16` + +**Interfaces:** +- Consumes: `webhook_settings.tolerance_seconds` and `webhook_settings.user_agent` used by Task 1. +- Produces: `WebhookSettings` containing delivery policy only; payload schema classes remain unchanged. + +- [ ] **Step 1: Prove the obsolete API is still present** + +Run: + +```powershell +rtk rg -n "SignatureAlgorithm|signature_algorithm|signature_header|timestamp_header|id_header" fastid +``` + +Expected: matches in `fastid/webhooks/config.py` and `fastid/webhooks/schemas.py`; no matches should remain after this task. + +- [ ] **Step 2: Remove legacy settings while preserving delivery policy** + +Make the beginning of `fastid/webhooks/config.py` exactly: + +```python +from pydantic_settings import SettingsConfigDict + +from fastid.core.schemas import ENV_PREFIX, BaseSettings + + +class WebhookSettings(BaseSettings): + tolerance_seconds: int = 300 + page_expires_in_seconds: int = 60 +``` + +Leave every setting from `retry_delays_seconds` through `user_agent`, the `model_config`, and the `webhook_settings` +instance unchanged. + +- [ ] **Step 3: Remove the unused signing enum** + +Make the imports and first model in `fastid/webhooks/schemas.py` exactly: + +```python +from typing import Any +from uuid import UUID + +from pydantic import Field + +from fastid.auth.schemas import UserDTO +from fastid.core.schemas import BaseModel +from fastid.webhooks.models import WebhookType + + +class SendWebhookRequest(BaseModel): + type: WebhookType + payload: dict[str, Any] = Field(default_factory=dict) +``` + +Leave `Event`, `WebhookPayload`, `Webhook`, `WebhookData`, `UserWebhookData`, and `UserWebhook` unchanged. + +- [ ] **Step 4: Verify removal and static correctness** + +Run: + +```powershell +rtk rg -n "SignatureAlgorithm|signature_algorithm|signature_header|timestamp_header|id_header" fastid +rtk ruff check fastid/webhooks/config.py fastid/webhooks/schemas.py +rtk mypy fastid/webhooks/config.py fastid/webhooks/schemas.py +``` + +Expected: `rg` prints no matches; Ruff and mypy exit successfully. + +- [ ] **Step 5: Commit the public-surface cleanup** + +```powershell +rtk git add fastid/webhooks/config.py fastid/webhooks/schemas.py +rtk git commit -m "refactor: remove legacy webhook configuration" +``` + +Expected: commit succeeds without staging `docker/Dockerfile`. + +--- + +### Task 3: Update the worker and documentation to the single protocol + +**Files:** +- Modify: `fastid/webhooks/worker.py:131-143` +- Test: `tests/api/webhooks/test_worker_delivery.py:39-73` +- Modify: `docs/docs/tutorial/webhooks.md:7-18` + +**Interfaces:** +- Consumes: `generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str)` from Task 1. +- Preserves: worker delivery requests contain a valid signature for the exact body and endpoint secret. + +- [ ] **Step 1: Run the existing worker contract test against the narrowed function** + +Run: + +```powershell +rtk pytest tests/api/webhooks/test_worker_delivery.py::test_worker_records_delivery_outcome -q +``` + +Expected: FAIL with a `TypeError` because the worker still passes the payload and internal delivery ID. + +- [ ] **Step 2: Update the worker to sign only the stable event ID and raw body** + +Replace the header-generation block in `WebhookWorker._process()` with: + +```python + headers = generate_delivery_headers( + body, + str(delivery.event_id), + timestamp, + delivery.endpoint_secret, + ) +``` + +Do not change payload serialization, sending, or attempt recording. + +- [ ] **Step 3: Run the worker webhook tests** + +Run: + +```powershell +rtk pytest tests/api/webhooks/test_worker_delivery.py tests/webhooks/test_worker.py tests/webhooks/test_sender.py -q +``` + +Expected: all tests pass, including signature verification at `test_worker_delivery.py:70-71`. + +- [ ] **Step 4: Remove the compatibility claim from the tutorial** + +Replace the opening of the request-format section in `docs/docs/tutorial/webhooks.md` with: + +```markdown +## Request format + +Each request is a JSON `POST` with Standard Webhooks headers: + +- `webhook-id`: logical event UUID, unchanged for retries. +- `webhook-timestamp`: Unix timestamp for the delivery attempt. +- `webhook-signature`: `v1,` signature. +``` + +Keep the raw-body verification explanation and all later delivery/security documentation unchanged. + +- [ ] **Step 5: Verify there are no legacy protocol references** + +Run: + +```powershell +rtk rg -n -i "x-webhook|generate_headers|generate_signature|verify_headers|SignatureAlgorithm" fastid tests docs/docs +rtk ruff check fastid tests +rtk mypy fastid +rtk pytest tests/security/test_webhooks.py tests/api/webhooks tests/webhooks -q +rtk git diff --check +``` + +Expected: `rg` prints no matches; Ruff, mypy, pytest, and `git diff --check` all exit successfully. + +- [ ] **Step 6: Commit the worker and tutorial migration** + +```powershell +rtk git add fastid/webhooks/worker.py docs/docs/tutorial/webhooks.md +rtk git commit -m "refactor: send only standard webhook headers" +``` + +Expected: commit succeeds without staging `docker/Dockerfile`. + +--- + +### Task 4: Run final regression verification + +**Files:** +- Verify only; do not modify unrelated files. + +**Interfaces:** +- Consumes: the standard-only webhook contract completed in Tasks 1-3. +- Produces: evidence that the cleanup is ready for the webhook examples. + +- [ ] **Step 1: Run the complete project checks** + +```powershell +rtk pytest -q +rtk ruff check . +rtk mypy fastid +rtk git diff --check +``` + +Expected: all commands exit successfully. If an unrelated pre-existing failure occurs, record its exact command and +output without modifying unrelated code. + +- [ ] **Step 2: Confirm scope and history** + +```powershell +rtk git status --short +rtk git log -4 --oneline +``` + +Expected: only the user's pre-existing `docker/Dockerfile` changes remain; the three cleanup commits follow the design +documentation commit `2f31b19`. From 3e3dd3d8ef89f101116ad193013e74bd58288670 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 13:35:26 +0300 Subject: [PATCH 07/28] chore: enforce LF for Docker files --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..24ed2d7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +docker/Dockerfile text eol=lf +docker/*.sh text eol=lf From be77bccc4379423a902be637a72ab891cc79a2ca Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 13:58:25 +0300 Subject: [PATCH 08/28] refactor: use standard webhook signatures only --- fastid/security/webhooks.py | 67 +-------------------------------- fastid/webhooks/worker.py | 2 - tests/mocks.py | 11 ------ tests/security/test_webhooks.py | 55 ++++++++++++++++++--------- 4 files changed, 40 insertions(+), 95 deletions(-) diff --git a/fastid/security/webhooks.py b/fastid/security/webhooks.py index 798acad..990b190 100644 --- a/fastid/security/webhooks.py +++ b/fastid/security/webhooks.py @@ -9,13 +9,6 @@ from fastid.database.utils import UUIDv7, uuid from fastid.webhooks.config import webhook_settings from fastid.webhooks.models import generate_webhook_secret -from fastid.webhooks.schemas import SignatureAlgorithm - -HASH_FUNCTIONS = { - SignatureAlgorithm.sha256: hashlib.sha256, - SignatureAlgorithm.sha512: hashlib.sha512, - SignatureAlgorithm.sha1: hashlib.sha1, -} STANDARD_ID_HEADER = "webhook-id" STANDARD_TIMESTAMP_HEADER = "webhook-timestamp" @@ -46,10 +39,8 @@ def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, se return f"v1,{base64.b64encode(digest).decode()}" -def generate_delivery_headers( # noqa: PLR0913 - payload: dict[str, Any], body: bytes, event_id: str, delivery_id: str, timestamp: int, secret_key: str -) -> dict[str, str]: - return generate_headers(payload, timestamp, delivery_id, secret_key) | { +def generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str) -> dict[str, str]: + return { STANDARD_ID_HEADER: event_id, STANDARD_TIMESTAMP_HEADER: str(timestamp), STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, event_id, timestamp, secret_key), @@ -58,60 +49,6 @@ def generate_delivery_headers( # noqa: PLR0913 } -def generate_headers( - payload: dict[str, Any], - timestamp: int, - webhook_id: str, - secret_key: str, - *, - algorithm: SignatureAlgorithm = webhook_settings.signature_algorithm, -) -> dict[str, str]: - signature = generate_signature(payload, webhook_id, timestamp, secret_key, algorithm=algorithm) - return { - webhook_settings.id_header: webhook_id, - webhook_settings.timestamp_header: str(timestamp), - webhook_settings.signature_header: signature, - } - - -def generate_signature( - payload: dict[str, Any], - webhook_id: str, - timestamp: int, - secret_key: str, - *, - algorithm: SignatureAlgorithm = webhook_settings.signature_algorithm, -) -> str: - payload_str = json.dumps(payload, sort_keys=True, separators=(",", ":")) - payload_str = f"{webhook_id}.{timestamp}.{payload_str}" - hash_func = HASH_FUNCTIONS[algorithm] - hmac_obj = hmac.new(key=secret_key.encode(), msg=payload_str.encode(), digestmod=hash_func) - return hmac_obj.hexdigest() - - -def verify_headers( - payload: dict[str, Any], - headers: dict[str, str], - secret_key: str, - tolerance_seconds: int = webhook_settings.tolerance_seconds, -) -> bool: - try: - timestamp = int(headers[webhook_settings.timestamp_header]) - webhook_id = headers[webhook_settings.id_header] - received_signature = headers[webhook_settings.signature_header] - except (KeyError, ValueError): # pragma: nocover - return False - - if not all([timestamp, webhook_id, received_signature]): - return False - - if not is_timestamp_valid(timestamp, tolerance_seconds): - return False - - expected_signature = generate_signature(payload, webhook_id, timestamp, secret_key) - return hmac.compare_digest(received_signature, expected_signature) - - def verify_standard_headers( body: bytes, headers: Mapping[str, str], diff --git a/fastid/webhooks/worker.py b/fastid/webhooks/worker.py index 2b33ab3..3f56b9d 100644 --- a/fastid/webhooks/worker.py +++ b/fastid/webhooks/worker.py @@ -132,10 +132,8 @@ async def _process(self, delivery: ClaimedDelivery) -> None: timestamp = get_timestamp() body = serialize_payload(delivery.payload) headers = generate_delivery_headers( - delivery.payload, body, str(delivery.event_id), - str(delivery.id), timestamp, delivery.endpoint_secret, ) diff --git a/tests/mocks.py b/tests/mocks.py index 987d58c..df9dc10 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -11,7 +11,6 @@ from fastid.oauth.schemas import OpenIDBearer from fastid.security.crypto import crypt_ctx from fastid.security.webhooks import get_timestamp, get_webhook_id -from fastid.webhooks.config import webhook_settings from fastid.webhooks.models import WebhookType from tests.utils.auth import generate_random_state @@ -141,16 +140,6 @@ def webhook_record_factory(webhook_type: WebhookType, **kwargs: Any) -> dict[str WEBHOOK_PAYLOAD = {"test": {"test1": 1, "test2": "hello", "hello3": True}} WEBHOOK_TIMESTAMP = get_timestamp() WEBHOOK_ID = str(get_webhook_id()) -WEBHOOK_SECRET_KEY = faker.pystr() -WEBHOOK_EXPIRED_TIMESTAMP = WEBHOOK_TIMESTAMP - webhook_settings.tolerance_seconds -WEBHOOK_WRONG_SECRET_KEY = faker.pystr() -WEBHOOK_TEST_DATA = [ - (WEBHOOK_TIMESTAMP, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, True), - (0, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, False), - (WEBHOOK_TIMESTAMP, "", WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, False), - (WEBHOOK_EXPIRED_TIMESTAMP, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, False), - (WEBHOOK_TIMESTAMP, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_WRONG_SECRET_KEY, False), -] class MockError(Exception): diff --git a/tests/security/test_webhooks.py b/tests/security/test_webhooks.py index b070019..454c708 100644 --- a/tests/security/test_webhooks.py +++ b/tests/security/test_webhooks.py @@ -2,30 +2,25 @@ from fastid.security.webhooks import ( generate_delivery_headers, - generate_headers, generate_secret, serialize_payload, - verify_headers, verify_standard_headers, ) -from tests.mocks import WEBHOOK_ID, WEBHOOK_PAYLOAD, WEBHOOK_TEST_DATA, WEBHOOK_TIMESTAMP +from tests.mocks import WEBHOOK_ID, WEBHOOK_PAYLOAD, WEBHOOK_TIMESTAMP -@pytest.mark.parametrize( - ("timestamp", "webhook_id", "generate_secret_key", "verify_secret_key", "expected"), WEBHOOK_TEST_DATA -) -def test_verify_headers( - timestamp: int, webhook_id: str, generate_secret_key: str, verify_secret_key: str, *, expected: bool -) -> None: - headers = generate_headers(WEBHOOK_PAYLOAD, timestamp, webhook_id, generate_secret_key) - assert verify_headers(WEBHOOK_PAYLOAD, headers, verify_secret_key) == expected - - -@pytest.mark.parametrize("secret", ["legacy-secret", generate_secret()]) +@pytest.mark.parametrize("secret", ["plain-secret", generate_secret()]) def test_verify_standard_headers(secret: str) -> None: body = serialize_payload(WEBHOOK_PAYLOAD) - headers = generate_delivery_headers(WEBHOOK_PAYLOAD, body, WEBHOOK_ID, "delivery-id", WEBHOOK_TIMESTAMP, secret) - + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, secret) + + assert set(headers) == { + "webhook-id", + "webhook-timestamp", + "webhook-signature", + "Content-Type", + "User-Agent", + } assert verify_standard_headers(body, headers, secret) assert not verify_standard_headers(body + b" ", headers, secret) assert not verify_standard_headers(body, headers, "wrong-secret") @@ -34,7 +29,33 @@ def test_verify_standard_headers(secret: str) -> None: def test_standard_signature_uses_exact_utf8_body() -> None: payload = {"message": "Привет"} body = serialize_payload(payload) - headers = generate_delivery_headers(payload, body, WEBHOOK_ID, "delivery-id", WEBHOOK_TIMESTAMP, "secret") + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") assert b"\\u" not in body assert verify_standard_headers(body, {key.upper(): value for key, value in headers.items()}, "secret") + + +@pytest.mark.parametrize( + "headers", + [ + {}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": "invalid", "webhook-signature": "v1,value"}, + {"webhook-id": "", "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": "v1,value"}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": ""}, + ], +) +def test_verify_standard_headers_rejects_malformed_headers(headers: dict[str, str]) -> None: + assert not verify_standard_headers(b"{}", headers, "secret") + + +def test_verify_standard_headers_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + monkeypatch.setattr("fastid.security.webhooks.time.time", lambda: WEBHOOK_TIMESTAMP + 301) + + assert not verify_standard_headers(body, headers, "secret") + + +def test_invalid_prefixed_secret_is_a_configuration_error() -> None: + with pytest.raises(ValueError, match="Invalid whsec_ webhook secret"): + generate_delivery_headers(b"{}", WEBHOOK_ID, WEBHOOK_TIMESTAMP, "whsec_not-base64") From 6d1ad9ba1f94e0eab93ee1ee051aa35d20d9fb86 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 13:59:08 +0300 Subject: [PATCH 09/28] refactor: remove legacy webhook configuration --- fastid/webhooks/config.py | 5 ----- fastid/webhooks/schemas.py | 9 +-------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/fastid/webhooks/config.py b/fastid/webhooks/config.py index 85f8210..4ced759 100644 --- a/fastid/webhooks/config.py +++ b/fastid/webhooks/config.py @@ -1,14 +1,9 @@ from pydantic_settings import SettingsConfigDict from fastid.core.schemas import ENV_PREFIX, BaseSettings -from fastid.webhooks.schemas import SignatureAlgorithm class WebhookSettings(BaseSettings): - signature_algorithm: SignatureAlgorithm = SignatureAlgorithm.sha256 - signature_header: str = "X-Webhook-Signature" - timestamp_header: str = "X-Webhook-Timestamp" - id_header: str = "X-Webhook-Id" tolerance_seconds: int = 300 page_expires_in_seconds: int = 60 retry_delays_seconds: tuple[int, ...] = (0, 5, 300, 1800, 7200, 18000, 36000, 50400, 72000, 86400) diff --git a/fastid/webhooks/schemas.py b/fastid/webhooks/schemas.py index 91cf953..60f6862 100644 --- a/fastid/webhooks/schemas.py +++ b/fastid/webhooks/schemas.py @@ -1,20 +1,13 @@ -from enum import auto from typing import Any from uuid import UUID from pydantic import Field from fastid.auth.schemas import UserDTO -from fastid.core.schemas import BaseEnum, BaseModel +from fastid.core.schemas import BaseModel from fastid.webhooks.models import WebhookType -class SignatureAlgorithm(BaseEnum): - sha256 = auto() - sha512 = auto() - sha1 = auto() - - class SendWebhookRequest(BaseModel): type: WebhookType payload: dict[str, Any] = Field(default_factory=dict) From 0928f2d1c0fda5dc913a718e5a7c94ce2d2f20c2 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:00:33 +0300 Subject: [PATCH 10/28] docs: document standard webhook headers only --- docs/docs/tutorial/webhooks.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/docs/tutorial/webhooks.md b/docs/docs/tutorial/webhooks.md index 09013a7..4934623 100644 --- a/docs/docs/tutorial/webhooks.md +++ b/docs/docs/tutorial/webhooks.md @@ -6,8 +6,7 @@ after a timeout or worker crash. Delivery order is not guaranteed. ## Request format -Each request is a JSON `POST`. The existing `X-Webhook-*` headers remain available, and new integrations should verify -the Standard Webhooks headers: +Each request is a JSON `POST` with Standard Webhooks headers: - `webhook-id`: logical event UUID, unchanged for retries. - `webhook-timestamp`: Unix timestamp for the delivery attempt. From c19c9e0b0e2e641ea263d653d766ade416e7512b Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:06:08 +0300 Subject: [PATCH 11/28] docs: design webhook receiver examples --- ...rovide-tiered-webhook-receiver-examples.md | 30 ++++ docs/glossary.md | 18 +++ ...-07-20-webhook-receiver-examples-design.md | 138 ++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 docs/adr/0002-provide-tiered-webhook-receiver-examples.md create mode 100644 docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md diff --git a/docs/adr/0002-provide-tiered-webhook-receiver-examples.md b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md new file mode 100644 index 0000000..f631bcd --- /dev/null +++ b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md @@ -0,0 +1,30 @@ +# ADR 0002: Provide Tiered Standalone Webhook Receiver Examples + +- Status: Accepted +- Date: 2026-07-20 + +## Context + +A minimal webhook receiver is useful for first success but omits production concerns. Adding storage abstractions and +database setup to that same file would obscure the authentication flow. Making examples import shared implementation +code would reduce duplication but make each example harder to copy into another service. + +## Decision + +FastID will provide three standalone FastAPI receiver applications: + +- a quick-start authentication example; +- an advanced in-memory idempotency reference; +- an advanced SQLAlchemy idempotency reference. + +Each application will independently implement Standard Webhooks verification using Python's standard library. Advanced +examples will share the same conceptual claim, complete, and release lifecycle but will not import each other. + +## Consequences + +- New users can reach a working receiver without database setup. +- Experienced users can compare volatile and durable idempotency boundaries. +- Each file can be copied independently. +- Signature-verification code is intentionally duplicated and must be changed consistently if the protocol evolves. +- The in-memory example is not production-durable and must say so prominently. +- The SQLAlchemy example introduces database concepts only in its dedicated file. diff --git a/docs/glossary.md b/docs/glossary.md index e708eb9..727bfc8 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -5,6 +5,17 @@ **Delivery attempt** : One HTTP request made for a webhook event. Retries are separate attempts with a new timestamp and signature. +**Claim** +: An atomic operation that reserves an event ID for processing. Only the request that creates the claim may apply the +event's side effects. + +**Complete** +: Mark a claimed event as successfully processed so later deliveries are acknowledged without repeating work. + +**Duplicate delivery** +: A delivery whose event ID has already been claimed or completed. Consumers acknowledge it without applying the event +again. + **Endpoint** : A consumer-owned URL registered to receive one FastID webhook event type. @@ -18,6 +29,13 @@ : A stable identifier atomically recorded by a consumer to prevent repeated side effects when an event is delivered more than once. +**Idempotency store** +: A component that atomically claims event IDs and records their processing state. A production implementation must be +shared by all receiver processes and survive restarts. + +**Release** +: Remove or expire an incomplete claim after processing fails, allowing a later FastID retry to claim the event again. + **Raw body** : The exact request bytes received before JSON parsing or re-serialization. These bytes are part of the signed content. diff --git a/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md new file mode 100644 index 0000000..0795726 --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md @@ -0,0 +1,138 @@ +# Webhook Receiver Examples Design + +## Context + +FastID now sends only Standard Webhooks authentication headers. The tutorial contains a short receiver fragment, but +`examples/` does not contain a complete webhook consumer that users can run or copy. A single example cannot remain +brief enough for a quick start while also teaching replay protection, concurrency, and durable idempotency. + +FastID will provide three standalone FastAPI applications. Each file intentionally repeats signature verification so a +reader can copy it without importing another example or installing FastID as a library. + +## Files and audiences + +### `examples/webhook_quickstart.py` + +The quick-start application is the shortest safe receiver. It requires `FASTID_WEBHOOK_SECRET` at startup, reads the +exact raw body, validates the three Standard Webhooks headers and timestamp, parses JSON only after authentication, +logs the generic event, and returns `204 No Content`. + +It demonstrates authentication but not idempotency. A comment warns that production consumers must atomically record +`webhook-id` before applying non-idempotent side effects. + +### `examples/webhook_advanced.py` + +The advanced application adds request-size enforcement, structural payload validation, header/payload event-ID +matching, explicit error responses, and an asynchronous idempotency-store protocol. Its in-memory adapter uses an +`asyncio.Lock` so concurrent requests in one process cannot claim the same event twice. + +The adapter is intentionally labeled as a reference implementation: it is not shared across processes and loses state +on restart. The protocol boundary shows where a production consumer supplies durable storage without mixing database +setup into this example. + +### `examples/webhook_sqlalchemy.py` + +The SQLAlchemy application is a standalone alternative to the advanced example. It implements an equivalent claim, +complete, and release lifecycle with a table whose event ID is unique. A duplicate insert is the atomic concurrency +boundary. + +`WEBHOOK_DATABASE_URL` configures the database and defaults to a local SQLite file so the example is runnable without +external infrastructure. Synchronous SQLAlchemy work runs in a worker thread so the asynchronous FastAPI endpoint does +not block the event loop. SQLite connections disable the same-thread check; other database URLs use their normal +driver settings. + +## Configuration + +All three applications require a non-empty `FASTID_WEBHOOK_SECRET`. Startup fails with a clear message if it is absent. +Secrets beginning with `whsec_` contain base64-encoded key bytes. Plain-text secrets remain accepted for compatibility +with existing endpoints. + +The advanced and SQLAlchemy applications use these fixed reference limits: + +- maximum request body: 1 MiB; +- timestamp tolerance: 300 seconds. + +The SQLAlchemy application additionally accepts `WEBHOOK_DATABASE_URL`, defaulting to +`sqlite:///webhook-events.sqlite3`. + +## Authentication + +Each receiver reads: + +- `webhook-id` as the stable event and idempotency identifier; +- `webhook-timestamp` as an integer Unix timestamp; +- `webhook-signature` as one or more space-separated versioned signatures. + +The expected `v1` signature is base64-encoded HMAC-SHA256 over the exact byte sequence +`webhook-id.webhook-timestamp.raw_body`. Verification compares signatures with `hmac.compare_digest`. The body is not +parsed or re-serialized before verification. + +Unknown signature versions are ignored so a future FastID rotation can send old and new signatures together. A request +is authenticated when any supplied `v1` signature matches. + +## Payload model + +The examples are event-type agnostic. After signature verification, they require a JSON object with: + +- `event.event_id`: a UUID string equal to `webhook-id`; +- `event.event_type`: a non-empty string; +- `event.timestamp`: an integer; +- `data`: a JSON object. + +They log the event ID and event type but do not dispatch business-specific handlers. + +The quick-start keeps payload checks minimal to stay concise. The advanced examples validate the complete generic +envelope before claiming the event. + +## Advanced request flow + +1. Reject a declared `Content-Length` greater than 1 MiB. +2. Read the raw body and reject an actual body greater than 1 MiB. +3. Validate required headers, integer timestamp, timestamp tolerance, secret encoding, and HMAC. +4. Parse and validate the generic JSON envelope. +5. Require `event.event_id` to equal `webhook-id`. +6. Atomically claim the event ID. +7. If already claimed or completed, return `204` without processing it again. +8. Log receipt and mark the claim complete. +9. If processing raises, release the claim and return `500` so FastID retries. + +The claim lifecycle prevents simultaneous duplicate processing in the demonstrated execution model. It does not promise +exactly-once side effects across a crash between the side effect and completion; production applications must make the +business change idempotent or commit an inbox record and business state in one transaction. + +## Error responses + +- `400 Bad Request`: missing or malformed webhook headers, stale timestamp, malformed JSON, invalid event envelope, or + header/payload event-ID mismatch; +- `401 Unauthorized`: a well-formed request with no matching signature; +- `413 Content Too Large`: declared or actual body exceeds 1 MiB; +- `500 Internal Server Error`: processing or idempotency storage fails, allowing FastID to retry; +- `204 No Content`: a new event was accepted or a duplicate was already claimed. + +Error bodies use FastAPI's normal JSON `detail` shape. Error messages do not include secrets, signatures, or raw body +contents. + +## Testing + +Tests import each standalone application with controlled environment variables and exercise it through FastAPI's test +client. Shared test-only helpers generate valid Standard Webhooks signatures; examples do not import those helpers. + +Coverage includes: + +- valid request acceptance for all three applications; +- exact-body signature tampering; +- missing, malformed, and non-matching signatures; +- stale timestamps; +- malformed JSON and invalid envelope fields; +- header/payload event-ID mismatch; +- declared and actual oversized bodies in advanced applications; +- repeated delivery acknowledgement without repeated processing; +- concurrent claims against the in-memory adapter; +- SQLAlchemy duplicate claims and persistence across store instances. + +Ruff, mypy, focused example tests, and the complete project suite must pass. + +## Documentation + +The webhook tutorial will link to the quick-start, advanced, and SQLAlchemy files and describe their intended audiences. +The examples remain source files rather than embedded copies so documentation cannot drift from runnable code. From a77f8eebfffe4a9e2e11d7f42212983a00b1d16b Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:09:13 +0300 Subject: [PATCH 12/28] docs: plan webhook receiver examples --- .../2026-07-20-webhook-receiver-examples.md | 406 ++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md diff --git a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md new file mode 100644 index 0000000..f86fd9b --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md @@ -0,0 +1,406 @@ +# Webhook Receiver Examples Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add three independently runnable FastAPI webhook receivers for quick-start, in-memory advanced, and SQLAlchemy-backed use cases. + +**Architecture:** Every example contains its own Standard Webhooks verifier so it can be copied independently. The quick-start authenticates and logs generic events; the advanced examples add a claim/complete/release idempotency lifecycle, implemented first with a locked in-memory store and then with a SQLAlchemy table. + +**Tech Stack:** Python 3.12, FastAPI, Pydantic, SQLAlchemy 2, Starlette TestClient, pytest, Ruff, mypy + +## Global Constraints + +- Use only `webhook-id`, `webhook-timestamp`, and `webhook-signature`. +- Verify `webhook-id.webhook-timestamp.raw_body` with HMAC-SHA256 before JSON parsing. +- Accept `v1,` signatures and `whsec_` secrets; retain plain-secret compatibility. +- Require `FASTID_WEBHOOK_SECRET` during FastAPI lifespan startup. +- Use a 300-second timestamp tolerance and a 1 MiB advanced-example body limit. +- Keep all three example files standalone; they must not import each other or `fastid.*`. +- Keep event processing generic: validate and log receipt, but do not dispatch event-specific handlers. +- Prefix every shell command with `rtk`. + +## File structure + +- `examples/webhook_quickstart.py`: minimal authenticated receiver. +- `examples/webhook_advanced.py`: validated receiver with an async in-memory idempotency boundary. +- `examples/webhook_sqlalchemy.py`: validated receiver with durable SQLAlchemy claims. +- `tests/examples/webhook_helpers.py`: test-only signer and payload factory. +- `tests/examples/test_webhook_quickstart.py`: quick-start behavior and startup configuration. +- `tests/examples/test_webhook_advanced.py`: advanced validation, limits, duplicates, and concurrent claims. +- `tests/examples/test_webhook_sqlalchemy.py`: database claims, persistence, duplicates, and HTTP behavior. +- `docs/docs/tutorial/webhooks.md`: links to each runnable example and explains the audience. + +--- + +### Task 1: Quick-start receiver + +**Files:** +- Create: `tests/examples/__init__.py` +- Create: `tests/examples/webhook_helpers.py` +- Create: `tests/examples/test_webhook_quickstart.py` +- Create: `examples/webhook_quickstart.py` + +**Interfaces:** +- Produces: `app: FastAPI` +- Produces: `verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool` +- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401` + +- [ ] **Step 1: Add a test-only Standard Webhooks signer** + +Create `tests/examples/__init__.py` as an empty file. Create `tests/examples/webhook_helpers.py` with: + +```python +import base64 +import hashlib +import hmac +import json +import time +from typing import Any +from uuid import uuid4 + + +SECRET = f"whsec_{base64.b64encode(b'test-webhook-secret').decode()}" + + +def payload(event_id: str | None = None) -> dict[str, Any]: + event_id = event_id or str(uuid4()) + return { + "event": {"event_id": event_id, "event_type": "user_registration", "timestamp": int(time.time())}, + "data": {"user": {"id": str(uuid4()), "email": "person@example.com"}}, + } + + +def signed_request(value: dict[str, Any], secret: str = SECRET) -> tuple[bytes, dict[str, str]]: + body = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() + event_id = str(value["event"]["event_id"]) + timestamp = int(time.time()) + key = base64.b64decode(secret.removeprefix("whsec_"), validate=True) + signed = b".".join((event_id.encode(), str(timestamp).encode(), body)) + signature = base64.b64encode(hmac.new(key, signed, hashlib.sha256).digest()).decode() + return body, { + "webhook-id": event_id, + "webhook-timestamp": str(timestamp), + "webhook-signature": f"v1,{signature}", + "content-type": "application/json", + } +``` + +- [ ] **Step 2: Write failing quick-start HTTP tests** + +Create `tests/examples/test_webhook_quickstart.py` with tests that use `monkeypatch.setenv`, `TestClient`, and +`signed_request()` to assert: + +```python +def test_accepts_valid_webhook(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + assert response.status_code == 204 + assert response.content == b"" + + +def test_requires_secret_at_startup(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FASTID_WEBHOOK_SECRET", raising=False) + with pytest.raises(RuntimeError, match="FASTID_WEBHOOK_SECRET is required"), TestClient(app): + pass + + +def test_rejects_tampered_body(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body + b" ", headers=headers) + assert response.status_code == 401 +``` + +Add separate tests for missing headers, a non-integer timestamp, a timestamp older than 300 seconds, malformed JSON, +an invalid envelope, and a header/payload event-ID mismatch. Assert `400` for each. + +- [ ] **Step 3: Run quick-start tests and verify RED** + +```powershell +rtk pytest tests/examples/test_webhook_quickstart.py -q +``` + +Expected: collection fails because `examples.webhook_quickstart` does not exist. + +- [ ] **Step 4: Implement the standalone quick-start application** + +Create `examples/webhook_quickstart.py` with: + +- a FastAPI lifespan that reads and validates `FASTID_WEBHOOK_SECRET` into `app.state.webhook_secret`; +- `_secret_bytes()` supporting prefixed and plain secrets; +- `verify_signature()` that ignores unknown versions and constant-time compares every `v1` candidate; +- `POST /fastid-webhooks`, which reads the raw body, validates headers and timestamp, verifies before parsing JSON, + validates `event.event_id`, `event.event_type`, and `data`, requires ID equality, logs the generic receipt, and returns + `Response(status_code=204)`; +- `HTTPException(400, ...)` for malformed requests and `HTTPException(401, "Invalid webhook signature")` for a + non-matching signature; +- a production-idempotency warning immediately above the log call; +- `if __name__ == "__main__": uvicorn.run("examples.webhook_quickstart:app", host="127.0.0.1", port=8000)`. + +Use this exact signature input: + +```python +signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) +expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() +return any( + version == "v1" and hmac.compare_digest(value, expected) + for signature in signatures.split() + if "," in signature + for version, value in (signature.split(",", 1),) +) +``` + +- [ ] **Step 5: Verify GREEN and static correctness** + +```powershell +rtk pytest tests/examples/test_webhook_quickstart.py -q +rtk ruff check examples/webhook_quickstart.py tests/examples +rtk mypy examples/webhook_quickstart.py tests/examples +``` + +Expected: all checks pass. + +- [ ] **Step 6: Commit** + +```powershell +rtk git add examples/webhook_quickstart.py tests/examples +rtk git commit -m "docs: add webhook quick-start example" +``` + +--- + +### Task 2: Advanced in-memory receiver + +**Files:** +- Create: `tests/examples/test_webhook_advanced.py` +- Create: `examples/webhook_advanced.py` + +**Interfaces:** +- Produces: `IdempotencyStore(Protocol)` with async `claim(event_id) -> bool`, `complete(event_id) -> None`, and + `release(event_id) -> None`. +- Produces: `InMemoryIdempotencyStore` implementing that protocol with one `asyncio.Lock`. +- Produces: `create_app(store: IdempotencyStore | None = None, processor: EventProcessor = process_event) -> FastAPI`. +- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401 | 413 | 500`. + +- [ ] **Step 1: Write failing store and HTTP tests** + +Create `tests/examples/test_webhook_advanced.py`. Test the wished-for API directly: + +```python +async def test_concurrent_claim_has_one_winner() -> None: + store = InMemoryIdempotencyStore() + results = await asyncio.gather(*(store.claim("event-1") for _ in range(20))) + assert results.count(True) == 1 + assert results.count(False) == 19 + + +async def test_release_allows_retry() -> None: + store = InMemoryIdempotencyStore() + assert await store.claim("event-1") + await store.release("event-1") + assert await store.claim("event-1") +``` + +Use a recording async processor injected through `create_app()` and assert a repeated signed delivery returns `204` +twice but records one call. Add tests for declared and actual bodies over `MAX_BODY_BYTES`, invalid Pydantic envelope data, +ID mismatch, and a raising processor that returns `500` and can be retried successfully. + +- [ ] **Step 2: Run advanced tests and verify RED** + +```powershell +rtk pytest tests/examples/test_webhook_advanced.py -q +``` + +Expected: collection fails because `examples.webhook_advanced` does not exist. + +- [ ] **Step 3: Implement the advanced standalone application** + +Create `examples/webhook_advanced.py` with: + +```python +MAX_BODY_BYTES = 1024 * 1024 +ClaimStatus = Literal["processing", "completed"] +EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] + + +class EventMetadata(BaseModel): + event_id: UUID + event_type: str = Field(min_length=1) + timestamp: int + + +class WebhookEnvelope(BaseModel): + event: EventMetadata + data: dict[str, Any] + + +class IdempotencyStore(Protocol): + async def claim(self, event_id: str) -> bool: ... + async def complete(self, event_id: str) -> None: ... + async def release(self, event_id: str) -> None: ... +``` + +`InMemoryIdempotencyStore` must guard a `dict[str, ClaimStatus]` with one lock. `claim()` inserts `processing` only when +absent, `complete()` changes an existing claim to `completed`, and `release()` removes only a `processing` claim. + +Repeat the complete verification implementation from the quick-start file. `create_app()` must enforce both declared +and actual body length, validate with `WebhookEnvelope.model_validate_json(body)`, compare the normalized UUID to +`webhook-id`, claim before calling the processor, acknowledge duplicates, complete success, and release then raise +`HTTPException(500, "Webhook processing failed")` on processing/storage errors. The default processor logs the event. + +Add the same direct `uvicorn.run()` entry point on port 8000. + +- [ ] **Step 4: Verify GREEN and static correctness** + +```powershell +rtk pytest tests/examples/test_webhook_advanced.py -q +rtk ruff check examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk mypy examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +``` + +Expected: all checks pass. + +- [ ] **Step 5: Commit** + +```powershell +rtk git add examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk git commit -m "docs: add advanced webhook receiver example" +``` + +--- + +### Task 3: SQLAlchemy-backed receiver + +**Files:** +- Create: `tests/examples/test_webhook_sqlalchemy.py` +- Create: `examples/webhook_sqlalchemy.py` + +**Interfaces:** +- Produces: `WebhookReceipt` with unique string `event_id`, `status`, `created_at`, and `updated_at` columns. +- Produces: `SQLAlchemyIdempotencyStore(engine: Engine)` with synchronous `claim`, `complete`, and `release` methods. +- Produces: `create_app(database_url: str | None = None, processor: EventProcessor = process_event) -> FastAPI`. +- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401 | 413 | 500`. + +- [ ] **Step 1: Write failing SQLAlchemy store and HTTP tests** + +Create `tests/examples/test_webhook_sqlalchemy.py`. Use `tmp_path / "webhooks.sqlite3"` and two engines/store instances. +Assert: + +```python +def test_claim_persists_across_store_instances(tmp_path: Path) -> None: + database_url = f"sqlite:///{tmp_path / 'webhooks.sqlite3'}" + first = SQLAlchemyIdempotencyStore(create_store_engine(database_url)) + second = SQLAlchemyIdempotencyStore(create_store_engine(database_url)) + first.create_schema() + assert first.claim("event-1") + first.complete("event-1") + assert not second.claim("event-1") +``` + +Add a concurrent claim test through `asyncio.to_thread`, a release/reclaim test, valid/duplicate HTTP delivery with a +recording processor, a processing-failure retry test, and one parameterized test proving the SQLAlchemy app returns the +same `400`, `401`, and `413` classes as the advanced app. + +- [ ] **Step 2: Run SQLAlchemy tests and verify RED** + +```powershell +rtk pytest tests/examples/test_webhook_sqlalchemy.py -q +``` + +Expected: collection fails because `examples.webhook_sqlalchemy` does not exist. + +- [ ] **Step 3: Implement the standalone SQLAlchemy application** + +Create `examples/webhook_sqlalchemy.py` with a local `DeclarativeBase`, `WebhookReceipt`, and +`SQLAlchemyIdempotencyStore`. `create_store_engine(database_url)` must set `check_same_thread=False` only for SQLite. + +Use `Session.begin()` transactions. `claim()` inserts status `processing`, flushes, and returns `False` on +`IntegrityError`. `complete()` updates the matching row to `completed`. `release()` deletes only a `processing` row. +`create_schema()` calls `Base.metadata.create_all(engine)`. + +Repeat the advanced file's signature, size, and envelope validation so this file remains standalone. FastAPI lifespan +must require `FASTID_WEBHOOK_SECRET`, select the explicit argument or `WEBHOOK_DATABASE_URL` or +`sqlite:///webhook-events.sqlite3`, create the engine/store/schema, and dispose the engine at shutdown. + +Run every synchronous store call through `starlette.concurrency.run_in_threadpool`. A duplicate returns `204`; success +completes; failure releases and returns `500`. Add the direct uvicorn entry point. + +- [ ] **Step 4: Verify GREEN and static correctness** + +```powershell +rtk pytest tests/examples/test_webhook_sqlalchemy.py -q +rtk ruff check examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk mypy examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +``` + +Expected: all checks pass. + +- [ ] **Step 5: Commit** + +```powershell +rtk git add examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk git commit -m "docs: add SQLAlchemy webhook receiver example" +``` + +--- + +### Task 4: Tutorial links and final verification + +**Files:** +- Modify: `docs/docs/tutorial/webhooks.md` + +**Interfaces:** +- Consumes: all three runnable example paths. +- Produces: one discoverable progression from quick-start to production-oriented references. + +- [ ] **Step 1: Add example links to the tutorial** + +After the existing signature-verification fragment, add: + +```markdown +Runnable receiver examples are available for different integration stages: + +- [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies and logs an event with minimal setup. +- [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds validation and an in-memory idempotency boundary. +- [`webhook_sqlalchemy.py`](../../../examples/webhook_sqlalchemy.py) persists atomic event claims with SQLAlchemy. + +The in-memory example is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared, +persistent idempotency store before applying non-idempotent production side effects. +``` + +- [ ] **Step 2: Run focused example verification** + +```powershell +rtk pytest tests/examples -q +rtk ruff check examples tests/examples +rtk mypy examples tests/examples +rtk codespell examples tests/examples docs/docs/tutorial/webhooks.md +``` + +Expected: all checks pass. + +- [ ] **Step 3: Run the complete project verification** + +```powershell +rtk pytest -q +rtk ruff check . +rtk mypy fastid examples tests/examples +rtk git diff --check +rtk git status --short +``` + +Expected: all tests and static checks pass; only the tutorial is pending before the final commit. + +- [ ] **Step 4: Commit documentation and confirm history** + +```powershell +rtk git add docs/docs/tutorial/webhooks.md +rtk git commit -m "docs: link webhook receiver examples" +rtk git status --short +rtk git log -5 --oneline +``` + +Expected: the worktree is clean and the four example commits are at the top of `feat/enhance-webhooks`. From e7d2e3ca52f37174ef780ca245b9e51b6e3ea667 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:12:28 +0300 Subject: [PATCH 13/28] docs: add webhook quick-start example --- examples/webhook_quickstart.py | 116 ++++++++++++++++++++++ tests/examples/__init__.py | 0 tests/examples/test_webhook_quickstart.py | 105 ++++++++++++++++++++ tests/examples/webhook_helpers.py | 42 ++++++++ 4 files changed, 263 insertions(+) create mode 100644 examples/webhook_quickstart.py create mode 100644 tests/examples/__init__.py create mode 100644 tests/examples/test_webhook_quickstart.py create mode 100644 tests/examples/webhook_helpers.py diff --git a/examples/webhook_quickstart.py b/examples/webhook_quickstart.py new file mode 100644 index 0000000..7922668 --- /dev/null +++ b/examples/webhook_quickstart.py @@ -0,0 +1,116 @@ +import base64 +import binascii +import hashlib +import hmac +import json +import logging +import os +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, HTTPException, Request, Response, status + +TIMESTAMP_TOLERANCE_SECONDS = 300 + +log = logging.getLogger(__name__) + + +def _secret_bytes(secret: str) -> bytes: + if not secret.startswith("whsec_"): + return secret.encode() + try: + return base64.b64decode(secret.removeprefix("whsec_"), validate=True) + except (binascii.Error, ValueError) as exc: + msg = "FASTID_WEBHOOK_SECRET contains invalid base64" + raise ValueError(msg) from exc + + +def verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() + return any( + version == "v1" and hmac.compare_digest(value, expected) + for signature in signatures.split() + if "," in signature + for version, value in (signature.split(",", 1),) + ) + + +def _required_header(request: Request, name: str) -> str: + value = request.headers.get(name) + if not value: + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Missing {name} header") + return value + + +def _validate_payload(value: Any, webhook_id: str) -> tuple[str, str]: + if not isinstance(value, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") + event = value.get("event") + data = value.get("data") + if not isinstance(event, dict) or not isinstance(data, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") + event_id = event.get("event_id") + event_type = event.get("event_type") + event_timestamp = event.get("timestamp") + if not isinstance(event_id, str) or not isinstance(event_type, str) or not event_type.strip(): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event") + if type(event_timestamp) is not int: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event") + try: + normalized_event_id = str(UUID(event_id)) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event ID") from exc + if normalized_event_id != webhook_id: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "webhook-id does not match event.event_id") + return normalized_event_id, event_type + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + secret = os.getenv("FASTID_WEBHOOK_SECRET") + if not secret: + msg = "FASTID_WEBHOOK_SECRET is required" + raise RuntimeError(msg) + try: + _secret_bytes(secret) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + app.state.webhook_secret = secret + yield + + +app = FastAPI(lifespan=lifespan) + + +@app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) +async def receive_webhook(request: Request) -> Response: + body = await request.body() + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") + try: + payload = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid JSON body") from exc + event_id, event_type = _validate_payload(payload, webhook_id) + + # Production consumers must atomically record event_id before applying non-idempotent side effects. + log.info("Received FastID webhook: event_id=%s event_type=%s", event_id, event_type) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +if __name__ == "__main__": + uvicorn.run("examples.webhook_quickstart:app", host="127.0.0.1", port=8000) diff --git a/tests/examples/__init__.py b/tests/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/examples/test_webhook_quickstart.py b/tests/examples/test_webhook_quickstart.py new file mode 100644 index 0000000..f0589c8 --- /dev/null +++ b/tests/examples/test_webhook_quickstart.py @@ -0,0 +1,105 @@ +import time + +import pytest +from fastapi.testclient import TestClient +from starlette import status + +from examples.webhook_quickstart import app +from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request + + +def test_accepts_valid_webhook(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_204_NO_CONTENT + assert response.content == b"" + + +def test_requires_secret_at_startup(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FASTID_WEBHOOK_SECRET", raising=False) + + with pytest.raises(RuntimeError, match="FASTID_WEBHOOK_SECRET is required"), TestClient(app): + pass + + +def test_rejects_tampered_body(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body + b" ", headers=headers) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +@pytest.mark.parametrize("header", ["webhook-id", "webhook-timestamp", "webhook-signature"]) +def test_rejects_missing_header(monkeypatch: pytest.MonkeyPatch, header: str) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + headers.pop(header) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_non_integer_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + headers["webhook-timestamp"] = "invalid" + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, str(value["event"]["event_id"]), timestamp=int(time.time()) - 301) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_malformed_json(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"not-json" + headers = headers_for(body, "event-1") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"{}" + headers = headers_for(body, "event-1") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_header_payload_id_mismatch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "different-event") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/tests/examples/webhook_helpers.py b/tests/examples/webhook_helpers.py new file mode 100644 index 0000000..7ffb926 --- /dev/null +++ b/tests/examples/webhook_helpers.py @@ -0,0 +1,42 @@ +import base64 +import hashlib +import hmac +import json +import time +from typing import Any +from uuid import uuid4 + +SECRET = f"whsec_{base64.b64encode(b'test-webhook-secret').decode()}" + + +def payload(event_id: str | None = None) -> dict[str, Any]: + event_id = event_id or str(uuid4()) + return { + "event": {"event_id": event_id, "event_type": "user_registration", "timestamp": int(time.time())}, + "data": {"user": {"id": str(uuid4()), "email": "person@example.com"}}, + } + + +def headers_for( + body: bytes, + event_id: str, + *, + secret: str = SECRET, + timestamp: int | None = None, +) -> dict[str, str]: + timestamp = timestamp or int(time.time()) + key = base64.b64decode(secret.removeprefix("whsec_"), validate=True) + signed = b".".join((event_id.encode(), str(timestamp).encode(), body)) + signature = base64.b64encode(hmac.new(key, signed, hashlib.sha256).digest()).decode() + return { + "webhook-id": event_id, + "webhook-timestamp": str(timestamp), + "webhook-signature": f"v1,{signature}", + "content-type": "application/json", + } + + +def signed_request(value: dict[str, Any], secret: str = SECRET) -> tuple[bytes, dict[str, str]]: + body = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() + event_id = str(value["event"]["event_id"]) + return body, headers_for(body, event_id, secret=secret) From d072e936beef1377f482c34d7739e55ec1549b85 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:16:04 +0300 Subject: [PATCH 14/28] docs: add advanced webhook receiver example --- examples/webhook_advanced.py | 205 ++++++++++++++++++++++++ tests/examples/test_webhook_advanced.py | 118 ++++++++++++++ 2 files changed, 323 insertions(+) create mode 100644 examples/webhook_advanced.py create mode 100644 tests/examples/test_webhook_advanced.py diff --git a/examples/webhook_advanced.py b/examples/webhook_advanced.py new file mode 100644 index 0000000..291e7a8 --- /dev/null +++ b/examples/webhook_advanced.py @@ -0,0 +1,205 @@ +import asyncio +import base64 +import binascii +import hashlib +import hmac +import logging +import os +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from typing import Any, Literal, Protocol +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, HTTPException, Request, Response, status +from pydantic import BaseModel, Field, ValidationError + +MAX_BODY_BYTES = 1024 * 1024 +TIMESTAMP_TOLERANCE_SECONDS = 300 +ClaimStatus = Literal["processing", "completed"] +EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] + +log = logging.getLogger(__name__) + + +class EventMetadata(BaseModel): + event_id: UUID + event_type: str = Field(min_length=1) + timestamp: int + + +class WebhookEnvelope(BaseModel): + event: EventMetadata + data: dict[str, Any] + + +class IdempotencyStore(Protocol): + async def claim(self, event_id: str) -> bool: ... + + async def complete(self, event_id: str) -> None: ... + + async def release(self, event_id: str) -> None: ... + + +class InMemoryIdempotencyStore: + """Concurrency-safe within one process, but neither shared nor durable.""" + + def __init__(self) -> None: + self._claims: dict[str, ClaimStatus] = {} + self._lock = asyncio.Lock() + + async def claim(self, event_id: str) -> bool: + async with self._lock: + if event_id in self._claims: + return False + self._claims[event_id] = "processing" + return True + + async def complete(self, event_id: str) -> None: + async with self._lock: + if event_id in self._claims: + self._claims[event_id] = "completed" + + async def release(self, event_id: str) -> None: + async with self._lock: + if self._claims.get(event_id) == "processing": + del self._claims[event_id] + + +def _secret_bytes(secret: str) -> bytes: + if not secret.startswith("whsec_"): + return secret.encode() + try: + return base64.b64decode(secret.removeprefix("whsec_"), validate=True) + except (binascii.Error, ValueError) as exc: + msg = "FASTID_WEBHOOK_SECRET contains invalid base64" + raise ValueError(msg) from exc + + +def verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() + return any( + version == "v1" and hmac.compare_digest(value, expected) + for signature in signatures.split() + if "," in signature + for version, value in (signature.split(",", 1),) + ) + + +def _required_header(request: Request, name: str) -> str: + value = request.headers.get(name) + if not value: + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Missing {name} header") + return value + + +def _content_length(request: Request) -> int | None: + value = request.headers.get("content-length") + if value is None: + return None + try: + return int(value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid Content-Length header") from exc + + +async def process_event(event: WebhookEnvelope) -> None: + log.info( + "Received FastID webhook: event_id=%s event_type=%s", + event.event.event_id, + event.event.event_type, + ) + + +async def _validated_event(request: Request) -> WebhookEnvelope: + declared_length = _content_length(request) + if declared_length is not None and declared_length > MAX_BODY_BYTES: + raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") + body = await request.body() + if len(body) > MAX_BODY_BYTES: + raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") + + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") + + try: + event = WebhookEnvelope.model_validate_json(body) + except ValidationError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") from exc + if str(event.event.event_id) != webhook_id: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "webhook-id does not match event.event_id") + return event + + +async def _receive_webhook( + request: Request, + idempotency_store: IdempotencyStore, + processor: EventProcessor, +) -> Response: + event = await _validated_event(request) + event_id = str(event.event.event_id) + try: + claimed = await idempotency_store.claim(event_id) + except Exception as exc: + log.exception("Could not claim FastID webhook event_id=%s", event_id) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc + if not claimed: + return Response(status_code=status.HTTP_204_NO_CONTENT) + + try: + await processor(event) + await idempotency_store.complete(event_id) + except Exception as exc: + log.exception("FastID webhook processing failed: event_id=%s", event_id) + try: + await idempotency_store.release(event_id) + except Exception: + log.exception("Could not release FastID webhook claim: event_id=%s", event_id) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def create_app( + store: IdempotencyStore | None = None, + processor: EventProcessor = process_event, +) -> FastAPI: + idempotency_store = store or InMemoryIdempotencyStore() + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + secret = os.getenv("FASTID_WEBHOOK_SECRET") + if not secret: + msg = "FASTID_WEBHOOK_SECRET is required" + raise RuntimeError(msg) + try: + _secret_bytes(secret) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + app.state.webhook_secret = secret + yield + + webhook_app = FastAPI(lifespan=lifespan) + + @webhook_app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) + async def receive_webhook(request: Request) -> Response: + return await _receive_webhook(request, idempotency_store, processor) + + return webhook_app + + +app = create_app() + + +if __name__ == "__main__": + uvicorn.run("examples.webhook_advanced:app", host="127.0.0.1", port=8000) diff --git a/tests/examples/test_webhook_advanced.py b/tests/examples/test_webhook_advanced.py new file mode 100644 index 0000000..860eb02 --- /dev/null +++ b/tests/examples/test_webhook_advanced.py @@ -0,0 +1,118 @@ +import asyncio + +import pytest +from fastapi.testclient import TestClient +from starlette import status + +from examples.webhook_advanced import ( + MAX_BODY_BYTES, + InMemoryIdempotencyStore, + WebhookEnvelope, + create_app, +) +from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request + +CLAIM_COUNT = 20 +EXPECTED_DUPLICATE_CLAIMS = CLAIM_COUNT - 1 +EXPECTED_PROCESS_CALLS = 2 + + +async def test_concurrent_claim_has_one_winner() -> None: + store = InMemoryIdempotencyStore() + + results = await asyncio.gather(*(store.claim("event-1") for _ in range(CLAIM_COUNT))) + + assert results.count(True) == 1 + assert results.count(False) == EXPECTED_DUPLICATE_CLAIMS + + +async def test_release_allows_retry() -> None: + store = InMemoryIdempotencyStore() + assert await store.claim("event-1") + + await store.release("event-1") + + assert await store.claim("event-1") + + +def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + processed: list[str] = [] + + async def record(event: WebhookEnvelope) -> None: + processed.append(str(event.event.event_id)) + + body, headers = signed_request(payload()) + with TestClient(create_app(processor=record)) as client: + first = client.post("/fastid-webhooks", content=body, headers=headers) + duplicate = client.post("/fastid-webhooks", content=body, headers=headers) + + assert first.status_code == status.HTTP_204_NO_CONTENT + assert duplicate.status_code == status.HTTP_204_NO_CONTENT + assert processed == [headers["webhook-id"]] + + +def test_rejects_declared_oversized_body(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + headers["content-length"] = str(MAX_BODY_BYTES + 1) + + with TestClient(create_app()) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE + + +def test_rejects_actual_oversized_body(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"x" * (MAX_BODY_BYTES + 1) + headers = headers_for(body, "event-1") + + with TestClient(create_app()) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE + + +def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + value["event"]["event_type"] = "" + body, headers = signed_request(value) + + with TestClient(create_app()) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_header_payload_id_mismatch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "00000000-0000-0000-0000-000000000000") + + with TestClient(create_app()) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_processing_failure_releases_claim(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + calls = 0 + + async def fail_once(event: WebhookEnvelope) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError(str(event.event.event_id)) + + body, headers = signed_request(payload()) + with TestClient(create_app(processor=fail_once)) as client: + failed = client.post("/fastid-webhooks", content=body, headers=headers) + retried = client.post("/fastid-webhooks", content=body, headers=headers) + + assert failed.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert retried.status_code == status.HTTP_204_NO_CONTENT + assert calls == EXPECTED_PROCESS_CALLS From 49704834c44736ddeaa825fac8ed27f917ad06fd Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:42:08 +0300 Subject: [PATCH 15/28] docs: define webhook identifier terminology --- docs/adr/0001-use-standard-webhook-headers.md | 2 +- ...rovide-tiered-webhook-receiver-examples.md | 4 +- docs/glossary.md | 17 ++++-- ...6-07-20-standard-webhook-headers-design.md | 16 ++--- ...0-webhook-identifier-terminology-design.md | 61 +++++++++++++++++++ ...-07-20-webhook-receiver-examples-design.md | 28 ++++----- 6 files changed, 100 insertions(+), 28 deletions(-) create mode 100644 docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md diff --git a/docs/adr/0001-use-standard-webhook-headers.md b/docs/adr/0001-use-standard-webhook-headers.md index f6c2e7b..b596b46 100644 --- a/docs/adr/0001-use-standard-webhook-headers.md +++ b/docs/adr/0001-use-standard-webhook-headers.md @@ -20,7 +20,7 @@ will be removed. ## Consequences - New examples and integrations have one authentication contract. -- The stable event ID is unambiguously the idempotency key. +- The stable Webhook ID is unambiguously the consumer idempotency key. - Verification authenticates the exact transmitted body. - Existing consumers of `X-Webhook-*` must migrate to Standard Webhooks headers. - Supporting a future signing change will require a new version in `webhook-signature`, not a second header family. diff --git a/docs/adr/0002-provide-tiered-webhook-receiver-examples.md b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md index f631bcd..ae6397e 100644 --- a/docs/adr/0002-provide-tiered-webhook-receiver-examples.md +++ b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md @@ -14,8 +14,8 @@ code would reduce duplication but make each example harder to copy into another FastID will provide three standalone FastAPI receiver applications: - a quick-start authentication example; -- an advanced in-memory idempotency reference; -- an advanced SQLAlchemy idempotency reference. +- an advanced in-memory Webhook-ID idempotency reference; +- an advanced SQLAlchemy Webhook-ID idempotency reference. Each application will independently implement Standard Webhooks verification using Python's standard library. Advanced examples will share the same conceptual claim, complete, and release lifecycle but will not import each other. diff --git a/docs/glossary.md b/docs/glossary.md index 727bfc8..4e5326b 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -6,14 +6,14 @@ : One HTTP request made for a webhook event. Retries are separate attempts with a new timestamp and signature. **Claim** -: An atomic operation that reserves an event ID for processing. Only the request that creates the claim may apply the +: An atomic operation that reserves a Webhook ID for processing. Only the request that creates the claim may apply the event's side effects. **Complete** : Mark a claimed event as successfully processed so later deliveries are acknowledged without repeating work. **Duplicate delivery** -: A delivery whose event ID has already been claimed or completed. Consumers acknowledge it without applying the event +: A delivery whose Webhook ID has already been claimed or completed. Consumers acknowledge it without applying the event again. **Endpoint** @@ -23,16 +23,25 @@ again. : A logical user-lifecycle occurrence emitted by FastID. One event can have multiple delivery attempts. **Event ID** -: The stable UUID sent in `webhook-id` and in `event.event_id`. Consumers use it as the idempotency key. +: The logical domain event UUID sent in `event.event_id`. One event can create separate messages for multiple webhook +endpoints. **Idempotency key** : A stable identifier atomically recorded by a consumer to prevent repeated side effects when an event is delivered more than once. **Idempotency store** -: A component that atomically claims event IDs and records their processing state. A production implementation must be +: A component that atomically claims Webhook IDs and records their processing state. A production implementation must be shared by all receiver processes and survive restarts. +**Webhook Endpoint** +: A configured webhook destination containing its URL, secret, event type, and activation state. FastID represents it +as `WebhookEndpoint`; a delivery refers to it through `endpoint_id`. + +**Webhook ID** +: The delivery identifier sent in `webhook-id`. It is stable across retries of that delivery and is the consumer's +idempotency key. FastID stores it as `WebhookDelivery.id`. + **Release** : Remove or expire an incomplete claim after processing fails, allowing a later FastID retry to claim the event again. diff --git a/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md b/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md index 1364d3d..b230937 100644 --- a/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md +++ b/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md @@ -7,9 +7,10 @@ FastID currently sends two independent webhook authentication protocols on every - legacy configurable `X-Webhook-*` headers, whose signature covers a re-serialized JSON payload; - fixed Standard Webhooks headers, whose signature covers the exact raw request body. -The legacy `X-Webhook-Id` identifies a delivery, while the standard `webhook-id` identifies the logical event and remains -stable across retries. Keeping both protocols makes the consumer contract ambiguous and complicates the webhook -examples. +Both the legacy `X-Webhook-Id` and standard `webhook-id` identify one endpoint delivery and remain stable across its +retries. The payload's `event.event_id` separately identifies the logical domain event, which can produce deliveries to +multiple endpoints. Keeping both authentication protocols makes the consumer contract ambiguous and complicates the +webhook examples. The repository has no production caller of the legacy signing or verification functions. The legacy protocol remains only in the delivery-header composition, configuration, tests, and a compatibility note in the webhook tutorial. There @@ -19,7 +20,7 @@ are no tagged releases establishing a published compatibility requirement. FastID will expose only the Standard Webhooks header family: -- `webhook-id`: stable logical event ID; +- `webhook-id`: stable Webhook ID, represented by `WebhookDelivery.id`; - `webhook-timestamp`: Unix timestamp for the delivery attempt; - `webhook-signature`: one or more space-separated `v1,` signatures. @@ -29,7 +30,7 @@ The signed content remains the exact byte sequence ## Code changes `generate_delivery_headers()` will directly construct the standard headers plus `Content-Type` and `User-Agent`. It -will no longer accept or use a separately generated delivery ID for authentication. +will sign `WebhookDelivery.id` as the Webhook ID. The following legacy API will be removed: @@ -47,8 +48,9 @@ internal identifier and is not sent as an authentication header. ## Request flow -For each delivery attempt, the worker serializes the payload once, generates a timestamp, and signs the stable event ID, -timestamp, and exact serialized bytes. The sender transmits those same bytes and the generated standard headers. +For each delivery attempt, the worker serializes the payload once, generates a timestamp, and signs the stable Webhook +ID, timestamp, and exact serialized bytes. The sender transmits those same bytes and the generated standard +headers. The payload retains its separate logical event ID. Consumers read the raw body, validate timestamp freshness, calculate the expected HMAC over the raw bytes, compare the signature in constant time, and only then parse JSON. Consumers use `webhook-id` as the idempotency key. diff --git a/docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md b/docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md new file mode 100644 index 0000000..268399a --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md @@ -0,0 +1,61 @@ +# Webhook Identifier Terminology Design + +## Problem + +FastID has three independent identifiers in its webhook flow: + +- the configured destination currently represented by `Webhook.id`; +- the individual delivery represented by `WebhookDelivery.id`; +- the domain event represented by `event.event_id`. + +Calling the configured destination simply a webhook makes `webhook_id` ambiguous. The Standard Webhooks +`webhook-id` header identifies the delivered webhook, not the configured destination or the domain event. + +## Decision + +FastID will use these domain terms: + +- **Webhook Endpoint**: the configured URL, secret, event type, and activation state; +- **Webhook ID**: the stable identifier of one delivered webhook, sent in `webhook-id` and represented by + `WebhookDelivery.id`; +- **Event ID**: the logical domain event identifier sent in `event.event_id`. + +The Webhook ID remains stable across retries of the same delivery. One event can create multiple webhook deliveries, +each with its own Webhook ID, when it is sent to multiple endpoints. + +## Code and persistence names + +Domain terminology remains explicit in documentation and user-facing text. Attributes use the shortest unambiguous +name within their owning model: + +| Concept | Code or persistence representation | +| --- | --- | +| Webhook Endpoint | `WebhookEndpoint` | +| Endpoint table | `webhook_endpoints` | +| Webhook Endpoint ID on a delivery | `WebhookDelivery.endpoint_id` | +| Endpoint relationship on a delivery | `WebhookDelivery.endpoint` | +| Webhook ID | `WebhookDelivery.id`, local variable `webhook_id`, HTTP header `webhook-id` | +| Event ID | `Event.event_id` and `WebhookDelivery.event_id` | + +The former `Webhook` model, `webhooks` table, `WebhookDelivery.webhook_id` foreign key, and +`WebhookDelivery.webhook` relationship will be renamed accordingly. Public routes may remain under `/webhooks` +because that path names the webhook feature rather than one identifier. + +## Signing and receiving + +FastID signs `webhook_id.timestamp.raw_body`, where `webhook_id` is `WebhookDelivery.id`. Sender and receiver code +will consistently use the variable name `webhook_id`; it will not use `message_id` or `delivery_id` as aliases. + +Receiver examples will treat `webhook-id` and `event.event_id` as independent values. The quick-start verifies only +the signature and treats the Webhook ID and timestamp as opaque signature inputs. Advanced examples additionally +validate timestamp freshness and use the Webhook ID as their idempotency key. + +## Migration and compatibility + +The database schema will rename `webhooks` to `webhook_endpoints` and the delivery foreign-key column to +`endpoint_id`, preserving existing identifiers and relationships. The external signing contract keeps the standard +header name `webhook-id`; only the value semantics are clarified as the Webhook ID represented by +`WebhookDelivery.id`. + +Tests will verify that the header Webhook ID equals `WebhookDelivery.id`, differs independently from the Event ID, +and remains stable across retries. diff --git a/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md index 0795726..2da1499 100644 --- a/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md +++ b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md @@ -17,8 +17,9 @@ The quick-start application is the shortest safe receiver. It requires `FASTID_W exact raw body, validates the three Standard Webhooks headers and timestamp, parses JSON only after authentication, logs the generic event, and returns `204 No Content`. -It demonstrates authentication but not idempotency. A comment warns that production consumers must atomically record -`webhook-id` before applying non-idempotent side effects. +It demonstrates authentication but not replay protection or idempotency. It reads `webhook-id` and +`webhook-timestamp` only because they are signature inputs, without checking timestamp freshness or interpreting the +Webhook ID. A comment directs production consumers to the advanced examples before applying side effects. ### `examples/webhook_advanced.py` @@ -33,7 +34,7 @@ setup into this example. ### `examples/webhook_sqlalchemy.py` The SQLAlchemy application is a standalone alternative to the advanced example. It implements an equivalent claim, -complete, and release lifecycle with a table whose event ID is unique. A duplicate insert is the atomic concurrency +complete, and release lifecycle with a table whose Webhook ID is unique. A duplicate insert is the atomic concurrency boundary. `WEBHOOK_DATABASE_URL` configures the database and defaults to a local SQLite file so the example is runnable without @@ -59,7 +60,7 @@ The SQLAlchemy application additionally accepts `WEBHOOK_DATABASE_URL`, defaulti Each receiver reads: -- `webhook-id` as the stable event and idempotency identifier; +- `webhook-id` as the stable Webhook ID and idempotency identifier; - `webhook-timestamp` as an integer Unix timestamp; - `webhook-signature` as one or more space-separated versioned signatures. @@ -68,13 +69,14 @@ The expected `v1` signature is base64-encoded HMAC-SHA256 over the exact byte se parsed or re-serialized before verification. Unknown signature versions are ignored so a future FastID rotation can send old and new signatures together. A request -is authenticated when any supplied `v1` signature matches. +is authenticated when any supplied `v1` signature matches. The quick-start stops at this authentication check. The +advanced examples additionally parse the timestamp as an integer and reject messages outside the tolerance. ## Payload model The examples are event-type agnostic. After signature verification, they require a JSON object with: -- `event.event_id`: a UUID string equal to `webhook-id`; +- `event.event_id`: the logical domain event UUID, independent of `webhook-id`; - `event.event_type`: a non-empty string; - `event.timestamp`: an integer; - `data`: a JSON object. @@ -90,11 +92,10 @@ envelope before claiming the event. 2. Read the raw body and reject an actual body greater than 1 MiB. 3. Validate required headers, integer timestamp, timestamp tolerance, secret encoding, and HMAC. 4. Parse and validate the generic JSON envelope. -5. Require `event.event_id` to equal `webhook-id`. -6. Atomically claim the event ID. -7. If already claimed or completed, return `204` without processing it again. -8. Log receipt and mark the claim complete. -9. If processing raises, release the claim and return `500` so FastID retries. +5. Atomically claim the Webhook ID from `webhook-id`. +6. If already claimed or completed, return `204` without processing it again. +7. Log receipt and mark the claim complete. +8. If processing raises, release the claim and return `500` so FastID retries. The claim lifecycle prevents simultaneous duplicate processing in the demonstrated execution model. It does not promise exactly-once side effects across a crash between the side effect and completion; production applications must make the @@ -102,8 +103,8 @@ business change idempotent or commit an inbox record and business state in one t ## Error responses -- `400 Bad Request`: missing or malformed webhook headers, stale timestamp, malformed JSON, invalid event envelope, or - header/payload event-ID mismatch; +- `400 Bad Request`: missing or malformed webhook headers, stale timestamp in advanced examples, malformed JSON, or an + invalid event envelope; - `401 Unauthorized`: a well-formed request with no matching signature; - `413 Content Too Large`: declared or actual body exceeds 1 MiB; - `500 Internal Server Error`: processing or idempotency storage fails, allowing FastID to retry; @@ -124,7 +125,6 @@ Coverage includes: - missing, malformed, and non-matching signatures; - stale timestamps; - malformed JSON and invalid envelope fields; -- header/payload event-ID mismatch; - declared and actual oversized bodies in advanced applications; - repeated delivery acknowledgement without repeated processing; - concurrent claims against the in-memory adapter; From b0c79b1b4d42648ead4d4fe6217b46238a45a5c5 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:49:18 +0300 Subject: [PATCH 16/28] docs: plan webhook identifier and example updates --- .../2026-07-20-webhook-receiver-examples.md | 678 ++++++++++++------ 1 file changed, 458 insertions(+), 220 deletions(-) diff --git a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md index f86fd9b..b6b15a9 100644 --- a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md +++ b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md @@ -2,25 +2,39 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Add three independently runnable FastAPI webhook receivers for quick-start, in-memory advanced, and SQLAlchemy-backed use cases. +**Goal:** Clarify FastID's webhook identifiers and provide three independently runnable FastAPI webhook receivers for quick-start, in-memory advanced, and SQLAlchemy-backed use cases. -**Architecture:** Every example contains its own Standard Webhooks verifier so it can be copied independently. The quick-start authenticates and logs generic events; the advanced examples add a claim/complete/release idempotency lifecycle, implemented first with a locked in-memory store and then with a SQLAlchemy table. +**Architecture:** FastID models a configured destination as `WebhookEndpoint`, a delivered webhook as `WebhookDelivery`, and a domain event as `Event`. `WebhookDelivery.id` is the Webhook ID signed into `webhook-id`; `WebhookDelivery.endpoint_id` links to the endpoint; `event.event_id` remains independent. Every example contains its own Standard Webhooks verifier so it can be copied without importing FastID. -**Tech Stack:** Python 3.12, FastAPI, Pydantic, SQLAlchemy 2, Starlette TestClient, pytest, Ruff, mypy +**Tech Stack:** Python 3.12, FastAPI, Pydantic, SQLAlchemy 2, Alembic, Starlette TestClient, pytest, Ruff, mypy ## Global Constraints -- Use only `webhook-id`, `webhook-timestamp`, and `webhook-signature`. +- Use **Webhook Endpoint ID**, **Webhook ID**, and **Event ID** as the domain terms. +- Use concise contextual attributes: `WebhookDelivery.endpoint_id` and `WebhookDelivery.endpoint`. +- Use `webhook_id` internally and `webhook-id` externally for `WebhookDelivery.id`; do not use `message_id` or `delivery_id` aliases for this value. +- Treat `event.event_id` as independent from `webhook-id`; never compare them. +- Use only `webhook-id`, `webhook-timestamp`, and `webhook-signature` authentication headers. - Verify `webhook-id.webhook-timestamp.raw_body` with HMAC-SHA256 before JSON parsing. - Accept `v1,` signatures and `whsec_` secrets; retain plain-secret compatibility. - Require `FASTID_WEBHOOK_SECRET` during FastAPI lifespan startup. -- Use a 300-second timestamp tolerance and a 1 MiB advanced-example body limit. +- The quick-start performs signature authentication only: no timestamp freshness, identifier-format, replay, or idempotency checks. +- Use a 300-second timestamp tolerance and a 1 MiB body limit in advanced examples only. - Keep all three example files standalone; they must not import each other or `fastid.*`. - Keep event processing generic: validate and log receipt, but do not dispatch event-specific handlers. +- Do not demonstrate a FastID application database in the quick-start or in-memory examples. - Prefix every shell command with `rtk`. ## File structure +- `fastid/webhooks/models.py`: endpoint, delivery, and attempt persistence models. +- `fastid/apps/models.py`: application-to-webhook-endpoint relationship. +- `fastid/webhooks/repositories.py`: endpoint and delivery repositories/specifications. +- `fastid/database/uow.py`: repository attributes used by webhook use cases and workers. +- `fastid/webhooks/use_cases.py`: creates one delivery per matching endpoint. +- `fastid/webhooks/worker.py`: signs Webhook IDs and resolves endpoint relationships. +- `fastid/security/webhooks.py`: Standard Webhooks signing and verification helpers. +- `migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py`: lossless endpoint table, version table, column, constraint, and index renames. - `examples/webhook_quickstart.py`: minimal authenticated receiver. - `examples/webhook_advanced.py`: validated receiver with an async in-memory idempotency boundary. - `examples/webhook_sqlalchemy.py`: validated receiver with durable SQLAlchemy claims. @@ -28,266 +42,455 @@ - `tests/examples/test_webhook_quickstart.py`: quick-start behavior and startup configuration. - `tests/examples/test_webhook_advanced.py`: advanced validation, limits, duplicates, and concurrent claims. - `tests/examples/test_webhook_sqlalchemy.py`: database claims, persistence, duplicates, and HTTP behavior. -- `docs/docs/tutorial/webhooks.md`: links to each runnable example and explains the audience. +- `docs/docs/tutorial/webhooks.md`: receiver contract and links to each runnable example. --- -### Task 1: Quick-start receiver +### Task 1: Rename configured webhooks to webhook endpoints **Files:** -- Create: `tests/examples/__init__.py` -- Create: `tests/examples/webhook_helpers.py` -- Create: `tests/examples/test_webhook_quickstart.py` -- Create: `examples/webhook_quickstart.py` +- Create: `tests/webhooks/test_endpoint_models.py` +- Create: `migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py` +- Modify: `fastid/webhooks/models.py` +- Modify: `fastid/apps/models.py` +- Modify: `fastid/webhooks/repositories.py` +- Modify: `fastid/database/models.py` +- Modify: `fastid/database/versioning.py` +- Modify: `fastid/database/uow.py` +- Modify: `fastid/webhooks/use_cases.py` +- Modify: `fastid/webhooks/worker.py` +- Modify: `fastid/admin/views/settings.py` +- Modify: `fastid/admin/views/versioning.py` +- Modify: `tests/utils/webhooks.py` +- Modify: `tests/api/conftest.py` +- Modify: webhook API tests importing the persistence model under `tests/api/auth/`, `tests/api/profile/`, and `tests/api/webhooks/` **Interfaces:** -- Produces: `app: FastAPI` -- Produces: `verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool` -- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401` +- Produces: `WebhookEndpoint(VersionedEntity)` mapped to `webhook_endpoints`. +- Produces: `WebhookDelivery.endpoint_id: UUID` and `WebhookDelivery.endpoint: WebhookEndpoint`. +- Produces: `App.webhook_endpoints: list[WebhookEndpoint]`. +- Produces: `WebhookEndpointRepository`, `WebhookEndpointTypeSpecification`, and `WebhookDeliveryEndpointIDSpecification`. +- Preserves: all existing endpoint, delivery, and version row IDs during migration. -- [ ] **Step 1: Add a test-only Standard Webhooks signer** +- [ ] **Step 1: Write the failing model-contract test** -Create `tests/examples/__init__.py` as an empty file. Create `tests/examples/webhook_helpers.py` with: +Create `tests/webhooks/test_endpoint_models.py`: ```python -import base64 -import hashlib -import hmac -import json -import time -from typing import Any -from uuid import uuid4 +from fastid.apps.models import App +from fastid.webhooks.models import WebhookDelivery, WebhookEndpoint -SECRET = f"whsec_{base64.b64encode(b'test-webhook-secret').decode()}" - +def test_webhook_endpoint_and_delivery_mapping_names() -> None: + assert WebhookEndpoint.__tablename__ == "webhook_endpoints" + assert WebhookDelivery.__table__.c.endpoint_id.name == "endpoint_id" + foreign_key = next(iter(WebhookDelivery.__table__.c.endpoint_id.foreign_keys)) + assert foreign_key.target_fullname == "webhook_endpoints.id" + assert WebhookDelivery.endpoint.property.mapper.class_ is WebhookEndpoint + assert App.webhook_endpoints.property.mapper.class_ is WebhookEndpoint +``` -def payload(event_id: str | None = None) -> dict[str, Any]: - event_id = event_id or str(uuid4()) - return { - "event": {"event_id": event_id, "event_type": "user_registration", "timestamp": int(time.time())}, - "data": {"user": {"id": str(uuid4()), "email": "person@example.com"}}, - } +- [ ] **Step 2: Run the test and verify RED** +Run: -def signed_request(value: dict[str, Any], secret: str = SECRET) -> tuple[bytes, dict[str, str]]: - body = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() - event_id = str(value["event"]["event_id"]) - timestamp = int(time.time()) - key = base64.b64decode(secret.removeprefix("whsec_"), validate=True) - signed = b".".join((event_id.encode(), str(timestamp).encode(), body)) - signature = base64.b64encode(hmac.new(key, signed, hashlib.sha256).digest()).decode() - return body, { - "webhook-id": event_id, - "webhook-timestamp": str(timestamp), - "webhook-signature": f"v1,{signature}", - "content-type": "application/json", - } +```powershell +rtk proxy poetry run pytest tests/webhooks/test_endpoint_models.py -q ``` -- [ ] **Step 2: Write failing quick-start HTTP tests** +Expected: collection fails because `WebhookEndpoint` does not exist. + +- [ ] **Step 3: Rename the ORM model and contextual attributes** -Create `tests/examples/test_webhook_quickstart.py` with tests that use `monkeypatch.setenv`, `TestClient`, and -`signed_request()` to assert: +Implement these exact mappings in `fastid/webhooks/models.py` and `fastid/apps/models.py`: ```python -def test_accepts_valid_webhook(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body, headers = signed_request(payload()) - with TestClient(app) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - assert response.status_code == 204 - assert response.content == b"" +class WebhookEndpoint(VersionedEntity): + __tablename__ = "webhook_endpoints" + + app_id: Mapped[UUID] = mapped_column(ForeignKey("apps.id"), index=True) + type: Mapped[WebhookType] + secret: Mapped[str] = mapped_column(default=generate_webhook_secret) + url: Mapped[str] + is_active: Mapped[bool] = mapped_column(default=True) + disabled_at: Mapped[datetime.datetime | None] + disabled_reason: Mapped[str | None] + + app: Mapped[App] = relationship(back_populates="webhook_endpoints") + deliveries: Mapped[list[WebhookDelivery]] = relationship(back_populates="endpoint", cascade="delete") + + +class WebhookDelivery(Entity): + __tablename__ = "webhook_deliveries" + + endpoint_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_endpoints.id"), index=True) + event_id: Mapped[UUID] = mapped_column(index=True) + event_type: Mapped[WebhookType] + payload: Mapped[dict[str, Any]] + status: Mapped[WebhookDeliveryStatus] = mapped_column(default=WebhookDeliveryStatus.pending, index=True) + attempt_count: Mapped[int] = mapped_column(default=0) + next_attempt_at: Mapped[datetime.datetime] = mapped_column(index=True) + leased_until: Mapped[datetime.datetime | None] = mapped_column(index=True) + completed_at: Mapped[datetime.datetime | None] + request: Mapped[dict[str, Any] | None] + status_code: Mapped[int | None] + response: Mapped[dict[str, Any] | None] + error: Mapped[str | None] + + endpoint: Mapped[WebhookEndpoint] = relationship(back_populates="deliveries") + attempts: Mapped[list[WebhookAttempt]] = relationship(back_populates="delivery", cascade="delete") +``` +```python +class App(VersionedEntity): + __tablename__ = "apps" -def test_requires_secret_at_startup(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("FASTID_WEBHOOK_SECRET", raising=False) - with pytest.raises(RuntimeError, match="FASTID_WEBHOOK_SECRET is required"), TestClient(app): - pass + name: Mapped[str] + slug: Mapped[str] = mapped_column(unique=True) + client_id: Mapped[str] = mapped_column(default=uuid_hex, index=True) + client_secret: Mapped[str] = mapped_column(default=uuid_hex) + redirect_uris: Mapped[str] = mapped_column(default="") + is_active: Mapped[bool] = mapped_column(default=True) + webhook_endpoints: Mapped[list[WebhookEndpoint]] = relationship(back_populates="app", cascade="delete") +``` -def test_rejects_tampered_body(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body, headers = signed_request(payload()) - with TestClient(app) as client: - response = client.post("/fastid-webhooks", content=body + b" ", headers=headers) - assert response.status_code == 401 +Apply these symbol mappings everywhere listed in **Files**: + +```text +Webhook -> WebhookEndpoint +WebhookVersion -> WebhookEndpointVersion +WebhookRepository -> WebhookEndpointRepository +WebhookTypeSpecification -> WebhookEndpointTypeSpecification +WebhookDeliveryWebhookIDSpecification -> WebhookDeliveryEndpointIDSpecification +delivery.webhook_id -> delivery.endpoint_id +delivery.webhook -> delivery.endpoint +App.webhooks -> App.webhook_endpoints ``` -Add separate tests for missing headers, a non-integer timestamp, a timestamp older than 300 seconds, malformed JSON, -an invalid envelope, and a header/payload event-ID mismatch. Assert `400` for each. +Keep the payload schema `fastid.webhooks.schemas.Webhook` unchanged; it is not the persistence model. -- [ ] **Step 3: Run quick-start tests and verify RED** +- [ ] **Step 4: Update use-case and worker data flow** -```powershell -rtk pytest tests/examples/test_webhook_quickstart.py -q -``` +Use endpoint terminology at the call sites: -Expected: collection fails because `examples.webhook_quickstart` does not exist. +```python +endpoint_page = await self.uow.webhook_endpoints.get_many(WebhookEndpointTypeSpecification(dto.type)) +for endpoint in endpoint_page.items: + delivery = WebhookDelivery( + id=get_webhook_id(), + endpoint_id=endpoint.id, + event_id=event_id, + event_type=dto.type, + payload=payload, + status=WebhookDeliveryStatus.pending, + next_attempt_at=now, + ) +``` -- [ ] **Step 4: Implement the standalone quick-start application** +In the worker, use `joinedload(WebhookDelivery.endpoint)`, read `delivery.endpoint.url` and +`delivery.endpoint.secret`, load `uow.webhook_endpoints.get(delivery.endpoint_id)`, and cancel sibling deliveries with: -Create `examples/webhook_quickstart.py` with: +```python +WebhookDelivery.endpoint_id == endpoint.id +``` -- a FastAPI lifespan that reads and validates `FASTID_WEBHOOK_SECRET` into `app.state.webhook_secret`; -- `_secret_bytes()` supporting prefixed and plain secrets; -- `verify_signature()` that ignores unknown versions and constant-time compares every `v1` candidate; -- `POST /fastid-webhooks`, which reads the raw body, validates headers and timestamp, verifies before parsing JSON, - validates `event.event_id`, `event.event_type`, and `data`, requires ID equality, logs the generic receipt, and returns - `Response(status_code=204)`; -- `HTTPException(400, ...)` for malformed requests and `HTTPException(401, "Invalid webhook signature")` for a - non-matching signature; -- a production-idempotency warning immediately above the log call; -- `if __name__ == "__main__": uvicorn.run("examples.webhook_quickstart:app", host="127.0.0.1", port=8000)`. +- [ ] **Step 5: Add the lossless Alembic rename migration** -Use this exact signature input: +Create `migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py` with: ```python -signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) -expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() -return any( - version == "v1" and hmac.compare_digest(value, expected) - for signature in signatures.split() - if "," in signature - for version, value in (signature.split(",", 1),) +"""rename webhooks to endpoints + +Revision ID: 5b2c8d7e9f10 +Revises: a4e1d2c3b5f6 +Create Date: 2026-07-20 12:00:00 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "5b2c8d7e9f10" +down_revision: str | None = "a4e1d2c3b5f6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +UPGRADE_RENAMES = ( + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_pkey TO webhook_endpoints_pkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_app_id_fkey TO webhook_endpoints_app_id_fkey", + "ALTER INDEX webhooks_app_id_idx RENAME TO webhook_endpoints_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhooks_version_pkey TO webhook_endpoints_version_pkey", + "ALTER INDEX webhooks_version_app_id_idx RENAME TO webhook_endpoints_version_app_id_idx", + "ALTER INDEX webhooks_version_end_transaction_id_idx RENAME TO webhook_endpoints_version_end_transaction_id_idx", + "ALTER INDEX webhooks_version_operation_type_idx RENAME TO webhook_endpoints_version_operation_type_idx", + "ALTER INDEX webhooks_version_transaction_id_idx RENAME TO webhook_endpoints_version_transaction_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_webhook_id_fkey TO webhook_deliveries_endpoint_id_fkey", + "ALTER INDEX webhook_deliveries_webhook_id_idx RENAME TO webhook_deliveries_endpoint_id_idx", ) + +DOWNGRADE_RENAMES = ( + "ALTER INDEX webhook_deliveries_endpoint_id_idx RENAME TO webhook_deliveries_webhook_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_endpoint_id_fkey TO webhook_deliveries_webhook_id_fkey", + "ALTER INDEX webhook_endpoints_version_transaction_id_idx RENAME TO webhooks_version_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_operation_type_idx RENAME TO webhooks_version_operation_type_idx", + "ALTER INDEX webhook_endpoints_version_end_transaction_id_idx RENAME TO webhooks_version_end_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_app_id_idx RENAME TO webhooks_version_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhook_endpoints_version_pkey TO webhooks_version_pkey", + "ALTER INDEX webhook_endpoints_app_id_idx RENAME TO webhooks_app_id_idx", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_app_id_fkey TO webhooks_app_id_fkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_pkey TO webhooks_pkey", +) + + +def upgrade() -> None: + op.rename_table("webhooks", "webhook_endpoints") + op.rename_table("webhooks_version", "webhook_endpoints_version") + op.alter_column("webhook_deliveries", "webhook_id", new_column_name="endpoint_id") + for statement in UPGRADE_RENAMES: + op.execute(statement) + + +def downgrade() -> None: + for statement in DOWNGRADE_RENAMES: + op.execute(statement) + op.alter_column("webhook_deliveries", "endpoint_id", new_column_name="webhook_id") + op.rename_table("webhook_endpoints_version", "webhooks_version") + op.rename_table("webhook_endpoints", "webhooks") ``` -- [ ] **Step 5: Verify GREEN and static correctness** +This migration must rename in place; it must not recreate or copy tables. + +- [ ] **Step 6: Verify the endpoint rename** + +Run: ```powershell -rtk pytest tests/examples/test_webhook_quickstart.py -q -rtk ruff check examples/webhook_quickstart.py tests/examples -rtk mypy examples/webhook_quickstart.py tests/examples +rtk proxy poetry run pytest tests/webhooks/test_endpoint_models.py tests/api/webhooks tests/api/auth tests/api/profile -q +rtk proxy poetry run ruff check fastid tests/webhooks/test_endpoint_models.py tests/api +rtk proxy poetry run mypy fastid tests/webhooks/test_endpoint_models.py +rtk rg -n "WebhookDelivery\.webhook_id|WebhookDelivery\.webhook\b|from fastid\.webhooks\.models import Webhook\b|class Webhook\(" fastid tests ``` -Expected: all checks pass. +Expected: tests and static checks pass; the final search reports only the intentionally unchanged payload schema if it +matches at all. -- [ ] **Step 6: Commit** +- [ ] **Step 7: Commit the endpoint rename** ```powershell -rtk git add examples/webhook_quickstart.py tests/examples -rtk git commit -m "docs: add webhook quick-start example" +rtk proxy git add fastid migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py tests +rtk proxy git commit -m "refactor: name webhook endpoints explicitly" ``` --- -### Task 2: Advanced in-memory receiver +### Task 2: Make Webhook ID the signed delivery identifier **Files:** -- Create: `tests/examples/test_webhook_advanced.py` -- Create: `examples/webhook_advanced.py` +- Modify: `tests/api/webhooks/test_worker_delivery.py` +- Modify: `tests/security/test_webhooks.py` +- Modify: `tests/mocks.py` +- Modify: `fastid/security/webhooks.py` +- Modify: `fastid/webhooks/worker.py` **Interfaces:** -- Produces: `IdempotencyStore(Protocol)` with async `claim(event_id) -> bool`, `complete(event_id) -> None`, and - `release(event_id) -> None`. -- Produces: `InMemoryIdempotencyStore` implementing that protocol with one `asyncio.Lock`. -- Produces: `create_app(store: IdempotencyStore | None = None, processor: EventProcessor = process_event) -> FastAPI`. -- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401 | 413 | 500`. +- Produces: `generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str`. +- Produces: `generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]`. +- Guarantees: `headers["webhook-id"] == str(WebhookDelivery.id)` and is independent of `event.event_id`. -- [ ] **Step 1: Write failing store and HTTP tests** +- [ ] **Step 1: Write the failing sender-semantics assertion** -Create `tests/examples/test_webhook_advanced.py`. Test the wished-for API directly: +In the successful worker delivery test, add: ```python -async def test_concurrent_claim_has_one_winner() -> None: - store = InMemoryIdempotencyStore() - results = await asyncio.gather(*(store.claim("event-1") for _ in range(20))) - assert results.count(True) == 1 - assert results.count(False) == 19 +headers = sender.requests[0].headers +assert headers["webhook-id"] == str(delivery.id) +assert headers["webhook-id"] != str(delivery.event_id) +``` +- [ ] **Step 2: Run the focused test and verify RED** -async def test_release_allows_retry() -> None: - store = InMemoryIdempotencyStore() - assert await store.claim("event-1") - await store.release("event-1") - assert await store.claim("event-1") +```powershell +rtk proxy poetry run pytest tests/api/webhooks/test_worker_delivery.py::test_worker_records_delivery_outcome -q ``` -Use a recording async processor injected through `create_app()` and assert a repeated signed delivery returns `204` -twice but records one call. Add tests for declared and actual bodies over `MAX_BODY_BYTES`, invalid Pydantic envelope data, -ID mismatch, and a raising processor that returns `500` and can be retried successfully. +Expected: the first assertion fails while the worker still signs the Event ID. -- [ ] **Step 2: Run advanced tests and verify RED** +- [ ] **Step 3: Use `webhook_id` consistently in signing code** + +Implement: + +```python +def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + digest = hmac.new(_secret_bytes(secret_key), signed, hashlib.sha256).digest() + return f"v1,{base64.b64encode(digest).decode()}" + + +def generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]: + return { + STANDARD_ID_HEADER: webhook_id, + STANDARD_TIMESTAMP_HEADER: str(timestamp), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, webhook_id, timestamp, secret_key), + "Content-Type": "application/json", + "User-Agent": webhook_settings.user_agent, + } +``` + +`verify_standard_headers()` must also name the parsed header value `webhook_id`. The worker passes +`str(delivery.id)` to `generate_delivery_headers()`. + +- [ ] **Step 4: Verify GREEN and commit** ```powershell -rtk pytest tests/examples/test_webhook_advanced.py -q +rtk proxy poetry run pytest tests/api/webhooks/test_worker_delivery.py tests/security/test_webhooks.py -q +rtk proxy poetry run ruff check fastid/security/webhooks.py fastid/webhooks/worker.py tests/security tests/api/webhooks/test_worker_delivery.py +rtk proxy poetry run mypy fastid/security/webhooks.py fastid/webhooks/worker.py +rtk proxy git add fastid/security/webhooks.py fastid/webhooks/worker.py tests/security/test_webhooks.py tests/api/webhooks/test_worker_delivery.py tests/mocks.py +rtk proxy git commit -m "refactor: separate webhook and event identifiers" ``` -Expected: collection fails because `examples.webhook_advanced` does not exist. +--- + +### Task 3: Align the quick-start receiver with signature-only verification -- [ ] **Step 3: Implement the advanced standalone application** +**Files:** +- Modify: `tests/examples/webhook_helpers.py` +- Modify: `tests/examples/test_webhook_quickstart.py` +- Modify: `examples/webhook_quickstart.py` + +**Interfaces:** +- Produces: `app: FastAPI`. +- Produces: `verify_signature(body: bytes, webhook_id: str, timestamp: str, signatures: str, secret: str) -> bool`. +- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401`. -Create `examples/webhook_advanced.py` with: +- [ ] **Step 1: Write the quick-start contract test** + +Replace freshness and ID-equality tests with: ```python -MAX_BODY_BYTES = 1024 * 1024 -ClaimStatus = Literal["processing", "completed"] -EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] +def test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = compact_json(payload()) + headers = headers_for(body, "opaque-webhook-id", timestamp="opaque-timestamp") + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + assert response.status_code == 204 +``` +Ensure `signed_request()` generates a Webhook ID independently of the payload Event ID. -class EventMetadata(BaseModel): - event_id: UUID - event_type: str = Field(min_length=1) - timestamp: int +- [ ] **Step 2: Run the quick-start tests and verify RED** +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_quickstart.py -q +``` -class WebhookEnvelope(BaseModel): - event: EventMetadata - data: dict[str, Any] +Expected: the opaque timestamp or independent Webhook ID is rejected by the old implementation. +- [ ] **Step 3: Implement signature-only behavior and concise naming** -class IdempotencyStore(Protocol): - async def claim(self, event_id: str) -> bool: ... - async def complete(self, event_id: str) -> None: ... - async def release(self, event_id: str) -> None: ... +Read required headers as strings and authenticate only this input: + +```python +webhook_id = _required_header(request, "webhook-id") +timestamp = _required_header(request, "webhook-timestamp") +signatures = _required_header(request, "webhook-signature") +if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + raise HTTPException(status_code=401, detail="Invalid webhook signature") ``` -`InMemoryIdempotencyStore` must guard a `dict[str, ClaimStatus]` with one lock. `claim()` inserts `processing` only when -absent, `complete()` changes an existing claim to `completed`, and `release()` removes only a `processing` claim. +Do not parse the timestamp, validate the Webhook ID format, compare it with `event.event_id`, or add an idempotency +store. Rename every `message_id` local and parameter to `webhook_id`. -Repeat the complete verification implementation from the quick-start file. `create_app()` must enforce both declared -and actual body length, validate with `WebhookEnvelope.model_validate_json(body)`, compare the normalized UUID to -`webhook-id`, claim before calling the processor, acknowledge duplicates, complete success, and release then raise -`HTTPException(500, "Webhook processing failed")` on processing/storage errors. The default processor logs the event. +- [ ] **Step 4: Verify GREEN and commit** -Add the same direct `uvicorn.run()` entry point on port 8000. +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_quickstart.py -q +rtk proxy poetry run ruff check examples/webhook_quickstart.py tests/examples +rtk proxy poetry run mypy examples/webhook_quickstart.py tests/examples +rtk proxy git add examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py tests/examples/webhook_helpers.py +rtk proxy git commit -m "docs: simplify webhook quick-start verification" +``` -- [ ] **Step 4: Verify GREEN and static correctness** +--- + +### Task 4: Align advanced in-memory idempotency with Webhook ID + +**Files:** +- Modify: `tests/examples/test_webhook_advanced.py` +- Modify: `examples/webhook_advanced.py` + +**Interfaces:** +- Produces: `IdempotencyStore` methods `claim(webhook_id)`, `complete(webhook_id)`, and `release(webhook_id)`. +- Produces: `InMemoryIdempotencyStore` implementing that protocol with one `asyncio.Lock`. +- Produces: `create_app(store: IdempotencyStore | None = None, processor: EventProcessor = process_event) -> FastAPI`. + +- [ ] **Step 1: Write independent-ID and replay tests** + +Use an independently generated `webhook-id`, assert a repeated signed request is processed once, and assert stale +timestamps still return `400`: + +```python +assert headers["webhook-id"] != value["event"]["event_id"] +assert first.status_code == 204 +assert duplicate.status_code == 204 +assert processed_event_ids == [value["event"]["event_id"]] +``` + +Keep concurrent claim and release/retry coverage, but name all store keys `webhook_id`. + +- [ ] **Step 2: Run the advanced tests and verify RED** ```powershell -rtk pytest tests/examples/test_webhook_advanced.py -q -rtk ruff check examples/webhook_advanced.py tests/examples/test_webhook_advanced.py -rtk mypy examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy poetry run pytest tests/examples/test_webhook_advanced.py -q +``` + +Expected: the old header/payload equality check or old API terminology fails. + +- [ ] **Step 3: Implement Webhook-ID idempotency** + +Use this protocol and data flow: + +```python +class IdempotencyStore(Protocol): + async def claim(self, webhook_id: str) -> bool: ... + async def complete(self, webhook_id: str) -> None: ... + async def release(self, webhook_id: str) -> None: ... ``` -Expected: all checks pass. +`_validated_event()` returns `(event, webhook_id)` without comparing identifiers. `_receive_webhook()` atomically +claims `webhook_id`, acknowledges a duplicate with `204`, completes it after processing, and releases it on failure. +Retain integer timestamp parsing, 300-second freshness, signature verification, Pydantic validation, and body limits. +Rename log fields from `message_id` to `webhook_id`. -- [ ] **Step 5: Commit** +- [ ] **Step 4: Verify GREEN and commit** ```powershell -rtk git add examples/webhook_advanced.py tests/examples/test_webhook_advanced.py -rtk git commit -m "docs: add advanced webhook receiver example" +rtk proxy poetry run pytest tests/examples/test_webhook_advanced.py -q +rtk proxy poetry run ruff check examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy poetry run mypy examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy git add examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy git commit -m "docs: use webhook IDs for receiver idempotency" ``` --- -### Task 3: SQLAlchemy-backed receiver +### Task 5: Add the standalone SQLAlchemy receiver **Files:** -- Create: `tests/examples/test_webhook_sqlalchemy.py` - Create: `examples/webhook_sqlalchemy.py` +- Modify: `tests/examples/test_webhook_sqlalchemy.py` **Interfaces:** -- Produces: `WebhookReceipt` with unique string `event_id`, `status`, `created_at`, and `updated_at` columns. +- Produces: `WebhookReceipt` with unique string `webhook_id`, `status`, `created_at`, and `updated_at` columns. - Produces: `SQLAlchemyIdempotencyStore(engine: Engine)` with synchronous `claim`, `complete`, and `release` methods. - Produces: `create_app(database_url: str | None = None, processor: EventProcessor = process_event) -> FastAPI`. - Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401 | 413 | 500`. -- [ ] **Step 1: Write failing SQLAlchemy store and HTTP tests** +- [ ] **Step 1: Finish the failing SQLAlchemy store tests** -Create `tests/examples/test_webhook_sqlalchemy.py`. Use `tmp_path / "webhooks.sqlite3"` and two engines/store instances. -Assert: +Use `tmp_path / "webhooks.sqlite3"` and two store instances: ```python def test_claim_persists_across_store_instances(tmp_path: Path) -> None: @@ -295,112 +498,147 @@ def test_claim_persists_across_store_instances(tmp_path: Path) -> None: first = SQLAlchemyIdempotencyStore(create_store_engine(database_url)) second = SQLAlchemyIdempotencyStore(create_store_engine(database_url)) first.create_schema() - assert first.claim("event-1") - first.complete("event-1") - assert not second.claim("event-1") + assert first.claim("webhook-1") + first.complete("webhook-1") + assert not second.claim("webhook-1") ``` -Add a concurrent claim test through `asyncio.to_thread`, a release/reclaim test, valid/duplicate HTTP delivery with a -recording processor, a processing-failure retry test, and one parameterized test proving the SQLAlchemy app returns the -same `400`, `401`, and `413` classes as the advanced app. +Add concurrent claim, release/reclaim, valid/duplicate HTTP delivery, processing-failure retry, and parameterized +`400`/`401`/`413` coverage. Use independent Webhook IDs and Event IDs throughout. -- [ ] **Step 2: Run SQLAlchemy tests and verify RED** +- [ ] **Step 2: Run the SQLAlchemy tests and verify RED** ```powershell -rtk pytest tests/examples/test_webhook_sqlalchemy.py -q +rtk proxy poetry run pytest tests/examples/test_webhook_sqlalchemy.py -q ``` Expected: collection fails because `examples.webhook_sqlalchemy` does not exist. -- [ ] **Step 3: Implement the standalone SQLAlchemy application** - -Create `examples/webhook_sqlalchemy.py` with a local `DeclarativeBase`, `WebhookReceipt`, and -`SQLAlchemyIdempotencyStore`. `create_store_engine(database_url)` must set `check_same_thread=False` only for SQLite. +- [ ] **Step 3: Implement the durable claim store** -Use `Session.begin()` transactions. `claim()` inserts status `processing`, flushes, and returns `False` on -`IntegrityError`. `complete()` updates the matching row to `completed`. `release()` deletes only a `processing` row. -`create_schema()` calls `Base.metadata.create_all(engine)`. +Create a local declarative model: -Repeat the advanced file's signature, size, and envelope validation so this file remains standalone. FastAPI lifespan -must require `FASTID_WEBHOOK_SECRET`, select the explicit argument or `WEBHOOK_DATABASE_URL` or -`sqlite:///webhook-events.sqlite3`, create the engine/store/schema, and dispose the engine at shutdown. +```python +def utc_now() -> datetime: + return datetime.now(UTC) -Run every synchronous store call through `starlette.concurrency.run_in_threadpool`. A duplicate returns `204`; success -completes; failure releases and returns `500`. Add the direct uvicorn entry point. -- [ ] **Step 4: Verify GREEN and static correctness** +class WebhookReceipt(Base): + __tablename__ = "webhook_receipts" -```powershell -rtk pytest tests/examples/test_webhook_sqlalchemy.py -q -rtk ruff check examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py -rtk mypy examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py + webhook_id: Mapped[str] = mapped_column(primary_key=True) + status: Mapped[str] + created_at: Mapped[datetime] = mapped_column(default=utc_now) + updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) ``` -Expected: all checks pass. +`claim(webhook_id)` inserts `processing` in `Session.begin()` and returns `False` on `IntegrityError`. +`complete(webhook_id)` updates the row to `completed`. `release(webhook_id)` deletes only a `processing` row. +`create_schema()` calls `Base.metadata.create_all(engine)`. `create_store_engine()` sets +`connect_args={"check_same_thread": False}` only for SQLite. + +- [ ] **Step 4: Implement the standalone FastAPI receiver** + +Repeat the advanced example's verifier, envelope, freshness, size, and error behavior without importing another example +or `fastid.*`. Lifespan selects the explicit argument, then `WEBHOOK_DATABASE_URL`, then +`sqlite:///webhook-events.sqlite3`; it creates the schema and disposes the engine at shutdown. Run every synchronous +store call through `starlette.concurrency.run_in_threadpool`: + +```python +claimed = await run_in_threadpool(store.claim, webhook_id) +if not claimed: + return Response(status_code=204) +try: + await processor(event) + await run_in_threadpool(store.complete, webhook_id) +except Exception as exc: + await run_in_threadpool(store.release, webhook_id) + raise HTTPException(status_code=500, detail="Webhook processing failed") from exc +return Response(status_code=204) +``` -- [ ] **Step 5: Commit** +- [ ] **Step 5: Verify GREEN and commit** ```powershell -rtk git add examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py -rtk git commit -m "docs: add SQLAlchemy webhook receiver example" +rtk proxy poetry run pytest tests/examples/test_webhook_sqlalchemy.py -q +rtk proxy poetry run ruff check examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy poetry run mypy examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy git add examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy git commit -m "docs: add SQLAlchemy webhook receiver example" ``` --- -### Task 4: Tutorial links and final verification +### Task 6: Update the tutorial and verify the complete feature **Files:** - Modify: `docs/docs/tutorial/webhooks.md` +- Modify: `docs/superpowers/plans/2026-07-20-standard-webhook-headers.md` +- Modify: `docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md` **Interfaces:** -- Consumes: all three runnable example paths. -- Produces: one discoverable progression from quick-start to production-oriented references. +- Consumes: the endpoint terminology and all three runnable receiver paths. +- Produces: one discoverable progression from signature-only quick-start to production-oriented references. + +- [ ] **Step 1: Remove obsolete terminology** + +Use `Webhook ID` in prose and `webhook_id` in code. Replace references to the configured `Webhook` entity with +`Webhook Endpoint` where the distinction matters. Verify with: + +```powershell +rtk rg -ni "message[ _-]?id|WebhookDelivery\.webhook_id|WebhookDelivery\.webhook\b|stable event id|header/payload.*match" docs examples fastid tests +``` + +Expected: no obsolete terminology remains except historical migration descriptions that explicitly name an old schema. -- [ ] **Step 1: Add example links to the tutorial** +- [ ] **Step 2: Add the receiver progression to the tutorial** -After the existing signature-verification fragment, add: +Document: ```markdown Runnable receiver examples are available for different integration stages: -- [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies and logs an event with minimal setup. -- [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds validation and an in-memory idempotency boundary. -- [`webhook_sqlalchemy.py`](../../../examples/webhook_sqlalchemy.py) persists atomic event claims with SQLAlchemy. +- [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies a signature and logs the event. +- [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds freshness checks, validation, limits, and an + in-memory Webhook-ID idempotency boundary. +- [`webhook_sqlalchemy.py`](../../../examples/webhook_sqlalchemy.py) persists atomic Webhook-ID claims with SQLAlchemy. -The in-memory example is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared, -persistent idempotency store before applying non-idempotent production side effects. +The quick-start authenticates the request but does not provide replay protection or idempotency. The in-memory example +is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared persistent store before +applying non-idempotent production side effects. ``` -- [ ] **Step 2: Run focused example verification** +- [ ] **Step 3: Run focused verification** ```powershell -rtk pytest tests/examples -q -rtk ruff check examples tests/examples -rtk mypy examples tests/examples -rtk codespell examples tests/examples docs/docs/tutorial/webhooks.md +rtk proxy poetry run pytest tests/security/test_webhooks.py tests/api/webhooks tests/webhooks tests/examples -q +rtk proxy poetry run ruff check fastid examples tests +rtk proxy poetry run mypy fastid examples tests/examples +rtk proxy poetry run codespell examples tests/examples docs/docs/tutorial/webhooks.md docs/glossary.md ``` -Expected: all checks pass. +Expected: all focused tests and static checks pass. -- [ ] **Step 3: Run the complete project verification** +- [ ] **Step 4: Run complete project verification** ```powershell -rtk pytest -q -rtk ruff check . -rtk mypy fastid examples tests/examples -rtk git diff --check -rtk git status --short +rtk proxy poetry run pytest -q +rtk proxy poetry run ruff check . +rtk proxy poetry run mypy fastid examples tests/examples +rtk proxy git diff --check +rtk proxy git status --short ``` -Expected: all tests and static checks pass; only the tutorial is pending before the final commit. +Expected: all tests and static checks pass; only the planned final documentation changes remain uncommitted. -- [ ] **Step 4: Commit documentation and confirm history** +- [ ] **Step 5: Commit documentation and confirm history** ```powershell -rtk git add docs/docs/tutorial/webhooks.md -rtk git commit -m "docs: link webhook receiver examples" -rtk git status --short -rtk git log -5 --oneline +rtk proxy git add docs examples tests fastid migrations +rtk proxy git commit -m "docs: finish webhook receiver examples" +rtk proxy git status --short +rtk proxy git log -8 --oneline ``` -Expected: the worktree is clean and the four example commits are at the top of `feat/enhance-webhooks`. +Expected: the worktree is clean and the endpoint, identifier, receiver, and tutorial commits are at the top of +`feat/enhance-webhooks`. From 75535f31f98d9971f7c399a0673ab4b8cfba78fd Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:56:01 +0300 Subject: [PATCH 17/28] refactor: name webhook endpoints explicitly --- fastid/admin/factory.py | 8 +-- fastid/admin/views/settings.py | 36 +++++------ fastid/admin/views/versioning.py | 8 +-- fastid/apps/models.py | 4 +- fastid/database/models.py | 8 +-- fastid/database/uow.py | 6 +- fastid/database/versioning.py | 4 +- fastid/webhooks/models.py | 12 ++-- fastid/webhooks/repositories.py | 18 +++--- fastid/webhooks/use_cases.py | 8 +-- fastid/webhooks/worker.py | 16 ++--- ...2c8d7e9f10_rename_webhooks_to_endpoints.py | 59 +++++++++++++++++++ .../api/auth/test_authorize_password_grant.py | 8 +-- tests/api/auth/test_register.py | 8 +-- tests/api/conftest.py | 16 ++--- tests/api/profile/test_delete_user.py | 8 +-- tests/api/profile/test_update_user_email.py | 8 +-- .../api/profile/test_update_user_password.py | 8 +-- tests/api/profile/test_update_user_profile.py | 12 ++-- tests/api/webhooks/test_worker_delivery.py | 14 ++--- tests/api/webhooks/test_wrong_url.py | 8 +-- tests/utils/webhooks.py | 8 +-- tests/webhooks/test_endpoint_models.py | 11 ++++ 23 files changed, 185 insertions(+), 111 deletions(-) create mode 100644 migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py create mode 100644 tests/webhooks/test_endpoint_models.py diff --git a/fastid/admin/factory.py b/fastid/admin/factory.py index 0f03c54..f56cba5 100644 --- a/fastid/admin/factory.py +++ b/fastid/admin/factory.py @@ -12,9 +12,9 @@ EmailTemplateAdmin, OAuthProviderAdmin, TelegramTemplateAdmin, - WebhookAdmin, WebhookAttemptAdmin, WebhookDeliveryAdmin, + WebhookEndpointAdmin, ) from fastid.admin.views.versioning import ( AppVersionAdmin, @@ -24,7 +24,7 @@ TelegramTemplateVersionAdmin, TransactionAdmin, UserVersionAdmin, - WebhookVersionAdmin, + WebhookEndpointVersionAdmin, ) from fastid.core.base import AppFactory @@ -55,7 +55,7 @@ def create(self) -> FastAPI: # Settings admin.add_view(AppAdmin) admin.add_view(OAuthProviderAdmin) - admin.add_view(WebhookAdmin) + admin.add_view(WebhookEndpointAdmin) admin.add_view(WebhookDeliveryAdmin) admin.add_view(WebhookAttemptAdmin) admin.add_view(EmailTemplateAdmin) @@ -66,7 +66,7 @@ def create(self) -> FastAPI: admin.add_view(OAuthAccountVersionAdmin) admin.add_view(AppVersionAdmin) admin.add_view(OAuthProviderVersionAdmin) - admin.add_view(WebhookVersionAdmin) + admin.add_view(WebhookEndpointVersionAdmin) admin.add_view(EmailTemplateVersionAdmin) admin.add_view(TelegramTemplateVersionAdmin) return app diff --git a/fastid/admin/views/settings.py b/fastid/admin/views/settings.py index a2fe175..becf4c4 100644 --- a/fastid/admin/views/settings.py +++ b/fastid/admin/views/settings.py @@ -7,9 +7,9 @@ EmailTemplate, OAuthProvider, TelegramTemplate, - Webhook, WebhookAttempt, WebhookDelivery, + WebhookEndpoint, ) @@ -97,26 +97,26 @@ class TelegramTemplateAdmin(BaseView, model=TelegramTemplate): ] -class WebhookAdmin(BaseView, model=Webhook): - name = "Webhook" - name_plural = "Webhooks" +class WebhookEndpointAdmin(BaseView, model=WebhookEndpoint): + name = "Webhook Endpoint" + name_plural = "Webhook Endpoints" icon = "fa-solid fa-globe" category = "Settings" column_list = [ - Webhook.id, - Webhook.app, - Webhook.url, - Webhook.is_active, - Webhook.disabled_reason, - Webhook.created_at, - Webhook.updated_at, + WebhookEndpoint.id, + WebhookEndpoint.app, + WebhookEndpoint.url, + WebhookEndpoint.is_active, + WebhookEndpoint.disabled_reason, + WebhookEndpoint.created_at, + WebhookEndpoint.updated_at, ] column_filters = [ - OperationColumnFilter(Webhook.app_id), - AllUniqueStringValuesFilter(Webhook.type), - OperationColumnFilter(Webhook.url), - BooleanFilter(Webhook.is_active), + OperationColumnFilter(WebhookEndpoint.app_id), + AllUniqueStringValuesFilter(WebhookEndpoint.type), + OperationColumnFilter(WebhookEndpoint.url), + BooleanFilter(WebhookEndpoint.is_active), ] @@ -133,8 +133,8 @@ class WebhookDeliveryAdmin(BaseView, model=WebhookDelivery): column_list = [ WebhookDelivery.id, WebhookDelivery.event_id, - WebhookDelivery.webhook, - "webhook.type", + WebhookDelivery.endpoint, + "endpoint.type", WebhookDelivery.status, WebhookDelivery.attempt_count, WebhookDelivery.status_code, @@ -144,7 +144,7 @@ class WebhookDeliveryAdmin(BaseView, model=WebhookDelivery): WebhookDelivery.created_at, ] column_filters = [ - OperationColumnFilter(WebhookDelivery.webhook_id), + OperationColumnFilter(WebhookDelivery.endpoint_id), OperationColumnFilter(WebhookDelivery.status_code), AllUniqueStringValuesFilter(WebhookDelivery.status), ] diff --git a/fastid/admin/views/versioning.py b/fastid/admin/views/versioning.py index 42f6e42..fa32282 100644 --- a/fastid/admin/views/versioning.py +++ b/fastid/admin/views/versioning.py @@ -10,7 +10,7 @@ TelegramTemplateVersion, Transaction, UserVersion, - WebhookVersion, + WebhookEndpointVersion, ) @@ -93,6 +93,6 @@ class TelegramTemplateVersionAdmin(BaseVersionView, model=TelegramTemplateVersio name_plural = "Telegram Template Versions" -class WebhookVersionAdmin(BaseVersionView, model=WebhookVersion): - name = "Webhook Version" - name_plural = "Webhook Versions" +class WebhookEndpointVersionAdmin(BaseVersionView, model=WebhookEndpointVersion): + name = "Webhook Endpoint Version" + name_plural = "Webhook Endpoint Versions" diff --git a/fastid/apps/models.py b/fastid/apps/models.py index 3fe668f..a20d1ad 100644 --- a/fastid/apps/models.py +++ b/fastid/apps/models.py @@ -5,7 +5,7 @@ from fastid.apps.exceptions import InvalidClientCredentialsError, InvalidRedirectURIError from fastid.database.base import VersionedEntity from fastid.database.utils import uuid_hex -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint class App(VersionedEntity): @@ -18,7 +18,7 @@ class App(VersionedEntity): redirect_uris: Mapped[str] = mapped_column(default="") is_active: Mapped[bool] = mapped_column(default=True) - webhooks: Mapped[list[Webhook]] = relationship(back_populates="app", cascade="delete") + webhook_endpoints: Mapped[list[WebhookEndpoint]] = relationship(back_populates="app", cascade="delete") def verify_redirect_uri(self, redirect_uri: str) -> None: if redirect_uri not in self.redirect_uris.split(";"): diff --git a/fastid/database/models.py b/fastid/database/models.py index c40919e..324e50c 100644 --- a/fastid/database/models.py +++ b/fastid/database/models.py @@ -6,7 +6,7 @@ from fastid.database.base import BaseOrm from fastid.notify.models import EmailTemplate, Notification, TelegramTemplate from fastid.oauth.models import OAuthAccount, OAuthProvider -from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDelivery +from fastid.webhooks.models import WebhookAttempt, WebhookDelivery, WebhookEndpoint configure_mappers() @@ -19,7 +19,7 @@ TelegramTemplateVersion, Transaction, UserVersion, - WebhookVersion, + WebhookEndpointVersion, ) __all__ = [ @@ -36,8 +36,8 @@ "EmailTemplateVersion", "TelegramTemplateVersion", "Transaction", - "WebhookVersion", - "Webhook", + "WebhookEndpointVersion", + "WebhookEndpoint", "WebhookAttempt", "WebhookDelivery", "OAuthAccountVersion", diff --git a/fastid/database/uow.py b/fastid/database/uow.py index 504604d..56fb371 100644 --- a/fastid/database/uow.py +++ b/fastid/database/uow.py @@ -9,7 +9,7 @@ from fastid.auth.repositories import UserRepository from fastid.notify.repositories import EmailTemplateRepository, NotificationRepository, TelegramTemplateRepository from fastid.oauth.repositories import OAuthAccountRepository, OAuthProviderRepository -from fastid.webhooks.repositories import WebhookAttemptRepository, WebhookDeliveryRepository, WebhookRepository +from fastid.webhooks.repositories import WebhookAttemptRepository, WebhookDeliveryRepository, WebhookEndpointRepository if TYPE_CHECKING: from types import TracebackType @@ -28,7 +28,7 @@ class SQLAlchemyUOW: email_templates: EmailTemplateRepository telegram_templates: TelegramTemplateRepository notifications: NotificationRepository - webhooks: WebhookRepository + webhook_endpoints: WebhookEndpointRepository webhook_deliveries: WebhookDeliveryRepository webhook_attempts: WebhookAttemptRepository @@ -47,7 +47,7 @@ async def begin(self) -> None: self.email_templates = EmailTemplateRepository(self.session) self.telegram_templates = TelegramTemplateRepository(self.session) self.notifications = NotificationRepository(self.session) - self.webhooks = WebhookRepository(self.session) + self.webhook_endpoints = WebhookEndpointRepository(self.session) self.webhook_deliveries = WebhookDeliveryRepository(self.session) self.webhook_attempts = WebhookAttemptRepository(self.session) diff --git a/fastid/database/versioning.py b/fastid/database/versioning.py index 5868dcb..c0a8bba 100644 --- a/fastid/database/versioning.py +++ b/fastid/database/versioning.py @@ -4,7 +4,7 @@ from fastid.auth.models import User from fastid.notify.models import EmailTemplate, TelegramTemplate from fastid.oauth.models import OAuthAccount, OAuthProvider -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint Transaction = transaction_class(User) @@ -14,4 +14,4 @@ OAuthProviderVersion = version_class(OAuthProvider) EmailTemplateVersion = version_class(EmailTemplate) TelegramTemplateVersion = version_class(TelegramTemplate) -WebhookVersion = version_class(Webhook) +WebhookEndpointVersion = version_class(WebhookEndpoint) diff --git a/fastid/webhooks/models.py b/fastid/webhooks/models.py index 18ceb62..2073f14 100644 --- a/fastid/webhooks/models.py +++ b/fastid/webhooks/models.py @@ -38,8 +38,8 @@ class WebhookDeliveryStatus(BaseEnum): cancelled = auto() -class Webhook(VersionedEntity): - __tablename__ = "webhooks" +class WebhookEndpoint(VersionedEntity): + __tablename__ = "webhook_endpoints" app_id: Mapped[UUID] = mapped_column(ForeignKey("apps.id"), index=True) type: Mapped[WebhookType] @@ -49,14 +49,14 @@ class Webhook(VersionedEntity): disabled_at: Mapped[datetime.datetime | None] disabled_reason: Mapped[str | None] - app: Mapped[App] = relationship(back_populates="webhooks") - deliveries: Mapped[list[WebhookDelivery]] = relationship(back_populates="webhook", cascade="delete") + app: Mapped[App] = relationship(back_populates="webhook_endpoints") + deliveries: Mapped[list[WebhookDelivery]] = relationship(back_populates="endpoint", cascade="delete") class WebhookDelivery(Entity): __tablename__ = "webhook_deliveries" - webhook_id: Mapped[UUID] = mapped_column(ForeignKey("webhooks.id"), index=True) + endpoint_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_endpoints.id"), index=True) event_id: Mapped[UUID] = mapped_column(index=True) event_type: Mapped[WebhookType] payload: Mapped[dict[str, Any]] @@ -70,7 +70,7 @@ class WebhookDelivery(Entity): response: Mapped[dict[str, Any] | None] error: Mapped[str | None] - webhook: Mapped[Webhook] = relationship(back_populates="deliveries") + endpoint: Mapped[WebhookEndpoint] = relationship(back_populates="deliveries") attempts: Mapped[list[WebhookAttempt]] = relationship(back_populates="delivery", cascade="delete") diff --git a/fastid/webhooks/repositories.py b/fastid/webhooks/repositories.py index ebc6144..bf7e030 100644 --- a/fastid/webhooks/repositories.py +++ b/fastid/webhooks/repositories.py @@ -3,19 +3,19 @@ from fastid.database.repository import SQLAlchemyRepository from fastid.database.specification import Specification from fastid.database.utils import UUIDv7 -from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDelivery, WebhookType +from fastid.webhooks.models import WebhookAttempt, WebhookDelivery, WebhookEndpoint, WebhookType -class WebhookRepository(SQLAlchemyRepository[Webhook]): - model_type = Webhook +class WebhookEndpointRepository(SQLAlchemyRepository[WebhookEndpoint]): + model_type = WebhookEndpoint -class WebhookTypeSpecification(Specification): +class WebhookEndpointTypeSpecification(Specification): def __init__(self, webhook_type: WebhookType) -> None: self.type = webhook_type def apply(self, stmt: Any) -> Any: - return stmt.where(Webhook.type == self.type, Webhook.is_active.is_(True)) + return stmt.where(WebhookEndpoint.type == self.type, WebhookEndpoint.is_active.is_(True)) class WebhookDeliveryRepository(SQLAlchemyRepository[WebhookDelivery]): @@ -26,9 +26,9 @@ class WebhookAttemptRepository(SQLAlchemyRepository[WebhookAttempt]): model_type = WebhookAttempt -class WebhookDeliveryWebhookIDSpecification(Specification): - def __init__(self, webhook_id: UUIDv7) -> None: - self.webhook_id = webhook_id +class WebhookDeliveryEndpointIDSpecification(Specification): + def __init__(self, endpoint_id: UUIDv7) -> None: + self.endpoint_id = endpoint_id def apply(self, stmt: Any) -> Any: - return stmt.where(WebhookDelivery.webhook_id == self.webhook_id) + return stmt.where(WebhookDelivery.endpoint_id == self.endpoint_id) diff --git a/fastid/webhooks/use_cases.py b/fastid/webhooks/use_cases.py index 2a978e7..523cb8e 100644 --- a/fastid/webhooks/use_cases.py +++ b/fastid/webhooks/use_cases.py @@ -3,7 +3,7 @@ from fastid.database.utils import naive_utc from fastid.security.webhooks import get_event_id, get_timestamp, get_webhook_id from fastid.webhooks.models import WebhookDelivery, WebhookDeliveryStatus -from fastid.webhooks.repositories import WebhookTypeSpecification +from fastid.webhooks.repositories import WebhookEndpointTypeSpecification from fastid.webhooks.schemas import Event, SendWebhookRequest, WebhookPayload @@ -12,16 +12,16 @@ def __init__(self, uow: UOWDep) -> None: self.uow = uow async def enqueue(self, dto: SendWebhookRequest) -> None: - webhook_page = await self.uow.webhooks.get_many(WebhookTypeSpecification(dto.type)) + endpoint_page = await self.uow.webhook_endpoints.get_many(WebhookEndpointTypeSpecification(dto.type)) event_id = get_event_id() event_timestamp = get_timestamp() event_dto = Event(event_type=dto.type, event_id=event_id, timestamp=event_timestamp) payload = WebhookPayload(event=event_dto, data=dto.payload).model_dump(mode="json") now = naive_utc() - for webhook in webhook_page.items: + for endpoint in endpoint_page.items: delivery = WebhookDelivery( id=get_webhook_id(), - webhook_id=webhook.id, + endpoint_id=endpoint.id, event_id=event_id, event_type=dto.type, payload=payload, diff --git a/fastid/webhooks/worker.py b/fastid/webhooks/worker.py index 3f56b9d..c5da90b 100644 --- a/fastid/webhooks/worker.py +++ b/fastid/webhooks/worker.py @@ -16,7 +16,7 @@ from fastid.security.webhooks import generate_delivery_headers, get_timestamp, serialize_payload from fastid.webhooks.config import webhook_settings from fastid.webhooks.metrics import ATTEMPT_DURATION, ATTEMPTS, DISABLED_ENDPOINTS, DUE_DELIVERIES -from fastid.webhooks.models import Webhook, WebhookAttempt, WebhookDelivery, WebhookDeliveryStatus +from fastid.webhooks.models import WebhookAttempt, WebhookDelivery, WebhookDeliveryStatus, WebhookEndpoint from fastid.webhooks.senders.dependencies import client from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender @@ -75,7 +75,7 @@ async def _claim(self) -> list[ClaimedDelivery]: async with uow: stmt = ( select(WebhookDelivery) - .options(joinedload(WebhookDelivery.webhook)) + .options(joinedload(WebhookDelivery.endpoint)) .where( WebhookDelivery.next_attempt_at <= now, or_( @@ -104,7 +104,7 @@ async def _claim(self) -> list[ClaimedDelivery]: DUE_DELIVERIES.set(due_count or 0) claimed: list[ClaimedDelivery] = [] for delivery in rows: - if not delivery.webhook.is_active: + if not delivery.endpoint.is_active: delivery.status = WebhookDeliveryStatus.cancelled delivery.completed_at = now delivery.error = "endpoint disabled" @@ -117,8 +117,8 @@ async def _claim(self) -> list[ClaimedDelivery]: event_id=delivery.event_id, event_type=str(delivery.event_type), payload=delivery.payload, - endpoint_url=delivery.webhook.url, - endpoint_secret=delivery.webhook.secret, + endpoint_url=delivery.endpoint.url, + endpoint_secret=delivery.endpoint.secret, ) ) return claimed @@ -152,7 +152,7 @@ async def _record( uow = get_uow_raw() async with uow: delivery = await uow.webhook_deliveries.get(delivery_id) - endpoint = await uow.webhooks.get(delivery.webhook_id) + endpoint = await uow.webhook_endpoints.get(delivery.endpoint_id) attempt_number = delivery.attempt_count + 1 stored_headers = { key: "[redacted]" if "signature" in key.lower() else value for key, value in headers.items() @@ -207,7 +207,7 @@ async def _record( @staticmethod async def _disable( - uow: SQLAlchemyUOW, endpoint: Webhook, delivery: WebhookDelivery, now: datetime, reason: str + uow: SQLAlchemyUOW, endpoint: WebhookEndpoint, delivery: WebhookDelivery, now: datetime, reason: str ) -> None: delivery.status = ( WebhookDeliveryStatus.exhausted if delivery.status_code != HTTP_GONE else WebhookDeliveryStatus.cancelled @@ -219,7 +219,7 @@ async def _disable( await uow.session.execute( update(WebhookDelivery) .where( - WebhookDelivery.webhook_id == endpoint.id, + WebhookDelivery.endpoint_id == endpoint.id, WebhookDelivery.id != delivery.id, WebhookDelivery.status.in_((WebhookDeliveryStatus.pending, WebhookDeliveryStatus.processing)), ) diff --git a/migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py b/migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py new file mode 100644 index 0000000..7aea5ba --- /dev/null +++ b/migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py @@ -0,0 +1,59 @@ +"""rename webhooks to endpoints + +Revision ID: 5b2c8d7e9f10 +Revises: a4e1d2c3b5f6 +Create Date: 2026-07-20 12:00:00 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "5b2c8d7e9f10" +down_revision: str | None = "a4e1d2c3b5f6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +UPGRADE_RENAMES = ( + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_pkey TO webhook_endpoints_pkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_app_id_fkey TO webhook_endpoints_app_id_fkey", + "ALTER INDEX webhooks_app_id_idx RENAME TO webhook_endpoints_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhooks_version_pkey TO webhook_endpoints_version_pkey", + "ALTER INDEX webhooks_version_app_id_idx RENAME TO webhook_endpoints_version_app_id_idx", + "ALTER INDEX webhooks_version_end_transaction_id_idx RENAME TO webhook_endpoints_version_end_transaction_id_idx", + "ALTER INDEX webhooks_version_operation_type_idx RENAME TO webhook_endpoints_version_operation_type_idx", + "ALTER INDEX webhooks_version_transaction_id_idx RENAME TO webhook_endpoints_version_transaction_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_webhook_id_fkey " + "TO webhook_deliveries_endpoint_id_fkey", + "ALTER INDEX webhook_deliveries_webhook_id_idx RENAME TO webhook_deliveries_endpoint_id_idx", +) + +DOWNGRADE_RENAMES = ( + "ALTER INDEX webhook_deliveries_endpoint_id_idx RENAME TO webhook_deliveries_webhook_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_endpoint_id_fkey " + "TO webhook_deliveries_webhook_id_fkey", + "ALTER INDEX webhook_endpoints_version_transaction_id_idx RENAME TO webhooks_version_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_operation_type_idx RENAME TO webhooks_version_operation_type_idx", + "ALTER INDEX webhook_endpoints_version_end_transaction_id_idx RENAME TO webhooks_version_end_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_app_id_idx RENAME TO webhooks_version_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhook_endpoints_version_pkey TO webhooks_version_pkey", + "ALTER INDEX webhook_endpoints_app_id_idx RENAME TO webhooks_app_id_idx", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_app_id_fkey TO webhooks_app_id_fkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_pkey TO webhooks_pkey", +) + + +def upgrade() -> None: + op.rename_table("webhooks", "webhook_endpoints") + op.rename_table("webhooks_version", "webhook_endpoints_version") + op.alter_column("webhook_deliveries", "webhook_id", new_column_name="endpoint_id") + for statement in UPGRADE_RENAMES: + op.execute(statement) + + +def downgrade() -> None: + for statement in DOWNGRADE_RENAMES: + op.execute(statement) + op.alter_column("webhook_deliveries", "endpoint_id", new_column_name="webhook_id") + op.rename_table("webhook_endpoints_version", "webhooks_version") + op.rename_table("webhook_endpoints", "webhooks") diff --git a/tests/api/auth/test_authorize_password_grant.py b/tests/api/auth/test_authorize_password_grant.py index 79bd820..160a86e 100644 --- a/tests/api/auth/test_authorize_password_grant.py +++ b/tests/api/auth/test_authorize_password_grant.py @@ -7,8 +7,8 @@ from fastid.core.config import core_settings from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests import mocks from tests.mocks import faker from tests.utils.auth import ( @@ -17,7 +17,7 @@ async def test_authorize_password_grant( - client: AsyncClient, user: UserDTO, webhook_login: Webhook, uow: SQLAlchemyUOW + client: AsyncClient, user: UserDTO, webhook_login: WebhookEndpoint, uow: SQLAlchemyUOW ) -> None: assert user.email is not None token = await authorize_password_grant(client, user.email, mocks.USER_CREATE.password) @@ -25,7 +25,7 @@ async def test_authorize_password_grant( assert token.refresh_token is not None try: - await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_login.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_login.id)) except NoResultFoundError: pytest.fail("No webhook delivery created") diff --git a/tests/api/auth/test_register.py b/tests/api/auth/test_register.py index bc0d8ce..4ff5236 100644 --- a/tests/api/auth/test_register.py +++ b/tests/api/auth/test_register.py @@ -5,12 +5,12 @@ from fastid.auth.schemas import UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import USER_CREATE -async def test_register(client: AsyncClient, webhook_registration: Webhook, uow: SQLAlchemyUOW) -> None: +async def test_register(client: AsyncClient, webhook_registration: WebhookEndpoint, uow: SQLAlchemyUOW) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) assert response.status_code == status.HTTP_201_CREATED user = UserDTO.model_validate_json(response.content) @@ -20,7 +20,7 @@ async def test_register(client: AsyncClient, webhook_registration: Webhook, uow: assert user.email == USER_CREATE.email try: - await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_registration.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_registration.id)) except NoResultFoundError: pytest.fail("No webhook delivery created") diff --git a/tests/api/conftest.py b/tests/api/conftest.py index ec3d949..e4323b8 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -14,7 +14,7 @@ from fastid.database.uow import SQLAlchemyUOW from fastid.notify.schemas import UserAction from fastid.security.crypto import generate_otp -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint from tests import mocks from tests.dependencies import get_test_cache, get_test_uow from tests.utils.auth import authorize_password_grant, register_user @@ -107,35 +107,35 @@ async def oauth_app(client: AsyncClient, user_su_token: TokenResponse) -> AppDTO @pytest.fixture -async def webhook_registration(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_registration(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_REGISTRATION_RECORD) @pytest.fixture -async def webhook_login(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_login(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_LOGIN_RECORD) @pytest.fixture -async def webhook_delete(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_delete(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_DELETE_RECORD) @pytest.fixture -async def webhook_profile_update(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_profile_update(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_UPDATE_PROFILE_RECORD) @pytest.fixture -async def webhook_change_password(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_change_password(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_CHANGE_PASSWORD) @pytest.fixture -async def webhook_change_email(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_change_email(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_CHANGE_EMAIL) @pytest.fixture -async def webhook_wrong_url(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_wrong_url(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_WRONG_URL) diff --git a/tests/api/profile/test_delete_user.py b/tests/api/profile/test_delete_user.py index 1e53ba6..6294782 100644 --- a/tests/api/profile/test_delete_user.py +++ b/tests/api/profile/test_delete_user.py @@ -6,8 +6,8 @@ from fastid.auth.schemas import TokenResponse, UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification async def test_delete_user( # noqa: PLR0913 @@ -15,7 +15,7 @@ async def test_delete_user( # noqa: PLR0913 user: UserDTO, user_token: TokenResponse, verify_token: str, - webhook_delete: Webhook, + webhook_delete: WebhookEndpoint, uow: SQLAlchemyUOW, ) -> None: client.cookies.set(vt_transport.name, verify_token) @@ -28,7 +28,7 @@ async def test_delete_user( # noqa: PLR0913 UserDTO.model_validate_json(response.content) try: - await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_delete.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_delete.id)) except NoResultFoundError: pytest.fail("No webhook delivery created") diff --git a/tests/api/profile/test_update_user_email.py b/tests/api/profile/test_update_user_email.py index 0b2e4a5..8ffff07 100644 --- a/tests/api/profile/test_update_user_email.py +++ b/tests/api/profile/test_update_user_email.py @@ -8,8 +8,8 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.security.crypto import generate_otp -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import faker @@ -19,7 +19,7 @@ async def test_update_user_email( # noqa: PLR0913 user: UserDTO, user_token: TokenResponse, verify_token: str, - webhook_change_email: Webhook, + webhook_change_email: WebhookEndpoint, uow: SQLAlchemyUOW, ) -> None: code = generate_otp() @@ -38,7 +38,7 @@ async def test_update_user_email( # noqa: PLR0913 assert user.email == new_email try: - await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_change_email.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_change_email.id)) except NoResultFoundError: pytest.fail("No webhook delivery created") diff --git a/tests/api/profile/test_update_user_password.py b/tests/api/profile/test_update_user_password.py index 8284100..69fba5b 100644 --- a/tests/api/profile/test_update_user_password.py +++ b/tests/api/profile/test_update_user_password.py @@ -6,8 +6,8 @@ from fastid.auth.schemas import TokenResponse, UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import faker @@ -16,7 +16,7 @@ async def test_update_user_password( # noqa: PLR0913 user: UserDTO, user_token: TokenResponse, verify_token: str, - webhook_change_password: Webhook, + webhook_change_password: WebhookEndpoint, uow: SQLAlchemyUOW, ) -> None: new_password = faker.password() @@ -31,7 +31,7 @@ async def test_update_user_password( # noqa: PLR0913 UserDTO.model_validate_json(response.content) try: - await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_change_password.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_change_password.id)) except NoResultFoundError: pytest.fail("No webhook delivery created") diff --git a/tests/api/profile/test_update_user_profile.py b/tests/api/profile/test_update_user_profile.py index 84ffd22..7af6f67 100644 --- a/tests/api/profile/test_update_user_profile.py +++ b/tests/api/profile/test_update_user_profile.py @@ -5,13 +5,17 @@ from fastid.auth.schemas import TokenResponse, UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests import mocks async def test_update_user_profile( - client: AsyncClient, user: UserDTO, user_token: TokenResponse, webhook_profile_update: Webhook, uow: SQLAlchemyUOW + client: AsyncClient, + user: UserDTO, + user_token: TokenResponse, + webhook_profile_update: WebhookEndpoint, + uow: SQLAlchemyUOW, ) -> None: response = await client.patch( "/users/me/profile", @@ -24,6 +28,6 @@ async def test_update_user_profile( assert user.last_name == mocks.USER_UPDATE.last_name try: - await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_profile_update.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_profile_update.id)) except NoResultFoundError: pytest.fail("No webhook delivery created") diff --git a/tests/api/webhooks/test_worker_delivery.py b/tests/api/webhooks/test_worker_delivery.py index e9e4012..f6b7a33 100644 --- a/tests/api/webhooks/test_worker_delivery.py +++ b/tests/api/webhooks/test_worker_delivery.py @@ -9,8 +9,8 @@ from fastid.database.uow import SQLAlchemyUOW from fastid.database.utils import naive_utc from fastid.security.webhooks import verify_standard_headers -from fastid.webhooks.models import Webhook, WebhookDeliveryStatus -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookDeliveryStatus, WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender from fastid.webhooks.worker import WebhookWorker from tests.dependencies import get_test_uow @@ -46,7 +46,7 @@ async def send(self, url: str, body: bytes, headers: dict[str, str]) -> WebhookR ) async def test_worker_records_delivery_outcome( # noqa: PLR0913 client: AsyncClient, - webhook_registration: Webhook, + webhook_registration: WebhookEndpoint, uow: SQLAlchemyUOW, monkeypatch: pytest.MonkeyPatch, status_code: int, @@ -61,7 +61,7 @@ async def test_worker_records_delivery_outcome( # noqa: PLR0913 assert await WebhookWorker(sender=sender).run_once() == 1 - delivery = await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_registration.id)) + delivery = await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_registration.id)) assert delivery.status == expected_status assert delivery.attempt_count == 1 assert delivery.status_code == status_code @@ -75,7 +75,7 @@ async def test_worker_records_delivery_outcome( # noqa: PLR0913 async def test_workers_do_not_claim_the_same_delivery( client: AsyncClient, - webhook_registration: Webhook, + webhook_registration: WebhookEndpoint, monkeypatch: pytest.MonkeyPatch, ) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) @@ -91,13 +91,13 @@ async def test_workers_do_not_claim_the_same_delivery( async def test_worker_recovers_an_expired_lease( client: AsyncClient, - webhook_registration: Webhook, + webhook_registration: WebhookEndpoint, uow: SQLAlchemyUOW, monkeypatch: pytest.MonkeyPatch, ) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) assert response.status_code == status.HTTP_201_CREATED - delivery = await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_registration.id)) + delivery = await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_registration.id)) delivery.status = WebhookDeliveryStatus.processing delivery.leased_until = naive_utc() - timedelta(seconds=1) await uow.commit() diff --git a/tests/api/webhooks/test_wrong_url.py b/tests/api/webhooks/test_wrong_url.py index 73874b0..75e0ac4 100644 --- a/tests/api/webhooks/test_wrong_url.py +++ b/tests/api/webhooks/test_wrong_url.py @@ -4,17 +4,17 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook, WebhookDeliveryStatus -from fastid.webhooks.repositories import WebhookDeliveryWebhookIDSpecification +from fastid.webhooks.models import WebhookDeliveryStatus, WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import USER_CREATE -async def test_wrong_url(client: AsyncClient, webhook_wrong_url: Webhook, uow: SQLAlchemyUOW) -> None: +async def test_wrong_url(client: AsyncClient, webhook_wrong_url: WebhookEndpoint, uow: SQLAlchemyUOW) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) assert response.status_code == status.HTTP_201_CREATED try: - delivery = await uow.webhook_deliveries.find(WebhookDeliveryWebhookIDSpecification(webhook_wrong_url.id)) + delivery = await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_wrong_url.id)) assert delivery.status == WebhookDeliveryStatus.pending assert delivery.status_code is None assert delivery.attempt_count == 0 diff --git a/tests/utils/webhooks.py b/tests/utils/webhooks.py index b7de23c..c3a10f6 100644 --- a/tests/utils/webhooks.py +++ b/tests/utils/webhooks.py @@ -2,11 +2,11 @@ from fastid.apps.schemas import AppDTO from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint -async def create_webhook(uow: SQLAlchemyUOW, oauth_app: AppDTO, data: dict[str, Any]) -> Webhook: - record = Webhook(app_id=oauth_app.id, **data) - await uow.webhooks.add(record) +async def create_webhook(uow: SQLAlchemyUOW, oauth_app: AppDTO, data: dict[str, Any]) -> WebhookEndpoint: + record = WebhookEndpoint(app_id=oauth_app.id, **data) + await uow.webhook_endpoints.add(record) await uow.commit() return record diff --git a/tests/webhooks/test_endpoint_models.py b/tests/webhooks/test_endpoint_models.py new file mode 100644 index 0000000..5fe7a31 --- /dev/null +++ b/tests/webhooks/test_endpoint_models.py @@ -0,0 +1,11 @@ +from fastid.apps.models import App +from fastid.webhooks.models import WebhookDelivery, WebhookEndpoint + + +def test_webhook_endpoint_and_delivery_mapping_names() -> None: + assert WebhookEndpoint.__tablename__ == "webhook_endpoints" + assert WebhookDelivery.__table__.c.endpoint_id.name == "endpoint_id" + foreign_key = next(iter(WebhookDelivery.__table__.c.endpoint_id.foreign_keys)) + assert foreign_key.target_fullname == "webhook_endpoints.id" + assert WebhookDelivery.endpoint.property.mapper.class_ is WebhookEndpoint + assert App.webhook_endpoints.property.mapper.class_ is WebhookEndpoint From 5ba75c5e7b780ad0bcc8815c090ff3a0e556a2f5 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 14:59:07 +0300 Subject: [PATCH 18/28] refactor: separate webhook and event identifiers --- fastid/security/webhooks.py | 6 +++--- fastid/webhooks/worker.py | 14 +++++++------- tests/api/webhooks/test_worker_delivery.py | 2 ++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/fastid/security/webhooks.py b/fastid/security/webhooks.py index 990b190..7352c05 100644 --- a/fastid/security/webhooks.py +++ b/fastid/security/webhooks.py @@ -39,11 +39,11 @@ def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, se return f"v1,{base64.b64encode(digest).decode()}" -def generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str) -> dict[str, str]: +def generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]: return { - STANDARD_ID_HEADER: event_id, + STANDARD_ID_HEADER: webhook_id, STANDARD_TIMESTAMP_HEADER: str(timestamp), - STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, event_id, timestamp, secret_key), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, webhook_id, timestamp, secret_key), "Content-Type": "application/json", "User-Agent": webhook_settings.user_agent, } diff --git a/fastid/webhooks/worker.py b/fastid/webhooks/worker.py index c5da90b..74daf2e 100644 --- a/fastid/webhooks/worker.py +++ b/fastid/webhooks/worker.py @@ -40,7 +40,7 @@ def get_retry_delay(attempt_number: int, retry_after_seconds: int | None, *, jit @dataclass(frozen=True) class ClaimedDelivery: - id: UUID + webhook_id: UUID event_id: UUID event_type: str payload: dict[str, object] @@ -113,7 +113,7 @@ async def _claim(self) -> list[ClaimedDelivery]: delivery.leased_until = lease claimed.append( ClaimedDelivery( - id=delivery.id, + webhook_id=delivery.id, event_id=delivery.event_id, event_type=str(delivery.event_type), payload=delivery.payload, @@ -133,16 +133,16 @@ async def _process(self, delivery: ClaimedDelivery) -> None: body = serialize_payload(delivery.payload) headers = generate_delivery_headers( body, - str(delivery.event_id), + str(delivery.webhook_id), timestamp, delivery.endpoint_secret, ) response = await self.sender.send(delivery.endpoint_url, body, headers) - await self._record(delivery.id, delivery.event_type, timestamp, headers, response) + await self._record(delivery.webhook_id, delivery.event_type, timestamp, headers, response) async def _record( self, - delivery_id: UUID, + webhook_id: UUID, event_type: str, timestamp: int, headers: dict[str, str], @@ -151,7 +151,7 @@ async def _record( now = naive_utc() uow = get_uow_raw() async with uow: - delivery = await uow.webhook_deliveries.get(delivery_id) + delivery = await uow.webhook_deliveries.get(webhook_id) endpoint = await uow.webhook_endpoints.get(delivery.endpoint_id) attempt_number = delivery.attempt_count + 1 stored_headers = { @@ -196,7 +196,7 @@ async def _record( ATTEMPTS.labels(event_type=event_type, outcome=outcome).inc() ATTEMPT_DURATION.labels(event_type=event_type).observe(response.duration_ms / 1000) log.info( - "Webhook attempt completed: delivery_id=%s event_id=%s attempt=%d status_code=%d state=%s duration_ms=%d", + "Webhook attempt completed: webhook_id=%s event_id=%s attempt=%d status_code=%d state=%s duration_ms=%d", delivery.id, delivery.event_id, attempt_number, diff --git a/tests/api/webhooks/test_worker_delivery.py b/tests/api/webhooks/test_worker_delivery.py index f6b7a33..6b353bf 100644 --- a/tests/api/webhooks/test_worker_delivery.py +++ b/tests/api/webhooks/test_worker_delivery.py @@ -68,6 +68,8 @@ async def test_worker_records_delivery_outcome( # noqa: PLR0913 attempts = (await uow.webhook_attempts.get_many()).items assert len(attempts) == 1 body, headers = sender.calls[0] + assert headers["webhook-id"] == str(delivery.id) + assert headers["webhook-id"] != str(delivery.event_id) assert verify_standard_headers(body, headers, webhook_registration.secret) await uow.session.refresh(webhook_registration) assert webhook_registration.is_active is active From 52ea5cd697a49026531bedb1ab4624f14c324d34 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 15:01:59 +0300 Subject: [PATCH 19/28] docs: simplify webhook quick-start verification --- examples/webhook_quickstart.py | 23 +++++----------- tests/examples/test_webhook_quickstart.py | 32 +++-------------------- tests/examples/webhook_helpers.py | 11 ++++---- 3 files changed, 15 insertions(+), 51 deletions(-) diff --git a/examples/webhook_quickstart.py b/examples/webhook_quickstart.py index 7922668..d8a4b7d 100644 --- a/examples/webhook_quickstart.py +++ b/examples/webhook_quickstart.py @@ -5,7 +5,6 @@ import json import logging import os -import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager from typing import Any @@ -14,8 +13,6 @@ import uvicorn from fastapi import FastAPI, HTTPException, Request, Response, status -TIMESTAMP_TOLERANCE_SECONDS = 300 - log = logging.getLogger(__name__) @@ -29,8 +26,8 @@ def _secret_bytes(secret: str) -> bytes: raise ValueError(msg) from exc -def verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool: - signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) +def verify_signature(body: bytes, webhook_id: str, timestamp: str, signatures: str, secret: str) -> bool: + signed = b".".join((webhook_id.encode(), timestamp.encode(), body)) expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() return any( version == "v1" and hmac.compare_digest(value, expected) @@ -47,7 +44,7 @@ def _required_header(request: Request, name: str) -> str: return value -def _validate_payload(value: Any, webhook_id: str) -> tuple[str, str]: +def _validate_payload(value: Any) -> tuple[str, str]: if not isinstance(value, dict): raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") event = value.get("event") @@ -65,8 +62,6 @@ def _validate_payload(value: Any, webhook_id: str) -> tuple[str, str]: normalized_event_id = str(UUID(event_id)) except ValueError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event ID") from exc - if normalized_event_id != webhook_id: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "webhook-id does not match event.event_id") return normalized_event_id, event_type @@ -91,23 +86,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: async def receive_webhook(request: Request) -> Response: body = await request.body() webhook_id = _required_header(request, "webhook-id") - timestamp_value = _required_header(request, "webhook-timestamp") + timestamp = _required_header(request, "webhook-timestamp") signatures = _required_header(request, "webhook-signature") - try: - timestamp = int(timestamp_value) - except ValueError as exc: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc - if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") try: payload = json.loads(body) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid JSON body") from exc - event_id, event_type = _validate_payload(payload, webhook_id) + event_id, event_type = _validate_payload(payload) - # Production consumers must atomically record event_id before applying non-idempotent side effects. + # Before production side effects, use an advanced receiver that checks freshness and claims webhook-id. log.info("Received FastID webhook: event_id=%s event_type=%s", event_id, event_type) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/tests/examples/test_webhook_quickstart.py b/tests/examples/test_webhook_quickstart.py index f0589c8..9d83f01 100644 --- a/tests/examples/test_webhook_quickstart.py +++ b/tests/examples/test_webhook_quickstart.py @@ -1,5 +1,3 @@ -import time - import pytest from fastapi.testclient import TestClient from starlette import status @@ -48,27 +46,17 @@ def test_rejects_missing_header(monkeypatch: pytest.MonkeyPatch, header: str) -> assert response.status_code == status.HTTP_400_BAD_REQUEST -def test_rejects_non_integer_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body, headers = signed_request(payload()) - headers["webhook-timestamp"] = "invalid" - - with TestClient(app) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - - assert response.status_code == status.HTTP_400_BAD_REQUEST - - -def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: +def test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) value = payload() body, _ = signed_request(value) - headers = headers_for(body, str(value["event"]["event_id"]), timestamp=int(time.time()) - 301) + headers = headers_for(body, "opaque-webhook-id", timestamp="opaque-timestamp") with TestClient(app) as client: response = client.post("/fastid-webhooks", content=body, headers=headers) - assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.status_code == status.HTTP_204_NO_CONTENT + assert headers["webhook-id"] != value["event"]["event_id"] def test_rejects_malformed_json(monkeypatch: pytest.MonkeyPatch) -> None: @@ -91,15 +79,3 @@ def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch) -> None: response = client.post("/fastid-webhooks", content=body, headers=headers) assert response.status_code == status.HTTP_400_BAD_REQUEST - - -def test_rejects_header_payload_id_mismatch(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - value = payload() - body, _ = signed_request(value) - headers = headers_for(body, "different-event") - - with TestClient(app) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - - assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/tests/examples/webhook_helpers.py b/tests/examples/webhook_helpers.py index 7ffb926..8906720 100644 --- a/tests/examples/webhook_helpers.py +++ b/tests/examples/webhook_helpers.py @@ -19,17 +19,17 @@ def payload(event_id: str | None = None) -> dict[str, Any]: def headers_for( body: bytes, - event_id: str, + webhook_id: str, *, secret: str = SECRET, - timestamp: int | None = None, + timestamp: int | str | None = None, ) -> dict[str, str]: timestamp = timestamp or int(time.time()) key = base64.b64decode(secret.removeprefix("whsec_"), validate=True) - signed = b".".join((event_id.encode(), str(timestamp).encode(), body)) + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) signature = base64.b64encode(hmac.new(key, signed, hashlib.sha256).digest()).decode() return { - "webhook-id": event_id, + "webhook-id": webhook_id, "webhook-timestamp": str(timestamp), "webhook-signature": f"v1,{signature}", "content-type": "application/json", @@ -38,5 +38,4 @@ def headers_for( def signed_request(value: dict[str, Any], secret: str = SECRET) -> tuple[bytes, dict[str, str]]: body = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() - event_id = str(value["event"]["event_id"]) - return body, headers_for(body, event_id, secret=secret) + return body, headers_for(body, str(uuid4()), secret=secret) From d0039271c99df333c62068bb118422550ef9e9fd Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 15:04:16 +0300 Subject: [PATCH 20/28] docs: use webhook IDs for receiver idempotency --- examples/webhook_advanced.py | 45 ++++++++++++------------- tests/examples/test_webhook_advanced.py | 11 +++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/webhook_advanced.py b/examples/webhook_advanced.py index 291e7a8..1b27c71 100644 --- a/examples/webhook_advanced.py +++ b/examples/webhook_advanced.py @@ -35,11 +35,11 @@ class WebhookEnvelope(BaseModel): class IdempotencyStore(Protocol): - async def claim(self, event_id: str) -> bool: ... + async def claim(self, webhook_id: str) -> bool: ... - async def complete(self, event_id: str) -> None: ... + async def complete(self, webhook_id: str) -> None: ... - async def release(self, event_id: str) -> None: ... + async def release(self, webhook_id: str) -> None: ... class InMemoryIdempotencyStore: @@ -49,22 +49,22 @@ def __init__(self) -> None: self._claims: dict[str, ClaimStatus] = {} self._lock = asyncio.Lock() - async def claim(self, event_id: str) -> bool: + async def claim(self, webhook_id: str) -> bool: async with self._lock: - if event_id in self._claims: + if webhook_id in self._claims: return False - self._claims[event_id] = "processing" + self._claims[webhook_id] = "processing" return True - async def complete(self, event_id: str) -> None: + async def complete(self, webhook_id: str) -> None: async with self._lock: - if event_id in self._claims: - self._claims[event_id] = "completed" + if webhook_id in self._claims: + self._claims[webhook_id] = "completed" - async def release(self, event_id: str) -> None: + async def release(self, webhook_id: str) -> None: async with self._lock: - if self._claims.get(event_id) == "processing": - del self._claims[event_id] + if self._claims.get(webhook_id) == "processing": + del self._claims[webhook_id] def _secret_bytes(secret: str) -> bytes: @@ -113,7 +113,7 @@ async def process_event(event: WebhookEnvelope) -> None: ) -async def _validated_event(request: Request) -> WebhookEnvelope: +async def _validated_event(request: Request) -> tuple[WebhookEnvelope, str]: declared_length = _content_length(request) if declared_length is not None and declared_length > MAX_BODY_BYTES: raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") @@ -137,9 +137,7 @@ async def _validated_event(request: Request) -> WebhookEnvelope: event = WebhookEnvelope.model_validate_json(body) except ValidationError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") from exc - if str(event.event.event_id) != webhook_id: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "webhook-id does not match event.event_id") - return event + return event, webhook_id async def _receive_webhook( @@ -147,25 +145,24 @@ async def _receive_webhook( idempotency_store: IdempotencyStore, processor: EventProcessor, ) -> Response: - event = await _validated_event(request) - event_id = str(event.event.event_id) + event, webhook_id = await _validated_event(request) try: - claimed = await idempotency_store.claim(event_id) + claimed = await idempotency_store.claim(webhook_id) except Exception as exc: - log.exception("Could not claim FastID webhook event_id=%s", event_id) + log.exception("Could not claim FastID webhook: webhook_id=%s", webhook_id) raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc if not claimed: return Response(status_code=status.HTTP_204_NO_CONTENT) try: await processor(event) - await idempotency_store.complete(event_id) + await idempotency_store.complete(webhook_id) except Exception as exc: - log.exception("FastID webhook processing failed: event_id=%s", event_id) + log.exception("FastID webhook processing failed: webhook_id=%s", webhook_id) try: - await idempotency_store.release(event_id) + await idempotency_store.release(webhook_id) except Exception: - log.exception("Could not release FastID webhook claim: event_id=%s", event_id) + log.exception("Could not release FastID webhook claim: webhook_id=%s", webhook_id) raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/tests/examples/test_webhook_advanced.py b/tests/examples/test_webhook_advanced.py index 860eb02..65f3931 100644 --- a/tests/examples/test_webhook_advanced.py +++ b/tests/examples/test_webhook_advanced.py @@ -1,4 +1,5 @@ import asyncio +import time import pytest from fastapi.testclient import TestClient @@ -42,14 +43,16 @@ def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch) -> None async def record(event: WebhookEnvelope) -> None: processed.append(str(event.event.event_id)) - body, headers = signed_request(payload()) + value = payload() + body, headers = signed_request(value) with TestClient(create_app(processor=record)) as client: first = client.post("/fastid-webhooks", content=body, headers=headers) duplicate = client.post("/fastid-webhooks", content=body, headers=headers) assert first.status_code == status.HTTP_204_NO_CONTENT assert duplicate.status_code == status.HTTP_204_NO_CONTENT - assert processed == [headers["webhook-id"]] + assert processed == [value["event"]["event_id"]] + assert headers["webhook-id"] != value["event"]["event_id"] def test_rejects_declared_oversized_body(monkeypatch: pytest.MonkeyPatch) -> None: @@ -86,11 +89,11 @@ def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch) -> None: assert response.status_code == status.HTTP_400_BAD_REQUEST -def test_rejects_header_payload_id_mismatch(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) value = payload() body, _ = signed_request(value) - headers = headers_for(body, "00000000-0000-0000-0000-000000000000") + headers = headers_for(body, "webhook-1", timestamp=int(time.time()) - 301) with TestClient(create_app()) as client: response = client.post("/fastid-webhooks", content=body, headers=headers) From d9da352dbd49339e03268053171a28d7dc920551 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 15:10:41 +0300 Subject: [PATCH 21/28] docs: design async SQLAlchemy webhook example --- ...rovide-tiered-webhook-receiver-examples.md | 2 + .../2026-07-20-webhook-receiver-examples.md | 77 +++++++++++++------ ...-07-20-webhook-receiver-examples-design.md | 18 +++-- 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/docs/adr/0002-provide-tiered-webhook-receiver-examples.md b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md index ae6397e..987bc94 100644 --- a/docs/adr/0002-provide-tiered-webhook-receiver-examples.md +++ b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md @@ -19,6 +19,8 @@ FastID will provide three standalone FastAPI receiver applications: Each application will independently implement Standard Webhooks verification using Python's standard library. Advanced examples will share the same conceptual claim, complete, and release lifecycle but will not import each other. +The SQLAlchemy example will use SQLAlchemy's async APIs with `aiosqlite` supplied by an optional Poetry `examples` +dependency group. ## Consequences diff --git a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md index b6b15a9..d716638 100644 --- a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md +++ b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md @@ -6,7 +6,7 @@ **Architecture:** FastID models a configured destination as `WebhookEndpoint`, a delivered webhook as `WebhookDelivery`, and a domain event as `Event`. `WebhookDelivery.id` is the Webhook ID signed into `webhook-id`; `WebhookDelivery.endpoint_id` links to the endpoint; `event.event_id` remains independent. Every example contains its own Standard Webhooks verifier so it can be copied without importing FastID. -**Tech Stack:** Python 3.12, FastAPI, Pydantic, SQLAlchemy 2, Alembic, Starlette TestClient, pytest, Ruff, mypy +**Tech Stack:** Python 3.12, FastAPI, Pydantic, SQLAlchemy 2 async APIs, aiosqlite, Alembic, Starlette TestClient, pytest, Ruff, mypy ## Global Constraints @@ -23,6 +23,8 @@ - Keep all three example files standalone; they must not import each other or `fastid.*`. - Keep event processing generic: validate and log receipt, but do not dispatch event-specific handlers. - Do not demonstrate a FastID application database in the quick-start or in-memory examples. +- Put `aiosqlite` in an optional Poetry `examples` dependency group; do not add it to FastID's runtime dependencies. +- Use async SQLAlchemy end-to-end in the SQLAlchemy example; do not use `run_in_threadpool` or synchronous sessions. - Prefix every shell command with `rtk`. ## File structure @@ -479,42 +481,67 @@ rtk proxy git commit -m "docs: use webhook IDs for receiver idempotency" ### Task 5: Add the standalone SQLAlchemy receiver **Files:** +- Modify: `pyproject.toml` +- Modify: `poetry.lock` - Create: `examples/webhook_sqlalchemy.py` - Modify: `tests/examples/test_webhook_sqlalchemy.py` **Interfaces:** - Produces: `WebhookReceipt` with unique string `webhook_id`, `status`, `created_at`, and `updated_at` columns. -- Produces: `SQLAlchemyIdempotencyStore(engine: Engine)` with synchronous `claim`, `complete`, and `release` methods. +- Produces: `SQLAlchemyIdempotencyStore(session_factory: async_sessionmaker[AsyncSession])` with async `claim`, + `complete`, and `release` methods. - Produces: `create_app(database_url: str | None = None, processor: EventProcessor = process_event) -> FastAPI`. - Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401 | 413 | 500`. -- [ ] **Step 1: Finish the failing SQLAlchemy store tests** +- [ ] **Step 1: Add the optional async SQLite driver group** + +Run: + +```powershell +rtk proxy poetry add --group examples --optional "aiosqlite@^0.21.0" +``` + +Expected `pyproject.toml` entries: + +```toml +[tool.poetry.group.examples] +optional = true + +[tool.poetry.group.examples.dependencies] +aiosqlite = "^0.21.0" +``` + +- [ ] **Step 2: Finish the failing async SQLAlchemy store tests** Use `tmp_path / "webhooks.sqlite3"` and two store instances: ```python -def test_claim_persists_across_store_instances(tmp_path: Path) -> None: - database_url = f"sqlite:///{tmp_path / 'webhooks.sqlite3'}" - first = SQLAlchemyIdempotencyStore(create_store_engine(database_url)) - second = SQLAlchemyIdempotencyStore(create_store_engine(database_url)) - first.create_schema() - assert first.claim("webhook-1") - first.complete("webhook-1") - assert not second.claim("webhook-1") +async def test_claim_persists_across_store_instances(tmp_path: Path) -> None: + database_url = f"sqlite+aiosqlite:///{tmp_path / 'webhooks.sqlite3'}" + first_engine = create_store_engine(database_url) + second_engine = create_store_engine(database_url) + first = SQLAlchemyIdempotencyStore(async_sessionmaker(first_engine, expire_on_commit=False)) + second = SQLAlchemyIdempotencyStore(async_sessionmaker(second_engine, expire_on_commit=False)) + await create_schema(first_engine) + assert await first.claim("webhook-1") + await first.complete("webhook-1") + assert not await second.claim("webhook-1") + await first_engine.dispose() + await second_engine.dispose() ``` Add concurrent claim, release/reclaim, valid/duplicate HTTP delivery, processing-failure retry, and parameterized `400`/`401`/`413` coverage. Use independent Webhook IDs and Event IDs throughout. -- [ ] **Step 2: Run the SQLAlchemy tests and verify RED** +- [ ] **Step 3: Run the SQLAlchemy tests and verify RED** ```powershell rtk proxy poetry run pytest tests/examples/test_webhook_sqlalchemy.py -q ``` -Expected: collection fails because `examples.webhook_sqlalchemy` does not exist. +Expected: tests fail because the draft store accepts a synchronous engine and returns non-awaitable values. -- [ ] **Step 3: Implement the durable claim store** +- [ ] **Step 4: Implement the durable async claim store** Create a local declarative model: @@ -532,38 +559,38 @@ class WebhookReceipt(Base): updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) ``` -`claim(webhook_id)` inserts `processing` in `Session.begin()` and returns `False` on `IntegrityError`. +`claim(webhook_id)` inserts `processing` in `AsyncSession.begin()` and returns `False` on `IntegrityError`. `complete(webhook_id)` updates the row to `completed`. `release(webhook_id)` deletes only a `processing` row. -`create_schema()` calls `Base.metadata.create_all(engine)`. `create_store_engine()` sets -`connect_args={"check_same_thread": False}` only for SQLite. +`create_schema(engine)` uses `async with engine.begin()` and `await connection.run_sync(Base.metadata.create_all)`. +`create_store_engine()` returns an `AsyncEngine` from `create_async_engine()`. -- [ ] **Step 4: Implement the standalone FastAPI receiver** +- [ ] **Step 5: Implement the standalone FastAPI receiver** Repeat the advanced example's verifier, envelope, freshness, size, and error behavior without importing another example or `fastid.*`. Lifespan selects the explicit argument, then `WEBHOOK_DATABASE_URL`, then -`sqlite:///webhook-events.sqlite3`; it creates the schema and disposes the engine at shutdown. Run every synchronous -store call through `starlette.concurrency.run_in_threadpool`: +`sqlite+aiosqlite:///webhook-events.sqlite3`; it creates the schema and awaits engine disposal at shutdown. Await every +store call directly: ```python -claimed = await run_in_threadpool(store.claim, webhook_id) +claimed = await store.claim(webhook_id) if not claimed: return Response(status_code=204) try: await processor(event) - await run_in_threadpool(store.complete, webhook_id) + await store.complete(webhook_id) except Exception as exc: - await run_in_threadpool(store.release, webhook_id) + await store.release(webhook_id) raise HTTPException(status_code=500, detail="Webhook processing failed") from exc return Response(status_code=204) ``` -- [ ] **Step 5: Verify GREEN and commit** +- [ ] **Step 6: Verify GREEN and commit** ```powershell rtk proxy poetry run pytest tests/examples/test_webhook_sqlalchemy.py -q rtk proxy poetry run ruff check examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py rtk proxy poetry run mypy examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py -rtk proxy git add examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy git add pyproject.toml poetry.lock examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py rtk proxy git commit -m "docs: add SQLAlchemy webhook receiver example" ``` diff --git a/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md index 2da1499..3451889 100644 --- a/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md +++ b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md @@ -14,7 +14,7 @@ reader can copy it without importing another example or installing FastID as a l ### `examples/webhook_quickstart.py` The quick-start application is the shortest safe receiver. It requires `FASTID_WEBHOOK_SECRET` at startup, reads the -exact raw body, validates the three Standard Webhooks headers and timestamp, parses JSON only after authentication, +exact raw body, uses the three Standard Webhooks headers to verify the signature, parses JSON only after authentication, logs the generic event, and returns `204 No Content`. It demonstrates authentication but not replay protection or idempotency. It reads `webhook-id` and @@ -23,9 +23,9 @@ Webhook ID. A comment directs production consumers to the advanced examples befo ### `examples/webhook_advanced.py` -The advanced application adds request-size enforcement, structural payload validation, header/payload event-ID -matching, explicit error responses, and an asynchronous idempotency-store protocol. Its in-memory adapter uses an -`asyncio.Lock` so concurrent requests in one process cannot claim the same event twice. +The advanced application adds request-size enforcement, structural payload validation, timestamp freshness, explicit +error responses, and an asynchronous idempotency-store protocol. Its in-memory adapter uses an `asyncio.Lock` so +concurrent requests in one process cannot claim the same Webhook ID twice. The adapter is intentionally labeled as a reference implementation: it is not shared across processes and loses state on restart. The protocol boundary shows where a production consumer supplies durable storage without mixing database @@ -38,9 +38,9 @@ complete, and release lifecycle with a table whose Webhook ID is unique. A dupli boundary. `WEBHOOK_DATABASE_URL` configures the database and defaults to a local SQLite file so the example is runnable without -external infrastructure. Synchronous SQLAlchemy work runs in a worker thread so the asynchronous FastAPI endpoint does -not block the event loop. SQLite connections disable the same-thread check; other database URLs use their normal -driver settings. +external infrastructure. The example uses SQLAlchemy's async engine, async session factory, and `AsyncSession` for all +schema and claim operations; it does not bridge synchronous database work through a worker thread. The default driver +is `aiosqlite`, installed from Poetry's optional `examples` dependency group. ## Configuration @@ -54,7 +54,8 @@ The advanced and SQLAlchemy applications use these fixed reference limits: - timestamp tolerance: 300 seconds. The SQLAlchemy application additionally accepts `WEBHOOK_DATABASE_URL`, defaulting to -`sqlite:///webhook-events.sqlite3`. +`sqlite+aiosqlite:///webhook-events.sqlite3`. Install its optional driver with +`poetry install --with examples`. ## Authentication @@ -129,6 +130,7 @@ Coverage includes: - repeated delivery acknowledgement without repeated processing; - concurrent claims against the in-memory adapter; - SQLAlchemy duplicate claims and persistence across store instances. +- async SQLAlchemy schema creation, claim, completion, release, and engine disposal without thread-pool calls. Ruff, mypy, focused example tests, and the complete project suite must pass. From 22d83a3a7bb352f5ac362008ae92698d158fec89 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 15:15:55 +0300 Subject: [PATCH 22/28] docs: add async SQLAlchemy webhook receiver example --- examples/webhook_sqlalchemy.py | 235 ++++++++++++++++++++++ poetry.lock | 23 ++- pyproject.toml | 7 + tests/examples/test_webhook_sqlalchemy.py | 153 ++++++++++++++ 4 files changed, 416 insertions(+), 2 deletions(-) create mode 100644 examples/webhook_sqlalchemy.py create mode 100644 tests/examples/test_webhook_sqlalchemy.py diff --git a/examples/webhook_sqlalchemy.py b/examples/webhook_sqlalchemy.py new file mode 100644 index 0000000..daabd52 --- /dev/null +++ b/examples/webhook_sqlalchemy.py @@ -0,0 +1,235 @@ +import base64 +import binascii +import hashlib +import hmac +import logging +import os +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, HTTPException, Request, Response, status +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field, ValidationError +from sqlalchemy import delete, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +MAX_BODY_BYTES = 1024 * 1024 +TIMESTAMP_TOLERANCE_SECONDS = 300 +DEFAULT_DATABASE_URL = "sqlite+aiosqlite:///webhook-events.sqlite3" +EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] + +log = logging.getLogger(__name__) + + +class EventMetadata(PydanticBaseModel): + event_id: UUID + event_type: str = Field(min_length=1) + timestamp: int + + +class WebhookEnvelope(PydanticBaseModel): + event: EventMetadata + data: dict[str, Any] + + +class Base(DeclarativeBase): + pass + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +class WebhookReceipt(Base): + __tablename__ = "webhook_receipts" + + webhook_id: Mapped[str] = mapped_column(primary_key=True) + status: Mapped[str] + created_at: Mapped[datetime] = mapped_column(default=utc_now) + updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) + + +def create_store_engine(database_url: str) -> AsyncEngine: + return create_async_engine(database_url) + + +async def create_schema(engine: AsyncEngine) -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + +class SQLAlchemyIdempotencyStore: + def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + self.session_factory = session_factory + + async def claim(self, webhook_id: str) -> bool: + try: + async with self.session_factory() as session, session.begin(): + session.add(WebhookReceipt(webhook_id=webhook_id, status="processing")) + await session.flush() + except IntegrityError: + return False + return True + + async def complete(self, webhook_id: str) -> None: + async with self.session_factory() as session, session.begin(): + await session.execute( + update(WebhookReceipt) + .where(WebhookReceipt.webhook_id == webhook_id) + .values(status="completed", updated_at=utc_now()) + ) + + async def release(self, webhook_id: str) -> None: + async with self.session_factory() as session, session.begin(): + await session.execute( + delete(WebhookReceipt).where( + WebhookReceipt.webhook_id == webhook_id, + WebhookReceipt.status == "processing", + ) + ) + + +def _secret_bytes(secret: str) -> bytes: + if not secret.startswith("whsec_"): + return secret.encode() + try: + return base64.b64decode(secret.removeprefix("whsec_"), validate=True) + except (binascii.Error, ValueError) as exc: + msg = "FASTID_WEBHOOK_SECRET contains invalid base64" + raise ValueError(msg) from exc + + +def verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() + return any( + version == "v1" and hmac.compare_digest(value, expected) + for signature in signatures.split() + if "," in signature + for version, value in (signature.split(",", 1),) + ) + + +def _required_header(request: Request, name: str) -> str: + value = request.headers.get(name) + if not value: + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Missing {name} header") + return value + + +def _content_length(request: Request) -> int | None: + value = request.headers.get("content-length") + if value is None: + return None + try: + return int(value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid Content-Length header") from exc + + +async def process_event(event: WebhookEnvelope) -> None: + log.info( + "Received FastID webhook: event_id=%s event_type=%s", + event.event.event_id, + event.event.event_type, + ) + + +async def _validated_event(request: Request) -> tuple[WebhookEnvelope, str]: + declared_length = _content_length(request) + if declared_length is not None and declared_length > MAX_BODY_BYTES: + raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") + body = await request.body() + if len(body) > MAX_BODY_BYTES: + raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") + + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") + + try: + event = WebhookEnvelope.model_validate_json(body) + except ValidationError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") from exc + return event, webhook_id + + +async def _receive_webhook(request: Request, processor: EventProcessor) -> Response: + event, webhook_id = await _validated_event(request) + store: SQLAlchemyIdempotencyStore = request.app.state.idempotency_store + try: + claimed = await store.claim(webhook_id) + except Exception as exc: + log.exception("Could not claim FastID webhook: webhook_id=%s", webhook_id) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc + if not claimed: + return Response(status_code=status.HTTP_204_NO_CONTENT) + + try: + await processor(event) + await store.complete(webhook_id) + except Exception as exc: + log.exception("FastID webhook processing failed: webhook_id=%s", webhook_id) + try: + await store.release(webhook_id) + except Exception: + log.exception("Could not release FastID webhook claim: webhook_id=%s", webhook_id) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def create_app( + database_url: str | None = None, + processor: EventProcessor = process_event, +) -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + secret = os.getenv("FASTID_WEBHOOK_SECRET") + if not secret: + msg = "FASTID_WEBHOOK_SECRET is required" + raise RuntimeError(msg) + try: + _secret_bytes(secret) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + + selected_url = database_url or os.getenv("WEBHOOK_DATABASE_URL") or DEFAULT_DATABASE_URL + engine = create_store_engine(selected_url) + store = SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) + await create_schema(engine) + app.state.webhook_secret = secret + app.state.idempotency_store = store + try: + yield + finally: + await engine.dispose() + + webhook_app = FastAPI(lifespan=lifespan) + + @webhook_app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) + async def receive_webhook(request: Request) -> Response: + return await _receive_webhook(request, processor) + + return webhook_app + + +app = create_app() + + +if __name__ == "__main__": + uvicorn.run("examples.webhook_sqlalchemy:app", host="127.0.0.1", port=8000) diff --git a/poetry.lock b/poetry.lock index 0158551..750e947 100644 --- a/poetry.lock +++ b/poetry.lock @@ -370,6 +370,25 @@ files = [ frozenlist = ">=1.1.0" typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} +[[package]] +name = "aiosqlite" +version = "0.21.0" +description = "asyncio bridge to the standard sqlite3 module" +optional = false +python-versions = ">=3.9" +groups = ["examples"] +files = [ + {file = "aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0"}, + {file = "aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3"}, +] + +[package.dependencies] +typing_extensions = ">=4.0" + +[package.extras] +dev = ["attribution (==1.7.1)", "black (==24.3.0)", "build (>=1.2)", "coverage[toml] (==7.6.10)", "flake8 (==7.0.0)", "flake8-bugbear (==24.12.12)", "flit (==3.10.1)", "mypy (==1.14.1)", "ufmt (==2.5.1)", "usort (==1.0.8.post1)"] +docs = ["sphinx (==8.1.3)", "sphinx-mdinclude (==0.6.1)"] + [[package]] name = "alembic" version = "1.18.0" @@ -4251,7 +4270,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "examples"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -5111,4 +5130,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "b1791a972d4445d49ce544f1c02ade34f45b590888d4c6e2d13b46f04ea078cf" +content-hash = "2306b9aa5f13f9d437b892c20eef3debb146485b53e2cf0778067ea592a3e673" diff --git a/pyproject.toml b/pyproject.toml index 16d867d..c5b8884 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,13 @@ mdx-include = "^1.4.2" mkdocs-markdownextradata-plugin = "^0.2.6" markdown-include-variants = "^0.0.4" + +[tool.poetry.group.examples] +optional = true + +[tool.poetry.group.examples.dependencies] +aiosqlite = "^0.21.0" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/tests/examples/test_webhook_sqlalchemy.py b/tests/examples/test_webhook_sqlalchemy.py new file mode 100644 index 0000000..df823f5 --- /dev/null +++ b/tests/examples/test_webhook_sqlalchemy.py @@ -0,0 +1,153 @@ +import asyncio +import time +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker +from starlette import status + +from examples.webhook_sqlalchemy import ( + MAX_BODY_BYTES, + SQLAlchemyIdempotencyStore, + WebhookEnvelope, + create_app, + create_schema, + create_store_engine, +) +from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request + +CLAIM_COUNT = 10 +EXPECTED_DUPLICATE_CLAIMS = CLAIM_COUNT - 1 +EXPECTED_PROCESS_CALLS = 2 + + +def database_url(tmp_path: Path) -> str: + return f"sqlite+aiosqlite:///{tmp_path / 'webhooks.sqlite3'}" + + +def store_for(engine: AsyncEngine) -> SQLAlchemyIdempotencyStore: + return SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) + + +async def test_claim_persists_across_store_instances(tmp_path: Path) -> None: + url = database_url(tmp_path) + first_engine = create_store_engine(url) + second_engine = create_store_engine(url) + first = store_for(first_engine) + second = store_for(second_engine) + await create_schema(first_engine) + + assert await first.claim("webhook-1") + await first.complete("webhook-1") + assert not await second.claim("webhook-1") + + await first_engine.dispose() + await second_engine.dispose() + + +async def test_concurrent_claim_has_one_winner(tmp_path: Path) -> None: + engine = create_store_engine(database_url(tmp_path)) + store = store_for(engine) + await create_schema(engine) + + results = await asyncio.gather(*(store.claim("webhook-1") for _ in range(CLAIM_COUNT))) + + assert results.count(True) == 1 + assert results.count(False) == EXPECTED_DUPLICATE_CLAIMS + await engine.dispose() + + +async def test_release_allows_retry(tmp_path: Path) -> None: + engine = create_store_engine(database_url(tmp_path)) + store = store_for(engine) + await create_schema(engine) + assert await store.claim("webhook-1") + + await store.release("webhook-1") + + assert await store.claim("webhook-1") + await engine.dispose() + + +def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + processed: list[str] = [] + + async def record(event: WebhookEnvelope) -> None: + processed.append(str(event.event.event_id)) + + value = payload() + body, headers = signed_request(value) + with TestClient(create_app(database_url(tmp_path), processor=record)) as client: + first = client.post("/fastid-webhooks", content=body, headers=headers) + duplicate = client.post("/fastid-webhooks", content=body, headers=headers) + + assert first.status_code == status.HTTP_204_NO_CONTENT + assert duplicate.status_code == status.HTTP_204_NO_CONTENT + assert processed == [value["event"]["event_id"]] + assert headers["webhook-id"] != value["event"]["event_id"] + + +def test_processing_failure_releases_claim(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + calls = 0 + + async def fail_once(event: WebhookEnvelope) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError(str(event.event.event_id)) + + body, headers = signed_request(payload()) + with TestClient(create_app(database_url(tmp_path), processor=fail_once)) as client: + failed = client.post("/fastid-webhooks", content=body, headers=headers) + retried = client.post("/fastid-webhooks", content=body, headers=headers) + + assert failed.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert retried.status_code == status.HTTP_204_NO_CONTENT + assert calls == EXPECTED_PROCESS_CALLS + + +def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"{}" + headers = headers_for(body, "webhook-1") + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_invalid_signature(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + headers["webhook-signature"] = "v1,invalid" + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, _ = signed_request(payload()) + headers = headers_for(body, "webhook-1", timestamp=int(time.time()) - 301) + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_oversized_body(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"x" * (MAX_BODY_BYTES + 1) + headers = headers_for(body, "webhook-1") + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE From 2d0d1167b840750e60df791c08752e3cb2fb33b2 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Mon, 20 Jul 2026 15:20:47 +0300 Subject: [PATCH 23/28] docs: link webhook receiver examples --- docs/docs/tutorial/webhooks.md | 22 +++++++++++++++---- .../2026-07-20-standard-webhook-headers.md | 22 +++++++++---------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/docs/tutorial/webhooks.md b/docs/docs/tutorial/webhooks.md index 4934623..1d0b72e 100644 --- a/docs/docs/tutorial/webhooks.md +++ b/docs/docs/tutorial/webhooks.md @@ -1,14 +1,14 @@ # Webhooks FastID sends user lifecycle events to every active endpoint subscribed to the event type. Delivery is asynchronous, -durable, and at least once: consumers must treat `webhook-id` as an idempotency key because a request can be repeated +durable, and at least once: consumers must treat `webhook-id` as an idempotency key because a delivery can be repeated after a timeout or worker crash. Delivery order is not guaranteed. ## Request format Each request is a JSON `POST` with Standard Webhooks headers: -- `webhook-id`: logical event UUID, unchanged for retries. +- `webhook-id`: Webhook ID, unchanged for retries of the same delivery. - `webhook-timestamp`: Unix timestamp for the delivery attempt. - `webhook-signature`: `v1,` signature. @@ -29,11 +29,11 @@ async def receive(request: Request) -> dict[str, bool]: if not verify_standard_headers(body, request.headers, "whsec_..."): return {"accepted": False} event = await request.json() - # Atomically record event["event"]["event_id"] before applying side effects. + # Atomically record request.headers["webhook-id"] before applying side effects. return {"accepted": True} ``` -The payload remains backward compatible: +The payload contains a separate logical event ID that can be shared by deliveries to multiple endpoints: ```json { @@ -54,6 +54,20 @@ The payload remains backward compatible: User events that historically exposed user fields directly in `data` continue to include those fields during the compatibility period; use `data.user` in new consumers. +## Receiver examples + +Runnable receivers are available for different integration stages: + +- [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies a signature and logs the event. It does + not check timestamp freshness or provide replay protection and idempotency. +- [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds freshness checks, validation, request limits, and + an in-memory Webhook-ID idempotency boundary. +- [`webhook_sqlalchemy.py`](../../../examples/webhook_sqlalchemy.py) persists atomic Webhook-ID claims with async + SQLAlchemy. Install its SQLite driver with `poetry install --with examples`. + +The in-memory example is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared +persistent idempotency store before applying non-idempotent production side effects. + ## Delivery behavior A `2xx` response completes delivery. Other responses and network failures are retried with jittered exponential diff --git a/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md b/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md index e157edf..16d8ba1 100644 --- a/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md +++ b/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md @@ -4,13 +4,13 @@ **Goal:** Remove FastID's legacy `X-Webhook-*` authentication protocol so every delivery uses only Standard Webhooks headers. -**Architecture:** Keep one signing path in `fastid.security.webhooks`: serialize the payload once, sign the exact bytes with the stable event ID, and send only the three standard authentication headers. Remove the legacy configurable protocol from settings and schemas, then update the worker and tutorial to the smaller contract. +**Architecture:** Keep one signing path in `fastid.security.webhooks`: serialize the payload once, sign the exact bytes with the stable Webhook ID, and send only the three standard authentication headers. Remove the legacy configurable protocol from settings and schemas, then update the worker and tutorial to the smaller contract. **Tech Stack:** Python 3.12, FastAPI, Pydantic Settings, pytest, Ruff, mypy ## Global Constraints -- Preserve `webhook-id` as the stable logical event ID and consumer idempotency key. +- Preserve `webhook-id` as the stable Webhook ID and consumer idempotency key. - Sign the exact bytes `webhook-id.webhook-timestamp.raw_body` with HMAC-SHA256. - Keep the `v1,` signature representation and `whsec_` secrets. - Do not change payload fields, retry behavior, persisted deliveries, or endpoint secret generation. @@ -22,7 +22,7 @@ - `fastid/security/webhooks.py`: the sole standard signing and verification implementation. - `fastid/webhooks/config.py`: delivery policy settings; no consumer-controlled authentication protocol settings. - `fastid/webhooks/schemas.py`: webhook payload schemas; no obsolete signing algorithm enum. -- `fastid/webhooks/worker.py`: signs a claimed delivery with its stable event ID and exact body. +- `fastid/webhooks/worker.py`: signs a claimed delivery with its stable Webhook ID and exact body. - `tests/security/test_webhooks.py`: standard header generation and verification behavior. - `tests/mocks.py`: shared webhook payload, timestamp, and event ID fixtures only. - `tests/api/webhooks/test_worker_delivery.py`: existing end-to-end worker/header contract test. @@ -38,7 +38,7 @@ - Modify: `fastid/security/webhooks.py` **Interfaces:** -- Produces: `generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str) -> dict[str, str]` +- Produces: `generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]` - Preserves: `verify_standard_headers(body: bytes, headers: Mapping[str, str], secret_key: str, tolerance_seconds: int = 300) -> bool` - Preserves: `serialize_payload(payload: dict[str, Any]) -> bytes` @@ -176,11 +176,11 @@ def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, se return f"v1,{base64.b64encode(digest).decode()}" -def generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str) -> dict[str, str]: +def generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]: return { - STANDARD_ID_HEADER: event_id, + STANDARD_ID_HEADER: webhook_id, STANDARD_TIMESTAMP_HEADER: str(timestamp), - STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, event_id, timestamp, secret_key), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, webhook_id, timestamp, secret_key), "Content-Type": "application/json", "User-Agent": webhook_settings.user_agent, } @@ -336,7 +336,7 @@ Expected: commit succeeds without staging `docker/Dockerfile`. - Modify: `docs/docs/tutorial/webhooks.md:7-18` **Interfaces:** -- Consumes: `generate_delivery_headers(body: bytes, event_id: str, timestamp: int, secret_key: str)` from Task 1. +- Consumes: `generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str)` from Task 1. - Preserves: worker delivery requests contain a valid signature for the exact body and endpoint secret. - [ ] **Step 1: Run the existing worker contract test against the narrowed function** @@ -349,14 +349,14 @@ rtk pytest tests/api/webhooks/test_worker_delivery.py::test_worker_records_deliv Expected: FAIL with a `TypeError` because the worker still passes the payload and internal delivery ID. -- [ ] **Step 2: Update the worker to sign only the stable event ID and raw body** +- [ ] **Step 2: Update the worker to sign only the stable Webhook ID and raw body** Replace the header-generation block in `WebhookWorker._process()` with: ```python headers = generate_delivery_headers( body, - str(delivery.event_id), + str(delivery.id), timestamp, delivery.endpoint_secret, ) @@ -383,7 +383,7 @@ Replace the opening of the request-format section in `docs/docs/tutorial/webhook Each request is a JSON `POST` with Standard Webhooks headers: -- `webhook-id`: logical event UUID, unchanged for retries. +- `webhook-id`: Webhook ID, unchanged for retries of the same delivery. - `webhook-timestamp`: Unix timestamp for the delivery attempt. - `webhook-signature`: `v1,` signature. ``` From f33eb80adf2fbb726bd7decc1ce51ab48b2b1de9 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Thu, 23 Jul 2026 16:24:38 +0300 Subject: [PATCH 24/28] docs: design webhook quickstart timestamp verification --- ...uickstart-timestamp-verification-design.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md diff --git a/docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md b/docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md new file mode 100644 index 0000000..fe0b5bb --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md @@ -0,0 +1,51 @@ +# Webhook Quickstart Timestamp Verification + +## Goal + +Protect the `webhook_quickstart` receiver from replayed webhook requests by +validating the signed `webhook-timestamp` header before accepting the request. + +## Behavior + +- The timestamp header must contain an integer Unix timestamp in seconds. +- The timestamp must be within 300 seconds of the receiver's current time. +- Timestamps more than 300 seconds in the past or future are rejected with + HTTP 400 and `Webhook timestamp is outside the allowed tolerance`. +- Non-integer timestamp values are rejected with HTTP 400 and + `Invalid webhook-timestamp header`. +- Timestamp validation occurs before signature validation. +- The original timestamp header value remains part of the signed message, so + existing valid webhook signatures continue to verify. +- Invalid signatures continue to return HTTP 401. + +## Implementation + +Add a `TIMESTAMP_TOLERANCE_SECONDS = 300` constant and import `time` in +`examples/webhook_quickstart.py`. In `receive_webhook`, parse the required +timestamp header as an integer, reject malformed values, compare it with +`int(time.time())`, and reject timestamps outside the tolerance before calling +`verify_signature`. + +The check stays inline to preserve the quickstart's linear, copyable structure +and mirrors the established behavior in `examples/webhook_advanced.py`. + +## Testing + +Update `tests/examples/test_webhook_quickstart.py` using the existing signed +request helpers: + +- Keep the valid-webhook test as coverage for a current signed timestamp. +- Replace the test that treats timestamps as opaque with a malformed-timestamp + rejection test. +- Add rejection tests for signed timestamps older and newer than the allowed + tolerance. +- Assert HTTP 400 and the relevant error detail for timestamp failures. + +Tests will be written and observed failing before the example is changed, then +the focused example tests and the broader relevant test suite will be run. + +## Scope + +This change affects only the quickstart example and its tests. It does not +change FastID's webhook sender, payload schema, signature format, advanced +receiver, or idempotency behavior. From 58cc299d424163791c9c1c4acff5a90a1defc5ab Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Thu, 23 Jul 2026 16:29:13 +0300 Subject: [PATCH 25/28] docs: plan webhook quickstart timestamp verification --- ...bhook-quickstart-timestamp-verification.md | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md diff --git a/docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md b/docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md new file mode 100644 index 0000000..c36c279 --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md @@ -0,0 +1,170 @@ +# Webhook Quickstart Timestamp Verification Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reject malformed and replay-prone timestamps in the `webhook_quickstart` example. + +**Architecture:** Keep timestamp verification inline in the quickstart's request handler so the security flow remains linear and copyable. Parse the signed timestamp header as a Unix timestamp, enforce the same five-minute past/future tolerance as the advanced example, and only then verify the signature. + +**Tech Stack:** Python 3.12, FastAPI, pytest, Starlette `TestClient` + +## Global Constraints + +- The allowed timestamp difference is exactly 300 seconds in either direction. +- Malformed timestamp headers return HTTP 400 with `Invalid webhook-timestamp header`. +- Out-of-tolerance timestamp headers return HTTP 400 with `Webhook timestamp is outside the allowed tolerance`. +- Invalid signatures continue to return HTTP 401. +- The unmodified timestamp header string remains part of the signature input. +- Only the quickstart example and its tests are changed. + +--- + +## File Structure + +- `examples/webhook_quickstart.py`: Continue to own the complete, standalone quickstart receiver, including inline timestamp and signature verification. +- `tests/examples/test_webhook_quickstart.py`: Cover accepted current timestamps and rejected malformed, stale, and excessively future timestamps through HTTP requests. +- `tests/examples/webhook_helpers.py`: Reuse without modification to generate signatures over chosen timestamp header values. + +### Task 1: Verify Quickstart Webhook Timestamp Freshness + +**Files:** + +- Modify: `tests/examples/test_webhook_quickstart.py:1-65` +- Modify: `examples/webhook_quickstart.py:1-96` + +**Interfaces:** + +- Consumes: `headers_for(body: bytes, webhook_id: str, *, secret: str = SECRET, timestamp: int | str | None = None) -> dict[str, str]` +- Produces: HTTP 400 responses for malformed or out-of-tolerance `webhook-timestamp` headers; no new public Python interface. + +- [ ] **Step 1: Write failing timestamp rejection tests** + +Add `import time` to `tests/examples/test_webhook_quickstart.py`. Replace +`test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs` with these +tests: + +```python +def test_rejects_malformed_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "opaque-webhook-id", timestamp="opaque-timestamp") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Invalid webhook-timestamp header"} + + +@pytest.mark.parametrize("offset", [-3600, 3600]) +def test_rejects_timestamp_outside_tolerance( + monkeypatch: pytest.MonkeyPatch, + offset: int, +) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "event-1", timestamp=int(time.time()) + offset) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Webhook timestamp is outside the allowed tolerance"} +``` + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: + +```powershell +rtk pytest -q tests/examples/test_webhook_quickstart.py +``` + +Expected: three timestamp cases fail because the current quickstart accepts +opaque, stale, and future timestamp strings and returns HTTP 204 instead of +HTTP 400. Existing tests pass. + +- [ ] **Step 3: Implement minimal inline timestamp verification** + +In `examples/webhook_quickstart.py`, add the standard-library import and +tolerance constant: + +```python +import os +import time +from collections.abc import AsyncIterator + +# ... + +log = logging.getLogger(__name__) + +TIMESTAMP_TOLERANCE_SECONDS = 300 +``` + +Replace the timestamp-header portion of `receive_webhook` with: + +```python + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp_value, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") +``` + +Keep `verify_signature` accepting `timestamp: str` so it signs the exact header +value instead of its parsed integer representation. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: + +```powershell +rtk pytest -q tests/examples/test_webhook_quickstart.py +``` + +Expected: all quickstart tests pass with no warnings or errors. + +- [ ] **Step 5: Run relevant example regression tests** + +Run: + +```powershell +rtk pytest -q tests/examples +``` + +Expected: all example tests pass with no warnings or errors. + +- [ ] **Step 6: Run lint checks for changed Python files** + +Run: + +```powershell +rtk ruff check examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py +rtk ruff format --check examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py +``` + +Expected: both commands exit successfully with no lint or formatting changes +required. + +- [ ] **Step 7: Commit the implementation** + +Review the staged scope before committing because the worktree contains +unrelated user changes: + +```powershell +rtk git status --short +rtk git add -- examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py +rtk git diff --cached --stat +rtk git commit -m "feat: verify webhook quickstart timestamps" +``` + +Expected: the staged diff contains exactly the quickstart example and its test +file, and the commit succeeds. From cd52074c6e75d3d5de4f2229457f6a31c4103ca4 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Thu, 23 Jul 2026 16:29:28 +0300 Subject: [PATCH 26/28] chore: ignore local worktrees --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 350c65e..36558ae 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,6 @@ cython_debug/ .dev.env !.env.example certs + +# Git worktrees +.worktrees/ From a40fe1474b6f0406916d689dd09d27073f80f376 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Thu, 23 Jul 2026 16:34:48 +0300 Subject: [PATCH 27/28] feat: verify webhook quickstart timestamps --- examples/webhook_quickstart.py | 13 ++++++++++-- tests/examples/test_webhook_quickstart.py | 25 ++++++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/examples/webhook_quickstart.py b/examples/webhook_quickstart.py index d8a4b7d..d6239dc 100644 --- a/examples/webhook_quickstart.py +++ b/examples/webhook_quickstart.py @@ -5,6 +5,7 @@ import json import logging import os +import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager from typing import Any @@ -15,6 +16,8 @@ log = logging.getLogger(__name__) +TIMESTAMP_TOLERANCE_SECONDS = 300 + def _secret_bytes(secret: str) -> bytes: if not secret.startswith("whsec_"): @@ -86,9 +89,15 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: async def receive_webhook(request: Request) -> Response: body = await request.body() webhook_id = _required_header(request, "webhook-id") - timestamp = _required_header(request, "webhook-timestamp") + timestamp_value = _required_header(request, "webhook-timestamp") signatures = _required_header(request, "webhook-signature") - if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp_value, signatures, request.app.state.webhook_secret): raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") try: payload = json.loads(body) diff --git a/tests/examples/test_webhook_quickstart.py b/tests/examples/test_webhook_quickstart.py index 9d83f01..38ff71f 100644 --- a/tests/examples/test_webhook_quickstart.py +++ b/tests/examples/test_webhook_quickstart.py @@ -1,3 +1,5 @@ +import time + import pytest from fastapi.testclient import TestClient from starlette import status @@ -46,7 +48,7 @@ def test_rejects_missing_header(monkeypatch: pytest.MonkeyPatch, header: str) -> assert response.status_code == status.HTTP_400_BAD_REQUEST -def test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_malformed_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) value = payload() body, _ = signed_request(value) @@ -55,8 +57,25 @@ def test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs(monkeypatch: with TestClient(app) as client: response = client.post("/fastid-webhooks", content=body, headers=headers) - assert response.status_code == status.HTTP_204_NO_CONTENT - assert headers["webhook-id"] != value["event"]["event_id"] + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Invalid webhook-timestamp header"} + + +@pytest.mark.parametrize("offset", [-3600, 3600]) +def test_rejects_timestamp_outside_tolerance( + monkeypatch: pytest.MonkeyPatch, + offset: int, +) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "event-1", timestamp=int(time.time()) + offset) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Webhook timestamp is outside the allowed tolerance"} def test_rejects_malformed_json(monkeypatch: pytest.MonkeyPatch) -> None: From b54b47e2bffa135d053fbd54a4c95c049a790099 Mon Sep 17 00:00:00 2001 From: "ivan.stasevich" Date: Thu, 23 Jul 2026 16:37:37 +0300 Subject: [PATCH 28/28] feat: implement SQLAlchemy-based idempotency store for webhooks --- docs/docs/tutorial/webhooks.md | 4 +- .../2026-07-20-webhook-receiver-examples.md | 2 +- examples/webhook_advanced.py | 113 ++++++--- examples/webhook_sqlalchemy.py | 235 ------------------ tests/examples/test_webhook_advanced.py | 128 ++++++---- tests/examples/test_webhook_sqlalchemy.py | 153 ------------ 6 files changed, 155 insertions(+), 480 deletions(-) delete mode 100644 examples/webhook_sqlalchemy.py delete mode 100644 tests/examples/test_webhook_sqlalchemy.py diff --git a/docs/docs/tutorial/webhooks.md b/docs/docs/tutorial/webhooks.md index 1d0b72e..d2fe79a 100644 --- a/docs/docs/tutorial/webhooks.md +++ b/docs/docs/tutorial/webhooks.md @@ -60,9 +60,7 @@ Runnable receivers are available for different integration stages: - [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies a signature and logs the event. It does not check timestamp freshness or provide replay protection and idempotency. -- [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds freshness checks, validation, request limits, and - an in-memory Webhook-ID idempotency boundary. -- [`webhook_sqlalchemy.py`](../../../examples/webhook_sqlalchemy.py) persists atomic Webhook-ID claims with async +- [`webhook_sqlalchemy.py`](../../../examples/webhook_advanced.py) persists atomic Webhook-ID claims with async SQLAlchemy. Install its SQLite driver with `poetry install --with examples`. The in-memory example is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared diff --git a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md index d716638..9fa398b 100644 --- a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md +++ b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md @@ -628,7 +628,7 @@ Runnable receiver examples are available for different integration stages: - [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies a signature and logs the event. - [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds freshness checks, validation, limits, and an in-memory Webhook-ID idempotency boundary. -- [`webhook_sqlalchemy.py`](../../../examples/webhook_sqlalchemy.py) persists atomic Webhook-ID claims with SQLAlchemy. +- [`webhook_sqlalchemy.py`](../../../examples/webhook_advanced.py) persists atomic Webhook-ID claims with SQLAlchemy. The quick-start authenticates the request but does not provide replay protection or idempotency. The in-memory example is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared persistent store before diff --git a/examples/webhook_advanced.py b/examples/webhook_advanced.py index 1b27c71..daabd52 100644 --- a/examples/webhook_advanced.py +++ b/examples/webhook_advanced.py @@ -1,4 +1,3 @@ -import asyncio import base64 import binascii import hashlib @@ -8,63 +7,93 @@ import time from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager -from typing import Any, Literal, Protocol +from datetime import UTC, datetime +from typing import Any from uuid import UUID import uvicorn from fastapi import FastAPI, HTTPException, Request, Response, status -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field, ValidationError +from sqlalchemy import delete, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column MAX_BODY_BYTES = 1024 * 1024 TIMESTAMP_TOLERANCE_SECONDS = 300 -ClaimStatus = Literal["processing", "completed"] +DEFAULT_DATABASE_URL = "sqlite+aiosqlite:///webhook-events.sqlite3" EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] log = logging.getLogger(__name__) -class EventMetadata(BaseModel): +class EventMetadata(PydanticBaseModel): event_id: UUID event_type: str = Field(min_length=1) timestamp: int -class WebhookEnvelope(BaseModel): +class WebhookEnvelope(PydanticBaseModel): event: EventMetadata data: dict[str, Any] -class IdempotencyStore(Protocol): - async def claim(self, webhook_id: str) -> bool: ... +class Base(DeclarativeBase): + pass - async def complete(self, webhook_id: str) -> None: ... - async def release(self, webhook_id: str) -> None: ... +def utc_now() -> datetime: + return datetime.now(UTC) -class InMemoryIdempotencyStore: - """Concurrency-safe within one process, but neither shared nor durable.""" +class WebhookReceipt(Base): + __tablename__ = "webhook_receipts" - def __init__(self) -> None: - self._claims: dict[str, ClaimStatus] = {} - self._lock = asyncio.Lock() + webhook_id: Mapped[str] = mapped_column(primary_key=True) + status: Mapped[str] + created_at: Mapped[datetime] = mapped_column(default=utc_now) + updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) + + +def create_store_engine(database_url: str) -> AsyncEngine: + return create_async_engine(database_url) + + +async def create_schema(engine: AsyncEngine) -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + +class SQLAlchemyIdempotencyStore: + def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + self.session_factory = session_factory async def claim(self, webhook_id: str) -> bool: - async with self._lock: - if webhook_id in self._claims: - return False - self._claims[webhook_id] = "processing" - return True + try: + async with self.session_factory() as session, session.begin(): + session.add(WebhookReceipt(webhook_id=webhook_id, status="processing")) + await session.flush() + except IntegrityError: + return False + return True async def complete(self, webhook_id: str) -> None: - async with self._lock: - if webhook_id in self._claims: - self._claims[webhook_id] = "completed" + async with self.session_factory() as session, session.begin(): + await session.execute( + update(WebhookReceipt) + .where(WebhookReceipt.webhook_id == webhook_id) + .values(status="completed", updated_at=utc_now()) + ) async def release(self, webhook_id: str) -> None: - async with self._lock: - if self._claims.get(webhook_id) == "processing": - del self._claims[webhook_id] + async with self.session_factory() as session, session.begin(): + await session.execute( + delete(WebhookReceipt).where( + WebhookReceipt.webhook_id == webhook_id, + WebhookReceipt.status == "processing", + ) + ) def _secret_bytes(secret: str) -> bytes: @@ -140,14 +169,11 @@ async def _validated_event(request: Request) -> tuple[WebhookEnvelope, str]: return event, webhook_id -async def _receive_webhook( - request: Request, - idempotency_store: IdempotencyStore, - processor: EventProcessor, -) -> Response: +async def _receive_webhook(request: Request, processor: EventProcessor) -> Response: event, webhook_id = await _validated_event(request) + store: SQLAlchemyIdempotencyStore = request.app.state.idempotency_store try: - claimed = await idempotency_store.claim(webhook_id) + claimed = await store.claim(webhook_id) except Exception as exc: log.exception("Could not claim FastID webhook: webhook_id=%s", webhook_id) raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc @@ -156,11 +182,11 @@ async def _receive_webhook( try: await processor(event) - await idempotency_store.complete(webhook_id) + await store.complete(webhook_id) except Exception as exc: log.exception("FastID webhook processing failed: webhook_id=%s", webhook_id) try: - await idempotency_store.release(webhook_id) + await store.release(webhook_id) except Exception: log.exception("Could not release FastID webhook claim: webhook_id=%s", webhook_id) raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc @@ -168,11 +194,9 @@ async def _receive_webhook( def create_app( - store: IdempotencyStore | None = None, + database_url: str | None = None, processor: EventProcessor = process_event, ) -> FastAPI: - idempotency_store = store or InMemoryIdempotencyStore() - @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: secret = os.getenv("FASTID_WEBHOOK_SECRET") @@ -183,14 +207,23 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: _secret_bytes(secret) except ValueError as exc: raise RuntimeError(str(exc)) from exc + + selected_url = database_url or os.getenv("WEBHOOK_DATABASE_URL") or DEFAULT_DATABASE_URL + engine = create_store_engine(selected_url) + store = SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) + await create_schema(engine) app.state.webhook_secret = secret - yield + app.state.idempotency_store = store + try: + yield + finally: + await engine.dispose() webhook_app = FastAPI(lifespan=lifespan) @webhook_app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) async def receive_webhook(request: Request) -> Response: - return await _receive_webhook(request, idempotency_store, processor) + return await _receive_webhook(request, processor) return webhook_app @@ -199,4 +232,4 @@ async def receive_webhook(request: Request) -> Response: if __name__ == "__main__": - uvicorn.run("examples.webhook_advanced:app", host="127.0.0.1", port=8000) + uvicorn.run("examples.webhook_sqlalchemy:app", host="127.0.0.1", port=8000) diff --git a/examples/webhook_sqlalchemy.py b/examples/webhook_sqlalchemy.py deleted file mode 100644 index daabd52..0000000 --- a/examples/webhook_sqlalchemy.py +++ /dev/null @@ -1,235 +0,0 @@ -import base64 -import binascii -import hashlib -import hmac -import logging -import os -import time -from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import asynccontextmanager -from datetime import UTC, datetime -from typing import Any -from uuid import UUID - -import uvicorn -from fastapi import FastAPI, HTTPException, Request, Response, status -from pydantic import BaseModel as PydanticBaseModel -from pydantic import Field, ValidationError -from sqlalchemy import delete, update -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - -MAX_BODY_BYTES = 1024 * 1024 -TIMESTAMP_TOLERANCE_SECONDS = 300 -DEFAULT_DATABASE_URL = "sqlite+aiosqlite:///webhook-events.sqlite3" -EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] - -log = logging.getLogger(__name__) - - -class EventMetadata(PydanticBaseModel): - event_id: UUID - event_type: str = Field(min_length=1) - timestamp: int - - -class WebhookEnvelope(PydanticBaseModel): - event: EventMetadata - data: dict[str, Any] - - -class Base(DeclarativeBase): - pass - - -def utc_now() -> datetime: - return datetime.now(UTC) - - -class WebhookReceipt(Base): - __tablename__ = "webhook_receipts" - - webhook_id: Mapped[str] = mapped_column(primary_key=True) - status: Mapped[str] - created_at: Mapped[datetime] = mapped_column(default=utc_now) - updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) - - -def create_store_engine(database_url: str) -> AsyncEngine: - return create_async_engine(database_url) - - -async def create_schema(engine: AsyncEngine) -> None: - async with engine.begin() as connection: - await connection.run_sync(Base.metadata.create_all) - - -class SQLAlchemyIdempotencyStore: - def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: - self.session_factory = session_factory - - async def claim(self, webhook_id: str) -> bool: - try: - async with self.session_factory() as session, session.begin(): - session.add(WebhookReceipt(webhook_id=webhook_id, status="processing")) - await session.flush() - except IntegrityError: - return False - return True - - async def complete(self, webhook_id: str) -> None: - async with self.session_factory() as session, session.begin(): - await session.execute( - update(WebhookReceipt) - .where(WebhookReceipt.webhook_id == webhook_id) - .values(status="completed", updated_at=utc_now()) - ) - - async def release(self, webhook_id: str) -> None: - async with self.session_factory() as session, session.begin(): - await session.execute( - delete(WebhookReceipt).where( - WebhookReceipt.webhook_id == webhook_id, - WebhookReceipt.status == "processing", - ) - ) - - -def _secret_bytes(secret: str) -> bytes: - if not secret.startswith("whsec_"): - return secret.encode() - try: - return base64.b64decode(secret.removeprefix("whsec_"), validate=True) - except (binascii.Error, ValueError) as exc: - msg = "FASTID_WEBHOOK_SECRET contains invalid base64" - raise ValueError(msg) from exc - - -def verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool: - signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) - expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() - return any( - version == "v1" and hmac.compare_digest(value, expected) - for signature in signatures.split() - if "," in signature - for version, value in (signature.split(",", 1),) - ) - - -def _required_header(request: Request, name: str) -> str: - value = request.headers.get(name) - if not value: - raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Missing {name} header") - return value - - -def _content_length(request: Request) -> int | None: - value = request.headers.get("content-length") - if value is None: - return None - try: - return int(value) - except ValueError as exc: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid Content-Length header") from exc - - -async def process_event(event: WebhookEnvelope) -> None: - log.info( - "Received FastID webhook: event_id=%s event_type=%s", - event.event.event_id, - event.event.event_type, - ) - - -async def _validated_event(request: Request) -> tuple[WebhookEnvelope, str]: - declared_length = _content_length(request) - if declared_length is not None and declared_length > MAX_BODY_BYTES: - raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") - body = await request.body() - if len(body) > MAX_BODY_BYTES: - raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") - - webhook_id = _required_header(request, "webhook-id") - timestamp_value = _required_header(request, "webhook-timestamp") - signatures = _required_header(request, "webhook-signature") - try: - timestamp = int(timestamp_value) - except ValueError as exc: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc - if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") - if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): - raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") - - try: - event = WebhookEnvelope.model_validate_json(body) - except ValidationError as exc: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") from exc - return event, webhook_id - - -async def _receive_webhook(request: Request, processor: EventProcessor) -> Response: - event, webhook_id = await _validated_event(request) - store: SQLAlchemyIdempotencyStore = request.app.state.idempotency_store - try: - claimed = await store.claim(webhook_id) - except Exception as exc: - log.exception("Could not claim FastID webhook: webhook_id=%s", webhook_id) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc - if not claimed: - return Response(status_code=status.HTTP_204_NO_CONTENT) - - try: - await processor(event) - await store.complete(webhook_id) - except Exception as exc: - log.exception("FastID webhook processing failed: webhook_id=%s", webhook_id) - try: - await store.release(webhook_id) - except Exception: - log.exception("Could not release FastID webhook claim: webhook_id=%s", webhook_id) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -def create_app( - database_url: str | None = None, - processor: EventProcessor = process_event, -) -> FastAPI: - @asynccontextmanager - async def lifespan(app: FastAPI) -> AsyncIterator[None]: - secret = os.getenv("FASTID_WEBHOOK_SECRET") - if not secret: - msg = "FASTID_WEBHOOK_SECRET is required" - raise RuntimeError(msg) - try: - _secret_bytes(secret) - except ValueError as exc: - raise RuntimeError(str(exc)) from exc - - selected_url = database_url or os.getenv("WEBHOOK_DATABASE_URL") or DEFAULT_DATABASE_URL - engine = create_store_engine(selected_url) - store = SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) - await create_schema(engine) - app.state.webhook_secret = secret - app.state.idempotency_store = store - try: - yield - finally: - await engine.dispose() - - webhook_app = FastAPI(lifespan=lifespan) - - @webhook_app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) - async def receive_webhook(request: Request) -> Response: - return await _receive_webhook(request, processor) - - return webhook_app - - -app = create_app() - - -if __name__ == "__main__": - uvicorn.run("examples.webhook_sqlalchemy:app", host="127.0.0.1", port=8000) diff --git a/tests/examples/test_webhook_advanced.py b/tests/examples/test_webhook_advanced.py index 65f3931..e05ce33 100644 --- a/tests/examples/test_webhook_advanced.py +++ b/tests/examples/test_webhook_advanced.py @@ -1,42 +1,76 @@ import asyncio import time +from pathlib import Path import pytest from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker from starlette import status from examples.webhook_advanced import ( MAX_BODY_BYTES, - InMemoryIdempotencyStore, + SQLAlchemyIdempotencyStore, WebhookEnvelope, create_app, + create_schema, + create_store_engine, ) from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request -CLAIM_COUNT = 20 +CLAIM_COUNT = 10 EXPECTED_DUPLICATE_CLAIMS = CLAIM_COUNT - 1 EXPECTED_PROCESS_CALLS = 2 -async def test_concurrent_claim_has_one_winner() -> None: - store = InMemoryIdempotencyStore() +def database_url(tmp_path: Path) -> str: + return f"sqlite+aiosqlite:///{tmp_path / 'webhooks.sqlite3'}" - results = await asyncio.gather(*(store.claim("event-1") for _ in range(CLAIM_COUNT))) + +def store_for(engine: AsyncEngine) -> SQLAlchemyIdempotencyStore: + return SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) + + +async def test_claim_persists_across_store_instances(tmp_path: Path) -> None: + url = database_url(tmp_path) + first_engine = create_store_engine(url) + second_engine = create_store_engine(url) + first = store_for(first_engine) + second = store_for(second_engine) + await create_schema(first_engine) + + assert await first.claim("webhook-1") + await first.complete("webhook-1") + assert not await second.claim("webhook-1") + + await first_engine.dispose() + await second_engine.dispose() + + +async def test_concurrent_claim_has_one_winner(tmp_path: Path) -> None: + engine = create_store_engine(database_url(tmp_path)) + store = store_for(engine) + await create_schema(engine) + + results = await asyncio.gather(*(store.claim("webhook-1") for _ in range(CLAIM_COUNT))) assert results.count(True) == 1 assert results.count(False) == EXPECTED_DUPLICATE_CLAIMS + await engine.dispose() -async def test_release_allows_retry() -> None: - store = InMemoryIdempotencyStore() - assert await store.claim("event-1") +async def test_release_allows_retry(tmp_path: Path) -> None: + engine = create_store_engine(database_url(tmp_path)) + store = store_for(engine) + await create_schema(engine) + assert await store.claim("webhook-1") - await store.release("event-1") + await store.release("webhook-1") - assert await store.claim("event-1") + assert await store.claim("webhook-1") + await engine.dispose() -def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch) -> None: +def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) processed: list[str] = [] @@ -45,7 +79,7 @@ async def record(event: WebhookEnvelope) -> None: value = payload() body, headers = signed_request(value) - with TestClient(create_app(processor=record)) as client: + with TestClient(create_app(database_url(tmp_path), processor=record)) as client: first = client.post("/fastid-webhooks", content=body, headers=headers) duplicate = client.post("/fastid-webhooks", content=body, headers=headers) @@ -55,67 +89,65 @@ async def record(event: WebhookEnvelope) -> None: assert headers["webhook-id"] != value["event"]["event_id"] -def test_rejects_declared_oversized_body(monkeypatch: pytest.MonkeyPatch) -> None: +def test_processing_failure_releases_claim(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body, headers = signed_request(payload()) - headers["content-length"] = str(MAX_BODY_BYTES + 1) + calls = 0 - with TestClient(create_app()) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) + async def fail_once(event: WebhookEnvelope) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError(str(event.event.event_id)) - assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE + body, headers = signed_request(payload()) + with TestClient(create_app(database_url(tmp_path), processor=fail_once)) as client: + failed = client.post("/fastid-webhooks", content=body, headers=headers) + retried = client.post("/fastid-webhooks", content=body, headers=headers) + + assert failed.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert retried.status_code == status.HTTP_204_NO_CONTENT + assert calls == EXPECTED_PROCESS_CALLS -def test_rejects_actual_oversized_body(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body = b"x" * (MAX_BODY_BYTES + 1) - headers = headers_for(body, "event-1") + body = b"{}" + headers = headers_for(body, "webhook-1") - with TestClient(create_app()) as client: + with TestClient(create_app(database_url(tmp_path))) as client: response = client.post("/fastid-webhooks", content=body, headers=headers) - assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE + assert response.status_code == status.HTTP_400_BAD_REQUEST -def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_invalid_signature(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - value = payload() - value["event"]["event_type"] = "" - body, headers = signed_request(value) + body, headers = signed_request(payload()) + headers["webhook-signature"] = "v1,invalid" - with TestClient(create_app()) as client: + with TestClient(create_app(database_url(tmp_path))) as client: response = client.post("/fastid-webhooks", content=body, headers=headers) - assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.status_code == status.HTTP_401_UNAUTHORIZED -def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - value = payload() - body, _ = signed_request(value) + body, _ = signed_request(payload()) headers = headers_for(body, "webhook-1", timestamp=int(time.time()) - 301) - with TestClient(create_app()) as client: + with TestClient(create_app(database_url(tmp_path))) as client: response = client.post("/fastid-webhooks", content=body, headers=headers) assert response.status_code == status.HTTP_400_BAD_REQUEST -def test_processing_failure_releases_claim(monkeypatch: pytest.MonkeyPatch) -> None: +def test_rejects_oversized_body(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - calls = 0 - - async def fail_once(event: WebhookEnvelope) -> None: - nonlocal calls - calls += 1 - if calls == 1: - raise RuntimeError(str(event.event.event_id)) + body = b"x" * (MAX_BODY_BYTES + 1) + headers = headers_for(body, "webhook-1") - body, headers = signed_request(payload()) - with TestClient(create_app(processor=fail_once)) as client: - failed = client.post("/fastid-webhooks", content=body, headers=headers) - retried = client.post("/fastid-webhooks", content=body, headers=headers) + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) - assert failed.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR - assert retried.status_code == status.HTTP_204_NO_CONTENT - assert calls == EXPECTED_PROCESS_CALLS + assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE diff --git a/tests/examples/test_webhook_sqlalchemy.py b/tests/examples/test_webhook_sqlalchemy.py deleted file mode 100644 index df823f5..0000000 --- a/tests/examples/test_webhook_sqlalchemy.py +++ /dev/null @@ -1,153 +0,0 @@ -import asyncio -import time -from pathlib import Path - -import pytest -from fastapi.testclient import TestClient -from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker -from starlette import status - -from examples.webhook_sqlalchemy import ( - MAX_BODY_BYTES, - SQLAlchemyIdempotencyStore, - WebhookEnvelope, - create_app, - create_schema, - create_store_engine, -) -from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request - -CLAIM_COUNT = 10 -EXPECTED_DUPLICATE_CLAIMS = CLAIM_COUNT - 1 -EXPECTED_PROCESS_CALLS = 2 - - -def database_url(tmp_path: Path) -> str: - return f"sqlite+aiosqlite:///{tmp_path / 'webhooks.sqlite3'}" - - -def store_for(engine: AsyncEngine) -> SQLAlchemyIdempotencyStore: - return SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) - - -async def test_claim_persists_across_store_instances(tmp_path: Path) -> None: - url = database_url(tmp_path) - first_engine = create_store_engine(url) - second_engine = create_store_engine(url) - first = store_for(first_engine) - second = store_for(second_engine) - await create_schema(first_engine) - - assert await first.claim("webhook-1") - await first.complete("webhook-1") - assert not await second.claim("webhook-1") - - await first_engine.dispose() - await second_engine.dispose() - - -async def test_concurrent_claim_has_one_winner(tmp_path: Path) -> None: - engine = create_store_engine(database_url(tmp_path)) - store = store_for(engine) - await create_schema(engine) - - results = await asyncio.gather(*(store.claim("webhook-1") for _ in range(CLAIM_COUNT))) - - assert results.count(True) == 1 - assert results.count(False) == EXPECTED_DUPLICATE_CLAIMS - await engine.dispose() - - -async def test_release_allows_retry(tmp_path: Path) -> None: - engine = create_store_engine(database_url(tmp_path)) - store = store_for(engine) - await create_schema(engine) - assert await store.claim("webhook-1") - - await store.release("webhook-1") - - assert await store.claim("webhook-1") - await engine.dispose() - - -def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - processed: list[str] = [] - - async def record(event: WebhookEnvelope) -> None: - processed.append(str(event.event.event_id)) - - value = payload() - body, headers = signed_request(value) - with TestClient(create_app(database_url(tmp_path), processor=record)) as client: - first = client.post("/fastid-webhooks", content=body, headers=headers) - duplicate = client.post("/fastid-webhooks", content=body, headers=headers) - - assert first.status_code == status.HTTP_204_NO_CONTENT - assert duplicate.status_code == status.HTTP_204_NO_CONTENT - assert processed == [value["event"]["event_id"]] - assert headers["webhook-id"] != value["event"]["event_id"] - - -def test_processing_failure_releases_claim(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - calls = 0 - - async def fail_once(event: WebhookEnvelope) -> None: - nonlocal calls - calls += 1 - if calls == 1: - raise RuntimeError(str(event.event.event_id)) - - body, headers = signed_request(payload()) - with TestClient(create_app(database_url(tmp_path), processor=fail_once)) as client: - failed = client.post("/fastid-webhooks", content=body, headers=headers) - retried = client.post("/fastid-webhooks", content=body, headers=headers) - - assert failed.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR - assert retried.status_code == status.HTTP_204_NO_CONTENT - assert calls == EXPECTED_PROCESS_CALLS - - -def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body = b"{}" - headers = headers_for(body, "webhook-1") - - with TestClient(create_app(database_url(tmp_path))) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - - assert response.status_code == status.HTTP_400_BAD_REQUEST - - -def test_rejects_invalid_signature(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body, headers = signed_request(payload()) - headers["webhook-signature"] = "v1,invalid" - - with TestClient(create_app(database_url(tmp_path))) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - - assert response.status_code == status.HTTP_401_UNAUTHORIZED - - -def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body, _ = signed_request(payload()) - headers = headers_for(body, "webhook-1", timestamp=int(time.time()) - 301) - - with TestClient(create_app(database_url(tmp_path))) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - - assert response.status_code == status.HTTP_400_BAD_REQUEST - - -def test_rejects_oversized_body(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) - body = b"x" * (MAX_BODY_BYTES + 1) - headers = headers_for(body, "webhook-1") - - with TestClient(create_app(database_url(tmp_path))) as client: - response = client.post("/fastid-webhooks", content=body, headers=headers) - - assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE