diff --git a/.codegraph/.gitignore b/.codegraph/.gitignore new file mode 100644 index 0000000..d20c0fe --- /dev/null +++ b/.codegraph/.gitignore @@ -0,0 +1,5 @@ +# CodeGraph data files — local to each machine, not for committing. +# Ignore everything in .codegraph/ except this file itself, so transient +# files (the database, daemon.pid, sockets, logs) never show up in git. +* +!.gitignore diff --git a/.env.example b/.env.example deleted file mode 100644 index 2aa8dd5..0000000 --- a/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -POSTGRES_PASSWORD="changethis" -REDIS_PASSWORD="changethis" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..24ed2d7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +docker/Dockerfile text eol=lf +docker/*.sh text eol=lf diff --git a/.gitignore b/.gitignore index 350c65e..36558ae 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,6 @@ cython_debug/ .dev.env !.env.example certs + +# Git worktrees +.worktrees/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d6a98c..9e7b2a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,7 +59,9 @@ repos: hooks: - id: detect-secrets args: [ - '--baseline', '.secrets.baseline', + '--exclude-files', 'docs', + '--exclude-files', 'migrations', + '--exclude-files', 'alembic.ini', ] exclude: package.lock.json stages: [ pre-commit, pre-merge-commit, manual ] diff --git a/.secrets.baseline b/.secrets.baseline deleted file mode 100644 index 943aa6a..0000000 --- a/.secrets.baseline +++ /dev/null @@ -1,165 +0,0 @@ -{ - "version": "1.5.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "DiscordBotTokenDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "GitLabTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "IPPublicDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "OpenAIDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "PypiTokenDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TelegramBotTokenDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_baseline_file", - "filename": ".secrets.baseline" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - }, - { - "path": "detect_secrets.filters.regex.should_exclude_file", - "pattern": [ - "migrations/*" - ] - } - ], - "results": { - ".env.example": [ - { - "type": "Secret Keyword", - "filename": ".env.example", - "hashed_secret": "cdb0e76c1a69873cbdcdbe0a142d56c023dc9f22", - "is_verified": false, - "line_number": 1 - } - ], - "alembic.ini": [ - { - "type": "Basic Auth Credentials", - "filename": "alembic.ini", - "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", - "is_verified": false, - "line_number": 62 - } - ], - "app\\db\\config.py": [ - { - "type": "Basic Auth Credentials", - "filename": "app\\db\\config.py", - "hashed_secret": "cdb0e76c1a69873cbdcdbe0a142d56c023dc9f22", - "is_verified": false, - "line_number": 7 - } - ] - }, - "generated_at": "2026-05-29T10:11:37Z" -} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8cc7dd1..2c95ca7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -80,5 +80,22 @@ services: retries: 3 start_period: 40s + fastid-webhook-worker: + build: + context: . + dockerfile: docker/Dockerfile + target: dev + env_file: .env + entrypoint: ["poetry", "run", "python", "-m", "fastid.webhooks.worker"] + depends_on: + fastid-app: + condition: service_healthy + environment: + FASTID_DB_URL: ${FASTID_DB_URL:-postgresql+asyncpg://${POSTGRES_USER:-fastid}:${POSTGRES_PASSWORD:?database password required}@postgres:5432/${POSTGRES_DB:-fastid}} + FASTID_WEBHOOK_ALLOW_INSECURE_URLS: ${FASTID_WEBHOOK_ALLOW_INSECURE_URLS:-true} + volumes: + - "./fastid:/opt/fastid/fastid" + restart: unless-stopped + volumes: pg_data: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c437f2a..f30dfb3 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -4,3 +4,8 @@ services: context: . dockerfile: docker/Dockerfile target: prod + fastid-webhook-worker: + build: + context: . + dockerfile: docker/Dockerfile + target: prod diff --git a/docker-compose.yml b/docker-compose.yml index f59ffe3..45616c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,5 +67,18 @@ services: retries: 3 start_period: 40s + fastid-webhook-worker: + env_file: + - .env + image: everysoftware/fastid:latest + entrypoint: ["python", "-m", "fastid.webhooks.worker"] + depends_on: + fastid-app: + condition: service_healthy + environment: + FASTID_DB_URL: ${FASTID_DB_URL:-postgresql+asyncpg://${POSTGRES_USER:-fastid}:${POSTGRES_PASSWORD:?database password required}@postgres:5432/${POSTGRES_DB:-fastid}} + FASTID_WEBHOOK_ALLOW_INSECURE_URLS: ${FASTID_WEBHOOK_ALLOW_INSECURE_URLS:-false} + restart: unless-stopped + volumes: pg_data: diff --git a/docs/adr/0001-use-standard-webhook-headers.md b/docs/adr/0001-use-standard-webhook-headers.md new file mode 100644 index 0000000..b596b46 --- /dev/null +++ b/docs/adr/0001-use-standard-webhook-headers.md @@ -0,0 +1,26 @@ +# ADR 0001: Use Only Standard Webhook Headers + +- Status: Accepted +- Date: 2026-07-20 + +## Context + +FastID sends both a legacy configurable `X-Webhook-*` signature protocol and the fixed Standard Webhooks protocol. The +protocols use different identifiers, signed representations, and signature encodings. Consumers therefore have two +ways to authenticate the same delivery, only one of which protects the exact transmitted bytes. + +## Decision + +FastID will send and support only `webhook-id`, `webhook-timestamp`, and `webhook-signature`. The signature will remain a +versioned base64 HMAC-SHA256 over `webhook-id.webhook-timestamp.raw_body`. + +The legacy signing functions, verification function, configurable header names, and configurable signature algorithm +will be removed. + +## Consequences + +- New examples and integrations have one authentication contract. +- The stable Webhook ID is unambiguously the consumer idempotency key. +- Verification authenticates the exact transmitted body. +- Existing consumers of `X-Webhook-*` must migrate to Standard Webhooks headers. +- Supporting a future signing change will require a new version in `webhook-signature`, not a second header family. diff --git a/docs/adr/0002-provide-tiered-webhook-receiver-examples.md b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md new file mode 100644 index 0000000..987bc94 --- /dev/null +++ b/docs/adr/0002-provide-tiered-webhook-receiver-examples.md @@ -0,0 +1,32 @@ +# ADR 0002: Provide Tiered Standalone Webhook Receiver Examples + +- Status: Accepted +- Date: 2026-07-20 + +## Context + +A minimal webhook receiver is useful for first success but omits production concerns. Adding storage abstractions and +database setup to that same file would obscure the authentication flow. Making examples import shared implementation +code would reduce duplication but make each example harder to copy into another service. + +## Decision + +FastID will provide three standalone FastAPI receiver applications: + +- a quick-start authentication example; +- an advanced in-memory Webhook-ID idempotency reference; +- an advanced SQLAlchemy Webhook-ID idempotency reference. + +Each application will independently implement Standard Webhooks verification using Python's standard library. Advanced +examples will share the same conceptual claim, complete, and release lifecycle but will not import each other. +The SQLAlchemy example will use SQLAlchemy's async APIs with `aiosqlite` supplied by an optional Poetry `examples` +dependency group. + +## Consequences + +- New users can reach a working receiver without database setup. +- Experienced users can compare volatile and durable idempotency boundaries. +- Each file can be copied independently. +- Signature-verification code is intentionally duplicated and must be changed consistently if the protocol evolves. +- The in-memory example is not production-durable and must say so prominently. +- The SQLAlchemy example introduces database concepts only in its dedicated file. diff --git a/docs/docs/tutorial/webhooks.md b/docs/docs/tutorial/webhooks.md new file mode 100644 index 0000000..d2fe79a --- /dev/null +++ b/docs/docs/tutorial/webhooks.md @@ -0,0 +1,89 @@ +# Webhooks + +FastID sends user lifecycle events to every active endpoint subscribed to the event type. Delivery is asynchronous, +durable, and at least once: consumers must treat `webhook-id` as an idempotency key because a delivery can be repeated +after a timeout or worker crash. Delivery order is not guaranteed. + +## Request format + +Each request is a JSON `POST` with Standard Webhooks headers: + +- `webhook-id`: Webhook ID, unchanged for retries of the same delivery. +- `webhook-timestamp`: Unix timestamp for the delivery attempt. +- `webhook-signature`: `v1,` signature. + +The signed value is the exact byte sequence `webhook-id.webhook-timestamp.raw_body`. Verify the raw request body before +parsing JSON; parsing and serializing it again can change the bytes. + +```python +from fastapi import FastAPI, Header, Request + +from fastid.security.webhooks import verify_standard_headers + +app = FastAPI() + + +@app.post("/fastid-webhooks") +async def receive(request: Request) -> dict[str, bool]: + body = await request.body() + if not verify_standard_headers(body, request.headers, "whsec_..."): + return {"accepted": False} + event = await request.json() + # Atomically record request.headers["webhook-id"] before applying side effects. + return {"accepted": True} +``` + +The payload contains a separate logical event ID that can be shared by deliveries to multiple endpoints: + +```json +{ + "event": { + "event_type": "user_registration", + "event_id": "019b...", + "timestamp": 1784293200 + }, + "data": { + "user": { + "id": "019b...", + "email": "person@example.com" + } + } +} +``` + +User events that historically exposed user fields directly in `data` continue to include those fields during the +compatibility period; use `data.user` in new consumers. + +## Receiver examples + +Runnable receivers are available for different integration stages: + +- [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies a signature and logs the event. It does + not check timestamp freshness or provide replay protection and idempotency. +- [`webhook_sqlalchemy.py`](../../../examples/webhook_advanced.py) persists atomic Webhook-ID claims with async + SQLAlchemy. Install its SQLite driver with `poetry install --with examples`. + +The in-memory example is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared +persistent idempotency store before applying non-idempotent production side effects. + +## Delivery behavior + +A `2xx` response completes delivery. Other responses and network failures are retried with jittered exponential +backoff for roughly three days. FastID respects `Retry-After` up to 24 hours, never follows redirects, disables an +endpoint immediately after `410 Gone`, and disables it after all attempts are exhausted. + +The API transaction stores delivery records; a separate worker sends them: + +```console +python -m fastid.webhooks.worker +``` + +Docker Compose starts this worker automatically. Its Prometheus metrics are exposed on port `9101` inside the Compose +network. Delivery and attempt history is also visible in the FastID admin application. + +## Endpoint security + +Production endpoints must use HTTPS, contain no URL credentials, and resolve only to public IP addresses. FastID pins +the validated address for the connection and does not follow redirects. Keep the worker in an egress-restricted network +as defense in depth. Local HTTP and private-address endpoints require the explicit +`FASTID_WEBHOOK_ALLOW_INSECURE_URLS=true` development setting. diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000..4e5326b --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,59 @@ +# Glossary + +## Webhooks + +**Delivery attempt** +: One HTTP request made for a webhook event. Retries are separate attempts with a new timestamp and signature. + +**Claim** +: An atomic operation that reserves a Webhook ID for processing. Only the request that creates the claim may apply the +event's side effects. + +**Complete** +: Mark a claimed event as successfully processed so later deliveries are acknowledged without repeating work. + +**Duplicate delivery** +: A delivery whose Webhook ID has already been claimed or completed. Consumers acknowledge it without applying the event +again. + +**Endpoint** +: A consumer-owned URL registered to receive one FastID webhook event type. + +**Event** +: A logical user-lifecycle occurrence emitted by FastID. One event can have multiple delivery attempts. + +**Event ID** +: The logical domain event UUID sent in `event.event_id`. One event can create separate messages for multiple webhook +endpoints. + +**Idempotency key** +: A stable identifier atomically recorded by a consumer to prevent repeated side effects when an event is delivered +more than once. + +**Idempotency store** +: A component that atomically claims Webhook IDs and records their processing state. A production implementation must be +shared by all receiver processes and survive restarts. + +**Webhook Endpoint** +: A configured webhook destination containing its URL, secret, event type, and activation state. FastID represents it +as `WebhookEndpoint`; a delivery refers to it through `endpoint_id`. + +**Webhook ID** +: The delivery identifier sent in `webhook-id`. It is stable across retries of that delivery and is the consumer's +idempotency key. FastID stores it as `WebhookDelivery.id`. + +**Release** +: Remove or expire an incomplete claim after processing fails, allowing a later FastID retry to claim the event again. + +**Raw body** +: The exact request bytes received before JSON parsing or re-serialization. These bytes are part of the signed content. + +**Standard Webhooks headers** +: The `webhook-id`, `webhook-timestamp`, and `webhook-signature` headers used to identify and authenticate a delivery. + +**Webhook secret** +: Endpoint-specific HMAC key material represented as `whsec_`. It must be stored and transmitted as a secret. + +**Webhook signature** +: A versioned, base64-encoded HMAC-SHA256 authenticating +`webhook-id.webhook-timestamp.raw_body`. FastID currently emits version `v1`. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index dd32e45..ac4fc50 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -126,6 +126,7 @@ nav: - tutorial/index.md - tutorial/get_started.md - tutorial/notifications.md + - tutorial/webhooks.md - tutorial/social.md - tutorial/observability.md - tutorial/api.md diff --git a/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md b/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md new file mode 100644 index 0000000..16d8ba1 --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-standard-webhook-headers.md @@ -0,0 +1,447 @@ +# Standard Webhook Headers Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove FastID's legacy `X-Webhook-*` authentication protocol so every delivery uses only Standard Webhooks headers. + +**Architecture:** Keep one signing path in `fastid.security.webhooks`: serialize the payload once, sign the exact bytes with the stable Webhook ID, and send only the three standard authentication headers. Remove the legacy configurable protocol from settings and schemas, then update the worker and tutorial to the smaller contract. + +**Tech Stack:** Python 3.12, FastAPI, Pydantic Settings, pytest, Ruff, mypy + +## Global Constraints + +- Preserve `webhook-id` as the stable Webhook ID and consumer idempotency key. +- Sign the exact bytes `webhook-id.webhook-timestamp.raw_body` with HMAC-SHA256. +- Keep the `v1,` signature representation and `whsec_` secrets. +- Do not change payload fields, retry behavior, persisted deliveries, or endpoint secret generation. +- Do not modify the user's existing `docker/Dockerfile` changes. +- Prefix every shell command with `rtk`. + +## File structure + +- `fastid/security/webhooks.py`: the sole standard signing and verification implementation. +- `fastid/webhooks/config.py`: delivery policy settings; no consumer-controlled authentication protocol settings. +- `fastid/webhooks/schemas.py`: webhook payload schemas; no obsolete signing algorithm enum. +- `fastid/webhooks/worker.py`: signs a claimed delivery with its stable Webhook ID and exact body. +- `tests/security/test_webhooks.py`: standard header generation and verification behavior. +- `tests/mocks.py`: shared webhook payload, timestamp, and event ID fixtures only. +- `tests/api/webhooks/test_worker_delivery.py`: existing end-to-end worker/header contract test. +- `docs/docs/tutorial/webhooks.md`: documents the single supported header family. + +--- + +### Task 1: Make Standard Webhooks the only generated and verified protocol + +**Files:** +- Modify: `tests/security/test_webhooks.py` +- Modify: `tests/mocks.py:8-10,141-153` +- Modify: `fastid/security/webhooks.py` + +**Interfaces:** +- Produces: `generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]` +- Preserves: `verify_standard_headers(body: bytes, headers: Mapping[str, str], secret_key: str, tolerance_seconds: int = 300) -> bool` +- Preserves: `serialize_payload(payload: dict[str, Any]) -> bytes` + +- [ ] **Step 1: Replace legacy-oriented security tests with the single-protocol contract** + +Replace `tests/security/test_webhooks.py` with: + +```python +import pytest + +import fastid.security.webhooks as webhook_security +from fastid.security.webhooks import ( + generate_delivery_headers, + generate_secret, + serialize_payload, + verify_standard_headers, +) +from tests.mocks import WEBHOOK_ID, WEBHOOK_PAYLOAD, WEBHOOK_TIMESTAMP + + +@pytest.mark.parametrize("secret", ["plain-secret", generate_secret()]) +def test_verify_standard_headers(secret: str) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, secret) + + assert set(headers) == { + "webhook-id", + "webhook-timestamp", + "webhook-signature", + "Content-Type", + "User-Agent", + } + assert verify_standard_headers(body, headers, secret) + assert not verify_standard_headers(body + b" ", headers, secret) + assert not verify_standard_headers(body, headers, "wrong-secret") + + +def test_standard_signature_uses_exact_utf8_body() -> None: + payload = {"message": "Привет"} + body = serialize_payload(payload) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + + assert b"\\u" not in body + assert verify_standard_headers(body, {key.upper(): value for key, value in headers.items()}, "secret") + + +@pytest.mark.parametrize( + "headers", + [ + {}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": "invalid", "webhook-signature": "v1,value"}, + {"webhook-id": "", "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": "v1,value"}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": ""}, + ], +) +def test_verify_standard_headers_rejects_malformed_headers(headers: dict[str, str]) -> None: + assert not verify_standard_headers(b"{}", headers, "secret") + + +def test_verify_standard_headers_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + monkeypatch.setattr(webhook_security.time, "time", lambda: WEBHOOK_TIMESTAMP + 301) + + assert not verify_standard_headers(body, headers, "secret") + + +def test_invalid_prefixed_secret_is_a_configuration_error() -> None: + with pytest.raises(ValueError, match="Invalid whsec_ webhook secret"): + generate_delivery_headers(b"{}", WEBHOOK_ID, WEBHOOK_TIMESTAMP, "whsec_not-base64") +``` + +In `tests/mocks.py`, remove the `webhook_settings` import and remove `WEBHOOK_SECRET_KEY`, +`WEBHOOK_EXPIRED_TIMESTAMP`, `WEBHOOK_WRONG_SECRET_KEY`, and `WEBHOOK_TEST_DATA`. Retain exactly these shared values: + +```python +WEBHOOK_PAYLOAD = {"test": {"test1": 1, "test2": "hello", "hello3": True}} +WEBHOOK_TIMESTAMP = get_timestamp() +WEBHOOK_ID = str(get_webhook_id()) +``` + +- [ ] **Step 2: Run the rewritten tests and verify the old API fails** + +Run: + +```powershell +rtk pytest tests/security/test_webhooks.py -q +``` + +Expected: FAIL because the existing `generate_delivery_headers()` still requires `payload` and `delivery_id`, and generated deliveries still contain legacy headers. + +- [ ] **Step 3: Replace the security module with the standard-only implementation** + +Replace `fastid/security/webhooks.py` with: + +```python +import base64 +import hashlib +import hmac +import json +import time +from collections.abc import Mapping +from typing import Any + +from fastid.database.utils import UUIDv7, uuid +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.models import generate_webhook_secret + +STANDARD_ID_HEADER = "webhook-id" +STANDARD_TIMESTAMP_HEADER = "webhook-timestamp" +STANDARD_SIGNATURE_HEADER = "webhook-signature" + + +def serialize_payload(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def generate_secret() -> str: + return generate_webhook_secret() + + +def _secret_bytes(secret_key: str) -> bytes: + if not secret_key.startswith("whsec_"): + return secret_key.encode() + try: + return base64.b64decode(secret_key.removeprefix("whsec_"), validate=True) + except ValueError as exc: + msg = "Invalid whsec_ webhook secret" + raise ValueError(msg) from exc + + +def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + digest = hmac.new(_secret_bytes(secret_key), signed, hashlib.sha256).digest() + return f"v1,{base64.b64encode(digest).decode()}" + + +def generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]: + return { + STANDARD_ID_HEADER: webhook_id, + STANDARD_TIMESTAMP_HEADER: str(timestamp), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, webhook_id, timestamp, secret_key), + "Content-Type": "application/json", + "User-Agent": webhook_settings.user_agent, + } + + +def verify_standard_headers( + body: bytes, + headers: Mapping[str, str], + secret_key: str, + tolerance_seconds: int = webhook_settings.tolerance_seconds, +) -> bool: + normalized = {key.lower(): value for key, value in headers.items()} + try: + timestamp = int(normalized[STANDARD_TIMESTAMP_HEADER]) + webhook_id = normalized[STANDARD_ID_HEADER] + signatures = normalized[STANDARD_SIGNATURE_HEADER].split() + except (KeyError, ValueError): + return False + if not webhook_id or not signatures or not is_timestamp_valid(timestamp, tolerance_seconds): + return False + expected = generate_standard_signature(body, webhook_id, timestamp, secret_key) + return any(hmac.compare_digest(signature, expected) for signature in signatures) + + +def get_event_id() -> UUIDv7: + return uuid() + + +def get_webhook_id() -> UUIDv7: + return uuid() + + +def get_timestamp() -> int: + return int(time.time()) + + +def is_timestamp_valid(timestamp: int, tolerance_seconds: int) -> bool: + current_time = int(time.time()) + return abs(current_time - timestamp) <= tolerance_seconds +``` + +- [ ] **Step 4: Run focused security checks** + +Run: + +```powershell +rtk pytest tests/security/test_webhooks.py -q +rtk ruff check fastid/security/webhooks.py tests/security/test_webhooks.py tests/mocks.py +rtk mypy fastid/security/webhooks.py tests/security/test_webhooks.py +``` + +Expected: all tests pass and both static checks exit successfully. + +- [ ] **Step 5: Commit the standard-only security contract** + +```powershell +rtk git add fastid/security/webhooks.py tests/security/test_webhooks.py tests/mocks.py +rtk git commit -m "refactor: use standard webhook signatures only" +``` + +Expected: commit succeeds without staging `docker/Dockerfile`. + +--- + +### Task 2: Remove the obsolete configuration and schema surface + +**Files:** +- Modify: `fastid/webhooks/config.py:1-12` +- Modify: `fastid/webhooks/schemas.py:1-16` + +**Interfaces:** +- Consumes: `webhook_settings.tolerance_seconds` and `webhook_settings.user_agent` used by Task 1. +- Produces: `WebhookSettings` containing delivery policy only; payload schema classes remain unchanged. + +- [ ] **Step 1: Prove the obsolete API is still present** + +Run: + +```powershell +rtk rg -n "SignatureAlgorithm|signature_algorithm|signature_header|timestamp_header|id_header" fastid +``` + +Expected: matches in `fastid/webhooks/config.py` and `fastid/webhooks/schemas.py`; no matches should remain after this task. + +- [ ] **Step 2: Remove legacy settings while preserving delivery policy** + +Make the beginning of `fastid/webhooks/config.py` exactly: + +```python +from pydantic_settings import SettingsConfigDict + +from fastid.core.schemas import ENV_PREFIX, BaseSettings + + +class WebhookSettings(BaseSettings): + tolerance_seconds: int = 300 + page_expires_in_seconds: int = 60 +``` + +Leave every setting from `retry_delays_seconds` through `user_agent`, the `model_config`, and the `webhook_settings` +instance unchanged. + +- [ ] **Step 3: Remove the unused signing enum** + +Make the imports and first model in `fastid/webhooks/schemas.py` exactly: + +```python +from typing import Any +from uuid import UUID + +from pydantic import Field + +from fastid.auth.schemas import UserDTO +from fastid.core.schemas import BaseModel +from fastid.webhooks.models import WebhookType + + +class SendWebhookRequest(BaseModel): + type: WebhookType + payload: dict[str, Any] = Field(default_factory=dict) +``` + +Leave `Event`, `WebhookPayload`, `Webhook`, `WebhookData`, `UserWebhookData`, and `UserWebhook` unchanged. + +- [ ] **Step 4: Verify removal and static correctness** + +Run: + +```powershell +rtk rg -n "SignatureAlgorithm|signature_algorithm|signature_header|timestamp_header|id_header" fastid +rtk ruff check fastid/webhooks/config.py fastid/webhooks/schemas.py +rtk mypy fastid/webhooks/config.py fastid/webhooks/schemas.py +``` + +Expected: `rg` prints no matches; Ruff and mypy exit successfully. + +- [ ] **Step 5: Commit the public-surface cleanup** + +```powershell +rtk git add fastid/webhooks/config.py fastid/webhooks/schemas.py +rtk git commit -m "refactor: remove legacy webhook configuration" +``` + +Expected: commit succeeds without staging `docker/Dockerfile`. + +--- + +### Task 3: Update the worker and documentation to the single protocol + +**Files:** +- Modify: `fastid/webhooks/worker.py:131-143` +- Test: `tests/api/webhooks/test_worker_delivery.py:39-73` +- Modify: `docs/docs/tutorial/webhooks.md:7-18` + +**Interfaces:** +- Consumes: `generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str)` from Task 1. +- Preserves: worker delivery requests contain a valid signature for the exact body and endpoint secret. + +- [ ] **Step 1: Run the existing worker contract test against the narrowed function** + +Run: + +```powershell +rtk pytest tests/api/webhooks/test_worker_delivery.py::test_worker_records_delivery_outcome -q +``` + +Expected: FAIL with a `TypeError` because the worker still passes the payload and internal delivery ID. + +- [ ] **Step 2: Update the worker to sign only the stable Webhook ID and raw body** + +Replace the header-generation block in `WebhookWorker._process()` with: + +```python + headers = generate_delivery_headers( + body, + str(delivery.id), + timestamp, + delivery.endpoint_secret, + ) +``` + +Do not change payload serialization, sending, or attempt recording. + +- [ ] **Step 3: Run the worker webhook tests** + +Run: + +```powershell +rtk pytest tests/api/webhooks/test_worker_delivery.py tests/webhooks/test_worker.py tests/webhooks/test_sender.py -q +``` + +Expected: all tests pass, including signature verification at `test_worker_delivery.py:70-71`. + +- [ ] **Step 4: Remove the compatibility claim from the tutorial** + +Replace the opening of the request-format section in `docs/docs/tutorial/webhooks.md` with: + +```markdown +## Request format + +Each request is a JSON `POST` with Standard Webhooks headers: + +- `webhook-id`: Webhook ID, unchanged for retries of the same delivery. +- `webhook-timestamp`: Unix timestamp for the delivery attempt. +- `webhook-signature`: `v1,` signature. +``` + +Keep the raw-body verification explanation and all later delivery/security documentation unchanged. + +- [ ] **Step 5: Verify there are no legacy protocol references** + +Run: + +```powershell +rtk rg -n -i "x-webhook|generate_headers|generate_signature|verify_headers|SignatureAlgorithm" fastid tests docs/docs +rtk ruff check fastid tests +rtk mypy fastid +rtk pytest tests/security/test_webhooks.py tests/api/webhooks tests/webhooks -q +rtk git diff --check +``` + +Expected: `rg` prints no matches; Ruff, mypy, pytest, and `git diff --check` all exit successfully. + +- [ ] **Step 6: Commit the worker and tutorial migration** + +```powershell +rtk git add fastid/webhooks/worker.py docs/docs/tutorial/webhooks.md +rtk git commit -m "refactor: send only standard webhook headers" +``` + +Expected: commit succeeds without staging `docker/Dockerfile`. + +--- + +### Task 4: Run final regression verification + +**Files:** +- Verify only; do not modify unrelated files. + +**Interfaces:** +- Consumes: the standard-only webhook contract completed in Tasks 1-3. +- Produces: evidence that the cleanup is ready for the webhook examples. + +- [ ] **Step 1: Run the complete project checks** + +```powershell +rtk pytest -q +rtk ruff check . +rtk mypy fastid +rtk git diff --check +``` + +Expected: all commands exit successfully. If an unrelated pre-existing failure occurs, record its exact command and +output without modifying unrelated code. + +- [ ] **Step 2: Confirm scope and history** + +```powershell +rtk git status --short +rtk git log -4 --oneline +``` + +Expected: only the user's pre-existing `docker/Dockerfile` changes remain; the three cleanup commits follow the design +documentation commit `2f31b19`. diff --git a/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md new file mode 100644 index 0000000..9fa398b --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md @@ -0,0 +1,671 @@ +# Webhook Receiver Examples Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Clarify FastID's webhook identifiers and provide three independently runnable FastAPI webhook receivers for quick-start, in-memory advanced, and SQLAlchemy-backed use cases. + +**Architecture:** FastID models a configured destination as `WebhookEndpoint`, a delivered webhook as `WebhookDelivery`, and a domain event as `Event`. `WebhookDelivery.id` is the Webhook ID signed into `webhook-id`; `WebhookDelivery.endpoint_id` links to the endpoint; `event.event_id` remains independent. Every example contains its own Standard Webhooks verifier so it can be copied without importing FastID. + +**Tech Stack:** Python 3.12, FastAPI, Pydantic, SQLAlchemy 2 async APIs, aiosqlite, Alembic, Starlette TestClient, pytest, Ruff, mypy + +## Global Constraints + +- Use **Webhook Endpoint ID**, **Webhook ID**, and **Event ID** as the domain terms. +- Use concise contextual attributes: `WebhookDelivery.endpoint_id` and `WebhookDelivery.endpoint`. +- Use `webhook_id` internally and `webhook-id` externally for `WebhookDelivery.id`; do not use `message_id` or `delivery_id` aliases for this value. +- Treat `event.event_id` as independent from `webhook-id`; never compare them. +- Use only `webhook-id`, `webhook-timestamp`, and `webhook-signature` authentication headers. +- Verify `webhook-id.webhook-timestamp.raw_body` with HMAC-SHA256 before JSON parsing. +- Accept `v1,` signatures and `whsec_` secrets; retain plain-secret compatibility. +- Require `FASTID_WEBHOOK_SECRET` during FastAPI lifespan startup. +- The quick-start performs signature authentication only: no timestamp freshness, identifier-format, replay, or idempotency checks. +- Use a 300-second timestamp tolerance and a 1 MiB body limit in advanced examples only. +- Keep all three example files standalone; they must not import each other or `fastid.*`. +- Keep event processing generic: validate and log receipt, but do not dispatch event-specific handlers. +- Do not demonstrate a FastID application database in the quick-start or in-memory examples. +- Put `aiosqlite` in an optional Poetry `examples` dependency group; do not add it to FastID's runtime dependencies. +- Use async SQLAlchemy end-to-end in the SQLAlchemy example; do not use `run_in_threadpool` or synchronous sessions. +- Prefix every shell command with `rtk`. + +## File structure + +- `fastid/webhooks/models.py`: endpoint, delivery, and attempt persistence models. +- `fastid/apps/models.py`: application-to-webhook-endpoint relationship. +- `fastid/webhooks/repositories.py`: endpoint and delivery repositories/specifications. +- `fastid/database/uow.py`: repository attributes used by webhook use cases and workers. +- `fastid/webhooks/use_cases.py`: creates one delivery per matching endpoint. +- `fastid/webhooks/worker.py`: signs Webhook IDs and resolves endpoint relationships. +- `fastid/security/webhooks.py`: Standard Webhooks signing and verification helpers. +- `migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py`: lossless endpoint table, version table, column, constraint, and index renames. +- `examples/webhook_quickstart.py`: minimal authenticated receiver. +- `examples/webhook_advanced.py`: validated receiver with an async in-memory idempotency boundary. +- `examples/webhook_sqlalchemy.py`: validated receiver with durable SQLAlchemy claims. +- `tests/examples/webhook_helpers.py`: test-only signer and payload factory. +- `tests/examples/test_webhook_quickstart.py`: quick-start behavior and startup configuration. +- `tests/examples/test_webhook_advanced.py`: advanced validation, limits, duplicates, and concurrent claims. +- `tests/examples/test_webhook_sqlalchemy.py`: database claims, persistence, duplicates, and HTTP behavior. +- `docs/docs/tutorial/webhooks.md`: receiver contract and links to each runnable example. + +--- + +### Task 1: Rename configured webhooks to webhook endpoints + +**Files:** +- Create: `tests/webhooks/test_endpoint_models.py` +- Create: `migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py` +- Modify: `fastid/webhooks/models.py` +- Modify: `fastid/apps/models.py` +- Modify: `fastid/webhooks/repositories.py` +- Modify: `fastid/database/models.py` +- Modify: `fastid/database/versioning.py` +- Modify: `fastid/database/uow.py` +- Modify: `fastid/webhooks/use_cases.py` +- Modify: `fastid/webhooks/worker.py` +- Modify: `fastid/admin/views/settings.py` +- Modify: `fastid/admin/views/versioning.py` +- Modify: `tests/utils/webhooks.py` +- Modify: `tests/api/conftest.py` +- Modify: webhook API tests importing the persistence model under `tests/api/auth/`, `tests/api/profile/`, and `tests/api/webhooks/` + +**Interfaces:** +- Produces: `WebhookEndpoint(VersionedEntity)` mapped to `webhook_endpoints`. +- Produces: `WebhookDelivery.endpoint_id: UUID` and `WebhookDelivery.endpoint: WebhookEndpoint`. +- Produces: `App.webhook_endpoints: list[WebhookEndpoint]`. +- Produces: `WebhookEndpointRepository`, `WebhookEndpointTypeSpecification`, and `WebhookDeliveryEndpointIDSpecification`. +- Preserves: all existing endpoint, delivery, and version row IDs during migration. + +- [ ] **Step 1: Write the failing model-contract test** + +Create `tests/webhooks/test_endpoint_models.py`: + +```python +from fastid.apps.models import App +from fastid.webhooks.models import WebhookDelivery, WebhookEndpoint + + +def test_webhook_endpoint_and_delivery_mapping_names() -> None: + assert WebhookEndpoint.__tablename__ == "webhook_endpoints" + assert WebhookDelivery.__table__.c.endpoint_id.name == "endpoint_id" + foreign_key = next(iter(WebhookDelivery.__table__.c.endpoint_id.foreign_keys)) + assert foreign_key.target_fullname == "webhook_endpoints.id" + assert WebhookDelivery.endpoint.property.mapper.class_ is WebhookEndpoint + assert App.webhook_endpoints.property.mapper.class_ is WebhookEndpoint +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```powershell +rtk proxy poetry run pytest tests/webhooks/test_endpoint_models.py -q +``` + +Expected: collection fails because `WebhookEndpoint` does not exist. + +- [ ] **Step 3: Rename the ORM model and contextual attributes** + +Implement these exact mappings in `fastid/webhooks/models.py` and `fastid/apps/models.py`: + +```python +class WebhookEndpoint(VersionedEntity): + __tablename__ = "webhook_endpoints" + + app_id: Mapped[UUID] = mapped_column(ForeignKey("apps.id"), index=True) + type: Mapped[WebhookType] + secret: Mapped[str] = mapped_column(default=generate_webhook_secret) + url: Mapped[str] + is_active: Mapped[bool] = mapped_column(default=True) + disabled_at: Mapped[datetime.datetime | None] + disabled_reason: Mapped[str | None] + + app: Mapped[App] = relationship(back_populates="webhook_endpoints") + deliveries: Mapped[list[WebhookDelivery]] = relationship(back_populates="endpoint", cascade="delete") + + +class WebhookDelivery(Entity): + __tablename__ = "webhook_deliveries" + + endpoint_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_endpoints.id"), index=True) + event_id: Mapped[UUID] = mapped_column(index=True) + event_type: Mapped[WebhookType] + payload: Mapped[dict[str, Any]] + status: Mapped[WebhookDeliveryStatus] = mapped_column(default=WebhookDeliveryStatus.pending, index=True) + attempt_count: Mapped[int] = mapped_column(default=0) + next_attempt_at: Mapped[datetime.datetime] = mapped_column(index=True) + leased_until: Mapped[datetime.datetime | None] = mapped_column(index=True) + completed_at: Mapped[datetime.datetime | None] + request: Mapped[dict[str, Any] | None] + status_code: Mapped[int | None] + response: Mapped[dict[str, Any] | None] + error: Mapped[str | None] + + endpoint: Mapped[WebhookEndpoint] = relationship(back_populates="deliveries") + attempts: Mapped[list[WebhookAttempt]] = relationship(back_populates="delivery", cascade="delete") +``` + +```python +class App(VersionedEntity): + __tablename__ = "apps" + + name: Mapped[str] + slug: Mapped[str] = mapped_column(unique=True) + client_id: Mapped[str] = mapped_column(default=uuid_hex, index=True) + client_secret: Mapped[str] = mapped_column(default=uuid_hex) + redirect_uris: Mapped[str] = mapped_column(default="") + is_active: Mapped[bool] = mapped_column(default=True) + + webhook_endpoints: Mapped[list[WebhookEndpoint]] = relationship(back_populates="app", cascade="delete") +``` + +Apply these symbol mappings everywhere listed in **Files**: + +```text +Webhook -> WebhookEndpoint +WebhookVersion -> WebhookEndpointVersion +WebhookRepository -> WebhookEndpointRepository +WebhookTypeSpecification -> WebhookEndpointTypeSpecification +WebhookDeliveryWebhookIDSpecification -> WebhookDeliveryEndpointIDSpecification +delivery.webhook_id -> delivery.endpoint_id +delivery.webhook -> delivery.endpoint +App.webhooks -> App.webhook_endpoints +``` + +Keep the payload schema `fastid.webhooks.schemas.Webhook` unchanged; it is not the persistence model. + +- [ ] **Step 4: Update use-case and worker data flow** + +Use endpoint terminology at the call sites: + +```python +endpoint_page = await self.uow.webhook_endpoints.get_many(WebhookEndpointTypeSpecification(dto.type)) +for endpoint in endpoint_page.items: + delivery = WebhookDelivery( + id=get_webhook_id(), + endpoint_id=endpoint.id, + event_id=event_id, + event_type=dto.type, + payload=payload, + status=WebhookDeliveryStatus.pending, + next_attempt_at=now, + ) +``` + +In the worker, use `joinedload(WebhookDelivery.endpoint)`, read `delivery.endpoint.url` and +`delivery.endpoint.secret`, load `uow.webhook_endpoints.get(delivery.endpoint_id)`, and cancel sibling deliveries with: + +```python +WebhookDelivery.endpoint_id == endpoint.id +``` + +- [ ] **Step 5: Add the lossless Alembic rename migration** + +Create `migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py` with: + +```python +"""rename webhooks to endpoints + +Revision ID: 5b2c8d7e9f10 +Revises: a4e1d2c3b5f6 +Create Date: 2026-07-20 12:00:00 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "5b2c8d7e9f10" +down_revision: str | None = "a4e1d2c3b5f6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +UPGRADE_RENAMES = ( + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_pkey TO webhook_endpoints_pkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_app_id_fkey TO webhook_endpoints_app_id_fkey", + "ALTER INDEX webhooks_app_id_idx RENAME TO webhook_endpoints_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhooks_version_pkey TO webhook_endpoints_version_pkey", + "ALTER INDEX webhooks_version_app_id_idx RENAME TO webhook_endpoints_version_app_id_idx", + "ALTER INDEX webhooks_version_end_transaction_id_idx RENAME TO webhook_endpoints_version_end_transaction_id_idx", + "ALTER INDEX webhooks_version_operation_type_idx RENAME TO webhook_endpoints_version_operation_type_idx", + "ALTER INDEX webhooks_version_transaction_id_idx RENAME TO webhook_endpoints_version_transaction_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_webhook_id_fkey TO webhook_deliveries_endpoint_id_fkey", + "ALTER INDEX webhook_deliveries_webhook_id_idx RENAME TO webhook_deliveries_endpoint_id_idx", +) + +DOWNGRADE_RENAMES = ( + "ALTER INDEX webhook_deliveries_endpoint_id_idx RENAME TO webhook_deliveries_webhook_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_endpoint_id_fkey TO webhook_deliveries_webhook_id_fkey", + "ALTER INDEX webhook_endpoints_version_transaction_id_idx RENAME TO webhooks_version_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_operation_type_idx RENAME TO webhooks_version_operation_type_idx", + "ALTER INDEX webhook_endpoints_version_end_transaction_id_idx RENAME TO webhooks_version_end_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_app_id_idx RENAME TO webhooks_version_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhook_endpoints_version_pkey TO webhooks_version_pkey", + "ALTER INDEX webhook_endpoints_app_id_idx RENAME TO webhooks_app_id_idx", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_app_id_fkey TO webhooks_app_id_fkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_pkey TO webhooks_pkey", +) + + +def upgrade() -> None: + op.rename_table("webhooks", "webhook_endpoints") + op.rename_table("webhooks_version", "webhook_endpoints_version") + op.alter_column("webhook_deliveries", "webhook_id", new_column_name="endpoint_id") + for statement in UPGRADE_RENAMES: + op.execute(statement) + + +def downgrade() -> None: + for statement in DOWNGRADE_RENAMES: + op.execute(statement) + op.alter_column("webhook_deliveries", "endpoint_id", new_column_name="webhook_id") + op.rename_table("webhook_endpoints_version", "webhooks_version") + op.rename_table("webhook_endpoints", "webhooks") +``` + +This migration must rename in place; it must not recreate or copy tables. + +- [ ] **Step 6: Verify the endpoint rename** + +Run: + +```powershell +rtk proxy poetry run pytest tests/webhooks/test_endpoint_models.py tests/api/webhooks tests/api/auth tests/api/profile -q +rtk proxy poetry run ruff check fastid tests/webhooks/test_endpoint_models.py tests/api +rtk proxy poetry run mypy fastid tests/webhooks/test_endpoint_models.py +rtk rg -n "WebhookDelivery\.webhook_id|WebhookDelivery\.webhook\b|from fastid\.webhooks\.models import Webhook\b|class Webhook\(" fastid tests +``` + +Expected: tests and static checks pass; the final search reports only the intentionally unchanged payload schema if it +matches at all. + +- [ ] **Step 7: Commit the endpoint rename** + +```powershell +rtk proxy git add fastid migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py tests +rtk proxy git commit -m "refactor: name webhook endpoints explicitly" +``` + +--- + +### Task 2: Make Webhook ID the signed delivery identifier + +**Files:** +- Modify: `tests/api/webhooks/test_worker_delivery.py` +- Modify: `tests/security/test_webhooks.py` +- Modify: `tests/mocks.py` +- Modify: `fastid/security/webhooks.py` +- Modify: `fastid/webhooks/worker.py` + +**Interfaces:** +- Produces: `generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str`. +- Produces: `generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]`. +- Guarantees: `headers["webhook-id"] == str(WebhookDelivery.id)` and is independent of `event.event_id`. + +- [ ] **Step 1: Write the failing sender-semantics assertion** + +In the successful worker delivery test, add: + +```python +headers = sender.requests[0].headers +assert headers["webhook-id"] == str(delivery.id) +assert headers["webhook-id"] != str(delivery.event_id) +``` + +- [ ] **Step 2: Run the focused test and verify RED** + +```powershell +rtk proxy poetry run pytest tests/api/webhooks/test_worker_delivery.py::test_worker_records_delivery_outcome -q +``` + +Expected: the first assertion fails while the worker still signs the Event ID. + +- [ ] **Step 3: Use `webhook_id` consistently in signing code** + +Implement: + +```python +def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + digest = hmac.new(_secret_bytes(secret_key), signed, hashlib.sha256).digest() + return f"v1,{base64.b64encode(digest).decode()}" + + +def generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]: + return { + STANDARD_ID_HEADER: webhook_id, + STANDARD_TIMESTAMP_HEADER: str(timestamp), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, webhook_id, timestamp, secret_key), + "Content-Type": "application/json", + "User-Agent": webhook_settings.user_agent, + } +``` + +`verify_standard_headers()` must also name the parsed header value `webhook_id`. The worker passes +`str(delivery.id)` to `generate_delivery_headers()`. + +- [ ] **Step 4: Verify GREEN and commit** + +```powershell +rtk proxy poetry run pytest tests/api/webhooks/test_worker_delivery.py tests/security/test_webhooks.py -q +rtk proxy poetry run ruff check fastid/security/webhooks.py fastid/webhooks/worker.py tests/security tests/api/webhooks/test_worker_delivery.py +rtk proxy poetry run mypy fastid/security/webhooks.py fastid/webhooks/worker.py +rtk proxy git add fastid/security/webhooks.py fastid/webhooks/worker.py tests/security/test_webhooks.py tests/api/webhooks/test_worker_delivery.py tests/mocks.py +rtk proxy git commit -m "refactor: separate webhook and event identifiers" +``` + +--- + +### Task 3: Align the quick-start receiver with signature-only verification + +**Files:** +- Modify: `tests/examples/webhook_helpers.py` +- Modify: `tests/examples/test_webhook_quickstart.py` +- Modify: `examples/webhook_quickstart.py` + +**Interfaces:** +- Produces: `app: FastAPI`. +- Produces: `verify_signature(body: bytes, webhook_id: str, timestamp: str, signatures: str, secret: str) -> bool`. +- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401`. + +- [ ] **Step 1: Write the quick-start contract test** + +Replace freshness and ID-equality tests with: + +```python +def test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = compact_json(payload()) + headers = headers_for(body, "opaque-webhook-id", timestamp="opaque-timestamp") + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + assert response.status_code == 204 +``` + +Ensure `signed_request()` generates a Webhook ID independently of the payload Event ID. + +- [ ] **Step 2: Run the quick-start tests and verify RED** + +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_quickstart.py -q +``` + +Expected: the opaque timestamp or independent Webhook ID is rejected by the old implementation. + +- [ ] **Step 3: Implement signature-only behavior and concise naming** + +Read required headers as strings and authenticate only this input: + +```python +webhook_id = _required_header(request, "webhook-id") +timestamp = _required_header(request, "webhook-timestamp") +signatures = _required_header(request, "webhook-signature") +if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + raise HTTPException(status_code=401, detail="Invalid webhook signature") +``` + +Do not parse the timestamp, validate the Webhook ID format, compare it with `event.event_id`, or add an idempotency +store. Rename every `message_id` local and parameter to `webhook_id`. + +- [ ] **Step 4: Verify GREEN and commit** + +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_quickstart.py -q +rtk proxy poetry run ruff check examples/webhook_quickstart.py tests/examples +rtk proxy poetry run mypy examples/webhook_quickstart.py tests/examples +rtk proxy git add examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py tests/examples/webhook_helpers.py +rtk proxy git commit -m "docs: simplify webhook quick-start verification" +``` + +--- + +### Task 4: Align advanced in-memory idempotency with Webhook ID + +**Files:** +- Modify: `tests/examples/test_webhook_advanced.py` +- Modify: `examples/webhook_advanced.py` + +**Interfaces:** +- Produces: `IdempotencyStore` methods `claim(webhook_id)`, `complete(webhook_id)`, and `release(webhook_id)`. +- Produces: `InMemoryIdempotencyStore` implementing that protocol with one `asyncio.Lock`. +- Produces: `create_app(store: IdempotencyStore | None = None, processor: EventProcessor = process_event) -> FastAPI`. + +- [ ] **Step 1: Write independent-ID and replay tests** + +Use an independently generated `webhook-id`, assert a repeated signed request is processed once, and assert stale +timestamps still return `400`: + +```python +assert headers["webhook-id"] != value["event"]["event_id"] +assert first.status_code == 204 +assert duplicate.status_code == 204 +assert processed_event_ids == [value["event"]["event_id"]] +``` + +Keep concurrent claim and release/retry coverage, but name all store keys `webhook_id`. + +- [ ] **Step 2: Run the advanced tests and verify RED** + +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_advanced.py -q +``` + +Expected: the old header/payload equality check or old API terminology fails. + +- [ ] **Step 3: Implement Webhook-ID idempotency** + +Use this protocol and data flow: + +```python +class IdempotencyStore(Protocol): + async def claim(self, webhook_id: str) -> bool: ... + async def complete(self, webhook_id: str) -> None: ... + async def release(self, webhook_id: str) -> None: ... +``` + +`_validated_event()` returns `(event, webhook_id)` without comparing identifiers. `_receive_webhook()` atomically +claims `webhook_id`, acknowledges a duplicate with `204`, completes it after processing, and releases it on failure. +Retain integer timestamp parsing, 300-second freshness, signature verification, Pydantic validation, and body limits. +Rename log fields from `message_id` to `webhook_id`. + +- [ ] **Step 4: Verify GREEN and commit** + +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_advanced.py -q +rtk proxy poetry run ruff check examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy poetry run mypy examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy git add examples/webhook_advanced.py tests/examples/test_webhook_advanced.py +rtk proxy git commit -m "docs: use webhook IDs for receiver idempotency" +``` + +--- + +### Task 5: Add the standalone SQLAlchemy receiver + +**Files:** +- Modify: `pyproject.toml` +- Modify: `poetry.lock` +- Create: `examples/webhook_sqlalchemy.py` +- Modify: `tests/examples/test_webhook_sqlalchemy.py` + +**Interfaces:** +- Produces: `WebhookReceipt` with unique string `webhook_id`, `status`, `created_at`, and `updated_at` columns. +- Produces: `SQLAlchemyIdempotencyStore(session_factory: async_sessionmaker[AsyncSession])` with async `claim`, + `complete`, and `release` methods. +- Produces: `create_app(database_url: str | None = None, processor: EventProcessor = process_event) -> FastAPI`. +- Endpoint: `POST /fastid-webhooks -> 204 | 400 | 401 | 413 | 500`. + +- [ ] **Step 1: Add the optional async SQLite driver group** + +Run: + +```powershell +rtk proxy poetry add --group examples --optional "aiosqlite@^0.21.0" +``` + +Expected `pyproject.toml` entries: + +```toml +[tool.poetry.group.examples] +optional = true + +[tool.poetry.group.examples.dependencies] +aiosqlite = "^0.21.0" +``` + +- [ ] **Step 2: Finish the failing async SQLAlchemy store tests** + +Use `tmp_path / "webhooks.sqlite3"` and two store instances: + +```python +async def test_claim_persists_across_store_instances(tmp_path: Path) -> None: + database_url = f"sqlite+aiosqlite:///{tmp_path / 'webhooks.sqlite3'}" + first_engine = create_store_engine(database_url) + second_engine = create_store_engine(database_url) + first = SQLAlchemyIdempotencyStore(async_sessionmaker(first_engine, expire_on_commit=False)) + second = SQLAlchemyIdempotencyStore(async_sessionmaker(second_engine, expire_on_commit=False)) + await create_schema(first_engine) + assert await first.claim("webhook-1") + await first.complete("webhook-1") + assert not await second.claim("webhook-1") + await first_engine.dispose() + await second_engine.dispose() +``` + +Add concurrent claim, release/reclaim, valid/duplicate HTTP delivery, processing-failure retry, and parameterized +`400`/`401`/`413` coverage. Use independent Webhook IDs and Event IDs throughout. + +- [ ] **Step 3: Run the SQLAlchemy tests and verify RED** + +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_sqlalchemy.py -q +``` + +Expected: tests fail because the draft store accepts a synchronous engine and returns non-awaitable values. + +- [ ] **Step 4: Implement the durable async claim store** + +Create a local declarative model: + +```python +def utc_now() -> datetime: + return datetime.now(UTC) + + +class WebhookReceipt(Base): + __tablename__ = "webhook_receipts" + + webhook_id: Mapped[str] = mapped_column(primary_key=True) + status: Mapped[str] + created_at: Mapped[datetime] = mapped_column(default=utc_now) + updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) +``` + +`claim(webhook_id)` inserts `processing` in `AsyncSession.begin()` and returns `False` on `IntegrityError`. +`complete(webhook_id)` updates the row to `completed`. `release(webhook_id)` deletes only a `processing` row. +`create_schema(engine)` uses `async with engine.begin()` and `await connection.run_sync(Base.metadata.create_all)`. +`create_store_engine()` returns an `AsyncEngine` from `create_async_engine()`. + +- [ ] **Step 5: Implement the standalone FastAPI receiver** + +Repeat the advanced example's verifier, envelope, freshness, size, and error behavior without importing another example +or `fastid.*`. Lifespan selects the explicit argument, then `WEBHOOK_DATABASE_URL`, then +`sqlite+aiosqlite:///webhook-events.sqlite3`; it creates the schema and awaits engine disposal at shutdown. Await every +store call directly: + +```python +claimed = await store.claim(webhook_id) +if not claimed: + return Response(status_code=204) +try: + await processor(event) + await store.complete(webhook_id) +except Exception as exc: + await store.release(webhook_id) + raise HTTPException(status_code=500, detail="Webhook processing failed") from exc +return Response(status_code=204) +``` + +- [ ] **Step 6: Verify GREEN and commit** + +```powershell +rtk proxy poetry run pytest tests/examples/test_webhook_sqlalchemy.py -q +rtk proxy poetry run ruff check examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy poetry run mypy examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy git add pyproject.toml poetry.lock examples/webhook_sqlalchemy.py tests/examples/test_webhook_sqlalchemy.py +rtk proxy git commit -m "docs: add SQLAlchemy webhook receiver example" +``` + +--- + +### Task 6: Update the tutorial and verify the complete feature + +**Files:** +- Modify: `docs/docs/tutorial/webhooks.md` +- Modify: `docs/superpowers/plans/2026-07-20-standard-webhook-headers.md` +- Modify: `docs/superpowers/plans/2026-07-20-webhook-receiver-examples.md` + +**Interfaces:** +- Consumes: the endpoint terminology and all three runnable receiver paths. +- Produces: one discoverable progression from signature-only quick-start to production-oriented references. + +- [ ] **Step 1: Remove obsolete terminology** + +Use `Webhook ID` in prose and `webhook_id` in code. Replace references to the configured `Webhook` entity with +`Webhook Endpoint` where the distinction matters. Verify with: + +```powershell +rtk rg -ni "message[ _-]?id|WebhookDelivery\.webhook_id|WebhookDelivery\.webhook\b|stable event id|header/payload.*match" docs examples fastid tests +``` + +Expected: no obsolete terminology remains except historical migration descriptions that explicitly name an old schema. + +- [ ] **Step 2: Add the receiver progression to the tutorial** + +Document: + +```markdown +Runnable receiver examples are available for different integration stages: + +- [`webhook_quickstart.py`](../../../examples/webhook_quickstart.py) verifies a signature and logs the event. +- [`webhook_advanced.py`](../../../examples/webhook_advanced.py) adds freshness checks, validation, limits, and an + in-memory Webhook-ID idempotency boundary. +- [`webhook_sqlalchemy.py`](../../../examples/webhook_advanced.py) persists atomic Webhook-ID claims with SQLAlchemy. + +The quick-start authenticates the request but does not provide replay protection or idempotency. The in-memory example +is a concurrency reference, not durable storage. Use the SQLAlchemy example or another shared persistent store before +applying non-idempotent production side effects. +``` + +- [ ] **Step 3: Run focused verification** + +```powershell +rtk proxy poetry run pytest tests/security/test_webhooks.py tests/api/webhooks tests/webhooks tests/examples -q +rtk proxy poetry run ruff check fastid examples tests +rtk proxy poetry run mypy fastid examples tests/examples +rtk proxy poetry run codespell examples tests/examples docs/docs/tutorial/webhooks.md docs/glossary.md +``` + +Expected: all focused tests and static checks pass. + +- [ ] **Step 4: Run complete project verification** + +```powershell +rtk proxy poetry run pytest -q +rtk proxy poetry run ruff check . +rtk proxy poetry run mypy fastid examples tests/examples +rtk proxy git diff --check +rtk proxy git status --short +``` + +Expected: all tests and static checks pass; only the planned final documentation changes remain uncommitted. + +- [ ] **Step 5: Commit documentation and confirm history** + +```powershell +rtk proxy git add docs examples tests fastid migrations +rtk proxy git commit -m "docs: finish webhook receiver examples" +rtk proxy git status --short +rtk proxy git log -8 --oneline +``` + +Expected: the worktree is clean and the endpoint, identifier, receiver, and tutorial commits are at the top of +`feat/enhance-webhooks`. diff --git a/docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md b/docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md new file mode 100644 index 0000000..c36c279 --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-webhook-quickstart-timestamp-verification.md @@ -0,0 +1,170 @@ +# Webhook Quickstart Timestamp Verification Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reject malformed and replay-prone timestamps in the `webhook_quickstart` example. + +**Architecture:** Keep timestamp verification inline in the quickstart's request handler so the security flow remains linear and copyable. Parse the signed timestamp header as a Unix timestamp, enforce the same five-minute past/future tolerance as the advanced example, and only then verify the signature. + +**Tech Stack:** Python 3.12, FastAPI, pytest, Starlette `TestClient` + +## Global Constraints + +- The allowed timestamp difference is exactly 300 seconds in either direction. +- Malformed timestamp headers return HTTP 400 with `Invalid webhook-timestamp header`. +- Out-of-tolerance timestamp headers return HTTP 400 with `Webhook timestamp is outside the allowed tolerance`. +- Invalid signatures continue to return HTTP 401. +- The unmodified timestamp header string remains part of the signature input. +- Only the quickstart example and its tests are changed. + +--- + +## File Structure + +- `examples/webhook_quickstart.py`: Continue to own the complete, standalone quickstart receiver, including inline timestamp and signature verification. +- `tests/examples/test_webhook_quickstart.py`: Cover accepted current timestamps and rejected malformed, stale, and excessively future timestamps through HTTP requests. +- `tests/examples/webhook_helpers.py`: Reuse without modification to generate signatures over chosen timestamp header values. + +### Task 1: Verify Quickstart Webhook Timestamp Freshness + +**Files:** + +- Modify: `tests/examples/test_webhook_quickstart.py:1-65` +- Modify: `examples/webhook_quickstart.py:1-96` + +**Interfaces:** + +- Consumes: `headers_for(body: bytes, webhook_id: str, *, secret: str = SECRET, timestamp: int | str | None = None) -> dict[str, str]` +- Produces: HTTP 400 responses for malformed or out-of-tolerance `webhook-timestamp` headers; no new public Python interface. + +- [ ] **Step 1: Write failing timestamp rejection tests** + +Add `import time` to `tests/examples/test_webhook_quickstart.py`. Replace +`test_treats_webhook_id_and_timestamp_as_opaque_signature_inputs` with these +tests: + +```python +def test_rejects_malformed_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "opaque-webhook-id", timestamp="opaque-timestamp") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Invalid webhook-timestamp header"} + + +@pytest.mark.parametrize("offset", [-3600, 3600]) +def test_rejects_timestamp_outside_tolerance( + monkeypatch: pytest.MonkeyPatch, + offset: int, +) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "event-1", timestamp=int(time.time()) + offset) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Webhook timestamp is outside the allowed tolerance"} +``` + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: + +```powershell +rtk pytest -q tests/examples/test_webhook_quickstart.py +``` + +Expected: three timestamp cases fail because the current quickstart accepts +opaque, stale, and future timestamp strings and returns HTTP 204 instead of +HTTP 400. Existing tests pass. + +- [ ] **Step 3: Implement minimal inline timestamp verification** + +In `examples/webhook_quickstart.py`, add the standard-library import and +tolerance constant: + +```python +import os +import time +from collections.abc import AsyncIterator + +# ... + +log = logging.getLogger(__name__) + +TIMESTAMP_TOLERANCE_SECONDS = 300 +``` + +Replace the timestamp-header portion of `receive_webhook` with: + +```python + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp_value, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") +``` + +Keep `verify_signature` accepting `timestamp: str` so it signs the exact header +value instead of its parsed integer representation. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: + +```powershell +rtk pytest -q tests/examples/test_webhook_quickstart.py +``` + +Expected: all quickstart tests pass with no warnings or errors. + +- [ ] **Step 5: Run relevant example regression tests** + +Run: + +```powershell +rtk pytest -q tests/examples +``` + +Expected: all example tests pass with no warnings or errors. + +- [ ] **Step 6: Run lint checks for changed Python files** + +Run: + +```powershell +rtk ruff check examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py +rtk ruff format --check examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py +``` + +Expected: both commands exit successfully with no lint or formatting changes +required. + +- [ ] **Step 7: Commit the implementation** + +Review the staged scope before committing because the worktree contains +unrelated user changes: + +```powershell +rtk git status --short +rtk git add -- examples/webhook_quickstart.py tests/examples/test_webhook_quickstart.py +rtk git diff --cached --stat +rtk git commit -m "feat: verify webhook quickstart timestamps" +``` + +Expected: the staged diff contains exactly the quickstart example and its test +file, and the commit succeeds. diff --git a/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md b/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md new file mode 100644 index 0000000..b230937 --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-standard-webhook-headers-design.md @@ -0,0 +1,91 @@ +# Standard Webhook Headers Design + +## Context + +FastID currently sends two independent webhook authentication protocols on every delivery: + +- legacy configurable `X-Webhook-*` headers, whose signature covers a re-serialized JSON payload; +- fixed Standard Webhooks headers, whose signature covers the exact raw request body. + +Both the legacy `X-Webhook-Id` and standard `webhook-id` identify one endpoint delivery and remain stable across its +retries. The payload's `event.event_id` separately identifies the logical domain event, which can produce deliveries to +multiple endpoints. Keeping both authentication protocols makes the consumer contract ambiguous and complicates the +webhook examples. + +The repository has no production caller of the legacy signing or verification functions. The legacy protocol remains +only in the delivery-header composition, configuration, tests, and a compatibility note in the webhook tutorial. There +are no tagged releases establishing a published compatibility requirement. + +## Decision + +FastID will expose only the Standard Webhooks header family: + +- `webhook-id`: stable Webhook ID, represented by `WebhookDelivery.id`; +- `webhook-timestamp`: Unix timestamp for the delivery attempt; +- `webhook-signature`: one or more space-separated `v1,` signatures. + +The signed content remains the exact byte sequence +`webhook-id.webhook-timestamp.raw_body`. Endpoint secrets retain the `whsec_` prefix and base64-encoded key material. + +## Code changes + +`generate_delivery_headers()` will directly construct the standard headers plus `Content-Type` and `User-Agent`. It +will sign `WebhookDelivery.id` as the Webhook ID. + +The following legacy API will be removed: + +- `generate_headers()`; +- `generate_signature()`; +- `verify_headers()`; +- the `SignatureAlgorithm` enum and algorithm lookup table; +- configurable legacy ID, timestamp, and signature header settings. + +`generate_standard_signature()`, `verify_standard_headers()`, payload serialization, timestamp validation, and secret +generation remain the supported implementation. + +All callers, tests, and documentation will be updated to use the single protocol. The delivery database ID remains an +internal identifier and is not sent as an authentication header. + +## Request flow + +For each delivery attempt, the worker serializes the payload once, generates a timestamp, and signs the stable Webhook +ID, timestamp, and exact serialized bytes. The sender transmits those same bytes and the generated standard +headers. The payload retains its separate logical event ID. + +Consumers read the raw body, validate timestamp freshness, calculate the expected HMAC over the raw bytes, compare the +signature in constant time, and only then parse JSON. Consumers use `webhook-id` as the idempotency key. + +## Compatibility + +This is an intentional breaking change for consumers that verify `X-Webhook-*`. Because the repository has no tagged +release and documentation already directs new integrations to Standard Webhooks, FastID will remove the legacy path +without a deprecation period. + +No payload fields, delivery retry behavior, endpoint secrets, or persisted delivery records change. + +## Error handling + +The sender continues treating header generation errors as programming or configuration failures. Consumer-side +verification continues returning `False` for missing headers, malformed timestamps, stale timestamps, and signature +mismatches. An invalid `whsec_` value continues raising `ValueError` so configuration errors are distinguishable from +untrusted requests. + +## Verification + +Tests will establish that: + +- generated deliveries contain only the standard authentication headers; +- signatures authenticate the exact UTF-8 body; +- header lookup is case-insensitive; +- body changes, wrong secrets, stale timestamps, and malformed headers fail verification; +- the worker passes valid standard headers to the sender; +- no legacy header settings, helpers, or documentation remain. + +The focused webhook security and worker tests will run first, followed by linting and the broader test suite where +practical. + +## Follow-up + +After this change is verified, FastID will add three standalone webhook receiver examples: a quick-start receiver, an +advanced receiver with an idempotency-store abstraction, and a SQLAlchemy-backed receiver. Their design and +implementation remain separate from this prerequisite cleanup. diff --git a/docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md b/docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md new file mode 100644 index 0000000..268399a --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-webhook-identifier-terminology-design.md @@ -0,0 +1,61 @@ +# Webhook Identifier Terminology Design + +## Problem + +FastID has three independent identifiers in its webhook flow: + +- the configured destination currently represented by `Webhook.id`; +- the individual delivery represented by `WebhookDelivery.id`; +- the domain event represented by `event.event_id`. + +Calling the configured destination simply a webhook makes `webhook_id` ambiguous. The Standard Webhooks +`webhook-id` header identifies the delivered webhook, not the configured destination or the domain event. + +## Decision + +FastID will use these domain terms: + +- **Webhook Endpoint**: the configured URL, secret, event type, and activation state; +- **Webhook ID**: the stable identifier of one delivered webhook, sent in `webhook-id` and represented by + `WebhookDelivery.id`; +- **Event ID**: the logical domain event identifier sent in `event.event_id`. + +The Webhook ID remains stable across retries of the same delivery. One event can create multiple webhook deliveries, +each with its own Webhook ID, when it is sent to multiple endpoints. + +## Code and persistence names + +Domain terminology remains explicit in documentation and user-facing text. Attributes use the shortest unambiguous +name within their owning model: + +| Concept | Code or persistence representation | +| --- | --- | +| Webhook Endpoint | `WebhookEndpoint` | +| Endpoint table | `webhook_endpoints` | +| Webhook Endpoint ID on a delivery | `WebhookDelivery.endpoint_id` | +| Endpoint relationship on a delivery | `WebhookDelivery.endpoint` | +| Webhook ID | `WebhookDelivery.id`, local variable `webhook_id`, HTTP header `webhook-id` | +| Event ID | `Event.event_id` and `WebhookDelivery.event_id` | + +The former `Webhook` model, `webhooks` table, `WebhookDelivery.webhook_id` foreign key, and +`WebhookDelivery.webhook` relationship will be renamed accordingly. Public routes may remain under `/webhooks` +because that path names the webhook feature rather than one identifier. + +## Signing and receiving + +FastID signs `webhook_id.timestamp.raw_body`, where `webhook_id` is `WebhookDelivery.id`. Sender and receiver code +will consistently use the variable name `webhook_id`; it will not use `message_id` or `delivery_id` as aliases. + +Receiver examples will treat `webhook-id` and `event.event_id` as independent values. The quick-start verifies only +the signature and treats the Webhook ID and timestamp as opaque signature inputs. Advanced examples additionally +validate timestamp freshness and use the Webhook ID as their idempotency key. + +## Migration and compatibility + +The database schema will rename `webhooks` to `webhook_endpoints` and the delivery foreign-key column to +`endpoint_id`, preserving existing identifiers and relationships. The external signing contract keeps the standard +header name `webhook-id`; only the value semantics are clarified as the Webhook ID represented by +`WebhookDelivery.id`. + +Tests will verify that the header Webhook ID equals `WebhookDelivery.id`, differs independently from the Event ID, +and remains stable across retries. diff --git a/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md new file mode 100644 index 0000000..3451889 --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-webhook-receiver-examples-design.md @@ -0,0 +1,140 @@ +# Webhook Receiver Examples Design + +## Context + +FastID now sends only Standard Webhooks authentication headers. The tutorial contains a short receiver fragment, but +`examples/` does not contain a complete webhook consumer that users can run or copy. A single example cannot remain +brief enough for a quick start while also teaching replay protection, concurrency, and durable idempotency. + +FastID will provide three standalone FastAPI applications. Each file intentionally repeats signature verification so a +reader can copy it without importing another example or installing FastID as a library. + +## Files and audiences + +### `examples/webhook_quickstart.py` + +The quick-start application is the shortest safe receiver. It requires `FASTID_WEBHOOK_SECRET` at startup, reads the +exact raw body, uses the three Standard Webhooks headers to verify the signature, parses JSON only after authentication, +logs the generic event, and returns `204 No Content`. + +It demonstrates authentication but not replay protection or idempotency. It reads `webhook-id` and +`webhook-timestamp` only because they are signature inputs, without checking timestamp freshness or interpreting the +Webhook ID. A comment directs production consumers to the advanced examples before applying side effects. + +### `examples/webhook_advanced.py` + +The advanced application adds request-size enforcement, structural payload validation, timestamp freshness, explicit +error responses, and an asynchronous idempotency-store protocol. Its in-memory adapter uses an `asyncio.Lock` so +concurrent requests in one process cannot claim the same Webhook ID twice. + +The adapter is intentionally labeled as a reference implementation: it is not shared across processes and loses state +on restart. The protocol boundary shows where a production consumer supplies durable storage without mixing database +setup into this example. + +### `examples/webhook_sqlalchemy.py` + +The SQLAlchemy application is a standalone alternative to the advanced example. It implements an equivalent claim, +complete, and release lifecycle with a table whose Webhook ID is unique. A duplicate insert is the atomic concurrency +boundary. + +`WEBHOOK_DATABASE_URL` configures the database and defaults to a local SQLite file so the example is runnable without +external infrastructure. The example uses SQLAlchemy's async engine, async session factory, and `AsyncSession` for all +schema and claim operations; it does not bridge synchronous database work through a worker thread. The default driver +is `aiosqlite`, installed from Poetry's optional `examples` dependency group. + +## Configuration + +All three applications require a non-empty `FASTID_WEBHOOK_SECRET`. Startup fails with a clear message if it is absent. +Secrets beginning with `whsec_` contain base64-encoded key bytes. Plain-text secrets remain accepted for compatibility +with existing endpoints. + +The advanced and SQLAlchemy applications use these fixed reference limits: + +- maximum request body: 1 MiB; +- timestamp tolerance: 300 seconds. + +The SQLAlchemy application additionally accepts `WEBHOOK_DATABASE_URL`, defaulting to +`sqlite+aiosqlite:///webhook-events.sqlite3`. Install its optional driver with +`poetry install --with examples`. + +## Authentication + +Each receiver reads: + +- `webhook-id` as the stable Webhook ID and idempotency identifier; +- `webhook-timestamp` as an integer Unix timestamp; +- `webhook-signature` as one or more space-separated versioned signatures. + +The expected `v1` signature is base64-encoded HMAC-SHA256 over the exact byte sequence +`webhook-id.webhook-timestamp.raw_body`. Verification compares signatures with `hmac.compare_digest`. The body is not +parsed or re-serialized before verification. + +Unknown signature versions are ignored so a future FastID rotation can send old and new signatures together. A request +is authenticated when any supplied `v1` signature matches. The quick-start stops at this authentication check. The +advanced examples additionally parse the timestamp as an integer and reject messages outside the tolerance. + +## Payload model + +The examples are event-type agnostic. After signature verification, they require a JSON object with: + +- `event.event_id`: the logical domain event UUID, independent of `webhook-id`; +- `event.event_type`: a non-empty string; +- `event.timestamp`: an integer; +- `data`: a JSON object. + +They log the event ID and event type but do not dispatch business-specific handlers. + +The quick-start keeps payload checks minimal to stay concise. The advanced examples validate the complete generic +envelope before claiming the event. + +## Advanced request flow + +1. Reject a declared `Content-Length` greater than 1 MiB. +2. Read the raw body and reject an actual body greater than 1 MiB. +3. Validate required headers, integer timestamp, timestamp tolerance, secret encoding, and HMAC. +4. Parse and validate the generic JSON envelope. +5. Atomically claim the Webhook ID from `webhook-id`. +6. If already claimed or completed, return `204` without processing it again. +7. Log receipt and mark the claim complete. +8. If processing raises, release the claim and return `500` so FastID retries. + +The claim lifecycle prevents simultaneous duplicate processing in the demonstrated execution model. It does not promise +exactly-once side effects across a crash between the side effect and completion; production applications must make the +business change idempotent or commit an inbox record and business state in one transaction. + +## Error responses + +- `400 Bad Request`: missing or malformed webhook headers, stale timestamp in advanced examples, malformed JSON, or an + invalid event envelope; +- `401 Unauthorized`: a well-formed request with no matching signature; +- `413 Content Too Large`: declared or actual body exceeds 1 MiB; +- `500 Internal Server Error`: processing or idempotency storage fails, allowing FastID to retry; +- `204 No Content`: a new event was accepted or a duplicate was already claimed. + +Error bodies use FastAPI's normal JSON `detail` shape. Error messages do not include secrets, signatures, or raw body +contents. + +## Testing + +Tests import each standalone application with controlled environment variables and exercise it through FastAPI's test +client. Shared test-only helpers generate valid Standard Webhooks signatures; examples do not import those helpers. + +Coverage includes: + +- valid request acceptance for all three applications; +- exact-body signature tampering; +- missing, malformed, and non-matching signatures; +- stale timestamps; +- malformed JSON and invalid envelope fields; +- declared and actual oversized bodies in advanced applications; +- repeated delivery acknowledgement without repeated processing; +- concurrent claims against the in-memory adapter; +- SQLAlchemy duplicate claims and persistence across store instances. +- async SQLAlchemy schema creation, claim, completion, release, and engine disposal without thread-pool calls. + +Ruff, mypy, focused example tests, and the complete project suite must pass. + +## Documentation + +The webhook tutorial will link to the quick-start, advanced, and SQLAlchemy files and describe their intended audiences. +The examples remain source files rather than embedded copies so documentation cannot drift from runnable code. diff --git a/docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md b/docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md new file mode 100644 index 0000000..fe0b5bb --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-webhook-quickstart-timestamp-verification-design.md @@ -0,0 +1,51 @@ +# Webhook Quickstart Timestamp Verification + +## Goal + +Protect the `webhook_quickstart` receiver from replayed webhook requests by +validating the signed `webhook-timestamp` header before accepting the request. + +## Behavior + +- The timestamp header must contain an integer Unix timestamp in seconds. +- The timestamp must be within 300 seconds of the receiver's current time. +- Timestamps more than 300 seconds in the past or future are rejected with + HTTP 400 and `Webhook timestamp is outside the allowed tolerance`. +- Non-integer timestamp values are rejected with HTTP 400 and + `Invalid webhook-timestamp header`. +- Timestamp validation occurs before signature validation. +- The original timestamp header value remains part of the signed message, so + existing valid webhook signatures continue to verify. +- Invalid signatures continue to return HTTP 401. + +## Implementation + +Add a `TIMESTAMP_TOLERANCE_SECONDS = 300` constant and import `time` in +`examples/webhook_quickstart.py`. In `receive_webhook`, parse the required +timestamp header as an integer, reject malformed values, compare it with +`int(time.time())`, and reject timestamps outside the tolerance before calling +`verify_signature`. + +The check stays inline to preserve the quickstart's linear, copyable structure +and mirrors the established behavior in `examples/webhook_advanced.py`. + +## Testing + +Update `tests/examples/test_webhook_quickstart.py` using the existing signed +request helpers: + +- Keep the valid-webhook test as coverage for a current signed timestamp. +- Replace the test that treats timestamps as opaque with a malformed-timestamp + rejection test. +- Add rejection tests for signed timestamps older and newer than the allowed + tolerance. +- Assert HTTP 400 and the relevant error detail for timestamp failures. + +Tests will be written and observed failing before the example is changed, then +the focused example tests and the broader relevant test suite will be run. + +## Scope + +This change affects only the quickstart example and its tests. It does not +change FastID's webhook sender, payload schema, signature format, advanced +receiver, or idempotency behavior. diff --git a/examples/webhook_advanced.py b/examples/webhook_advanced.py new file mode 100644 index 0000000..daabd52 --- /dev/null +++ b/examples/webhook_advanced.py @@ -0,0 +1,235 @@ +import base64 +import binascii +import hashlib +import hmac +import logging +import os +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, HTTPException, Request, Response, status +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field, ValidationError +from sqlalchemy import delete, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +MAX_BODY_BYTES = 1024 * 1024 +TIMESTAMP_TOLERANCE_SECONDS = 300 +DEFAULT_DATABASE_URL = "sqlite+aiosqlite:///webhook-events.sqlite3" +EventProcessor = Callable[["WebhookEnvelope"], Awaitable[None]] + +log = logging.getLogger(__name__) + + +class EventMetadata(PydanticBaseModel): + event_id: UUID + event_type: str = Field(min_length=1) + timestamp: int + + +class WebhookEnvelope(PydanticBaseModel): + event: EventMetadata + data: dict[str, Any] + + +class Base(DeclarativeBase): + pass + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +class WebhookReceipt(Base): + __tablename__ = "webhook_receipts" + + webhook_id: Mapped[str] = mapped_column(primary_key=True) + status: Mapped[str] + created_at: Mapped[datetime] = mapped_column(default=utc_now) + updated_at: Mapped[datetime] = mapped_column(default=utc_now, onupdate=utc_now) + + +def create_store_engine(database_url: str) -> AsyncEngine: + return create_async_engine(database_url) + + +async def create_schema(engine: AsyncEngine) -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + +class SQLAlchemyIdempotencyStore: + def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + self.session_factory = session_factory + + async def claim(self, webhook_id: str) -> bool: + try: + async with self.session_factory() as session, session.begin(): + session.add(WebhookReceipt(webhook_id=webhook_id, status="processing")) + await session.flush() + except IntegrityError: + return False + return True + + async def complete(self, webhook_id: str) -> None: + async with self.session_factory() as session, session.begin(): + await session.execute( + update(WebhookReceipt) + .where(WebhookReceipt.webhook_id == webhook_id) + .values(status="completed", updated_at=utc_now()) + ) + + async def release(self, webhook_id: str) -> None: + async with self.session_factory() as session, session.begin(): + await session.execute( + delete(WebhookReceipt).where( + WebhookReceipt.webhook_id == webhook_id, + WebhookReceipt.status == "processing", + ) + ) + + +def _secret_bytes(secret: str) -> bytes: + if not secret.startswith("whsec_"): + return secret.encode() + try: + return base64.b64decode(secret.removeprefix("whsec_"), validate=True) + except (binascii.Error, ValueError) as exc: + msg = "FASTID_WEBHOOK_SECRET contains invalid base64" + raise ValueError(msg) from exc + + +def verify_signature(body: bytes, webhook_id: str, timestamp: int, signatures: str, secret: str) -> bool: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() + return any( + version == "v1" and hmac.compare_digest(value, expected) + for signature in signatures.split() + if "," in signature + for version, value in (signature.split(",", 1),) + ) + + +def _required_header(request: Request, name: str) -> str: + value = request.headers.get(name) + if not value: + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Missing {name} header") + return value + + +def _content_length(request: Request) -> int | None: + value = request.headers.get("content-length") + if value is None: + return None + try: + return int(value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid Content-Length header") from exc + + +async def process_event(event: WebhookEnvelope) -> None: + log.info( + "Received FastID webhook: event_id=%s event_type=%s", + event.event.event_id, + event.event.event_type, + ) + + +async def _validated_event(request: Request) -> tuple[WebhookEnvelope, str]: + declared_length = _content_length(request) + if declared_length is not None and declared_length > MAX_BODY_BYTES: + raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") + body = await request.body() + if len(body) > MAX_BODY_BYTES: + raise HTTPException(status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, "Webhook body is too large") + + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") + + try: + event = WebhookEnvelope.model_validate_json(body) + except ValidationError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") from exc + return event, webhook_id + + +async def _receive_webhook(request: Request, processor: EventProcessor) -> Response: + event, webhook_id = await _validated_event(request) + store: SQLAlchemyIdempotencyStore = request.app.state.idempotency_store + try: + claimed = await store.claim(webhook_id) + except Exception as exc: + log.exception("Could not claim FastID webhook: webhook_id=%s", webhook_id) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc + if not claimed: + return Response(status_code=status.HTTP_204_NO_CONTENT) + + try: + await processor(event) + await store.complete(webhook_id) + except Exception as exc: + log.exception("FastID webhook processing failed: webhook_id=%s", webhook_id) + try: + await store.release(webhook_id) + except Exception: + log.exception("Could not release FastID webhook claim: webhook_id=%s", webhook_id) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Webhook processing failed") from exc + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def create_app( + database_url: str | None = None, + processor: EventProcessor = process_event, +) -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + secret = os.getenv("FASTID_WEBHOOK_SECRET") + if not secret: + msg = "FASTID_WEBHOOK_SECRET is required" + raise RuntimeError(msg) + try: + _secret_bytes(secret) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + + selected_url = database_url or os.getenv("WEBHOOK_DATABASE_URL") or DEFAULT_DATABASE_URL + engine = create_store_engine(selected_url) + store = SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) + await create_schema(engine) + app.state.webhook_secret = secret + app.state.idempotency_store = store + try: + yield + finally: + await engine.dispose() + + webhook_app = FastAPI(lifespan=lifespan) + + @webhook_app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) + async def receive_webhook(request: Request) -> Response: + return await _receive_webhook(request, processor) + + return webhook_app + + +app = create_app() + + +if __name__ == "__main__": + uvicorn.run("examples.webhook_sqlalchemy:app", host="127.0.0.1", port=8000) diff --git a/examples/webhook_quickstart.py b/examples/webhook_quickstart.py new file mode 100644 index 0000000..d6239dc --- /dev/null +++ b/examples/webhook_quickstart.py @@ -0,0 +1,114 @@ +import base64 +import binascii +import hashlib +import hmac +import json +import logging +import os +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, HTTPException, Request, Response, status + +log = logging.getLogger(__name__) + +TIMESTAMP_TOLERANCE_SECONDS = 300 + + +def _secret_bytes(secret: str) -> bytes: + if not secret.startswith("whsec_"): + return secret.encode() + try: + return base64.b64decode(secret.removeprefix("whsec_"), validate=True) + except (binascii.Error, ValueError) as exc: + msg = "FASTID_WEBHOOK_SECRET contains invalid base64" + raise ValueError(msg) from exc + + +def verify_signature(body: bytes, webhook_id: str, timestamp: str, signatures: str, secret: str) -> bool: + signed = b".".join((webhook_id.encode(), timestamp.encode(), body)) + expected = base64.b64encode(hmac.new(_secret_bytes(secret), signed, hashlib.sha256).digest()).decode() + return any( + version == "v1" and hmac.compare_digest(value, expected) + for signature in signatures.split() + if "," in signature + for version, value in (signature.split(",", 1),) + ) + + +def _required_header(request: Request, name: str) -> str: + value = request.headers.get(name) + if not value: + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Missing {name} header") + return value + + +def _validate_payload(value: Any) -> tuple[str, str]: + if not isinstance(value, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") + event = value.get("event") + data = value.get("data") + if not isinstance(event, dict) or not isinstance(data, dict): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook payload") + event_id = event.get("event_id") + event_type = event.get("event_type") + event_timestamp = event.get("timestamp") + if not isinstance(event_id, str) or not isinstance(event_type, str) or not event_type.strip(): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event") + if type(event_timestamp) is not int: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event") + try: + normalized_event_id = str(UUID(event_id)) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook event ID") from exc + return normalized_event_id, event_type + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + secret = os.getenv("FASTID_WEBHOOK_SECRET") + if not secret: + msg = "FASTID_WEBHOOK_SECRET is required" + raise RuntimeError(msg) + try: + _secret_bytes(secret) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + app.state.webhook_secret = secret + yield + + +app = FastAPI(lifespan=lifespan) + + +@app.post("/fastid-webhooks", status_code=status.HTTP_204_NO_CONTENT) +async def receive_webhook(request: Request) -> Response: + body = await request.body() + webhook_id = _required_header(request, "webhook-id") + timestamp_value = _required_header(request, "webhook-timestamp") + signatures = _required_header(request, "webhook-signature") + try: + timestamp = int(timestamp_value) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid webhook-timestamp header") from exc + if abs(int(time.time()) - timestamp) > TIMESTAMP_TOLERANCE_SECONDS: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Webhook timestamp is outside the allowed tolerance") + if not verify_signature(body, webhook_id, timestamp_value, signatures, request.app.state.webhook_secret): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid webhook signature") + try: + payload = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid JSON body") from exc + event_id, event_type = _validate_payload(payload) + + # Before production side effects, use an advanced receiver that checks freshness and claims webhook-id. + log.info("Received FastID webhook: event_id=%s event_type=%s", event_id, event_type) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +if __name__ == "__main__": + uvicorn.run("examples.webhook_quickstart:app", host="127.0.0.1", port=8000) diff --git a/fastid/admin/factory.py b/fastid/admin/factory.py index 13064ca..f56cba5 100644 --- a/fastid/admin/factory.py +++ b/fastid/admin/factory.py @@ -12,8 +12,9 @@ EmailTemplateAdmin, OAuthProviderAdmin, TelegramTemplateAdmin, - WebhookAdmin, - WebhookEventAdmin, + WebhookAttemptAdmin, + WebhookDeliveryAdmin, + WebhookEndpointAdmin, ) from fastid.admin.views.versioning import ( AppVersionAdmin, @@ -23,7 +24,7 @@ TelegramTemplateVersionAdmin, TransactionAdmin, UserVersionAdmin, - WebhookVersionAdmin, + WebhookEndpointVersionAdmin, ) from fastid.core.base import AppFactory @@ -54,8 +55,9 @@ def create(self) -> FastAPI: # Settings admin.add_view(AppAdmin) admin.add_view(OAuthProviderAdmin) - admin.add_view(WebhookAdmin) - admin.add_view(WebhookEventAdmin) + admin.add_view(WebhookEndpointAdmin) + admin.add_view(WebhookDeliveryAdmin) + admin.add_view(WebhookAttemptAdmin) admin.add_view(EmailTemplateAdmin) admin.add_view(TelegramTemplateAdmin) # Versioning @@ -64,7 +66,7 @@ def create(self) -> FastAPI: admin.add_view(OAuthAccountVersionAdmin) admin.add_view(AppVersionAdmin) admin.add_view(OAuthProviderVersionAdmin) - admin.add_view(WebhookVersionAdmin) + admin.add_view(WebhookEndpointVersionAdmin) admin.add_view(EmailTemplateVersionAdmin) admin.add_view(TelegramTemplateVersionAdmin) return app diff --git a/fastid/admin/views/settings.py b/fastid/admin/views/settings.py index 9187cff..becf4c4 100644 --- a/fastid/admin/views/settings.py +++ b/fastid/admin/views/settings.py @@ -2,7 +2,15 @@ from fastid.admin.views.base import BaseView from fastid.admin.views.utils import json_format -from fastid.database.models import App, EmailTemplate, OAuthProvider, TelegramTemplate, Webhook, WebhookEvent +from fastid.database.models import ( + App, + EmailTemplate, + OAuthProvider, + TelegramTemplate, + WebhookAttempt, + WebhookDelivery, + WebhookEndpoint, +) class AppAdmin(BaseView, model=App): @@ -89,48 +97,78 @@ class TelegramTemplateAdmin(BaseView, model=TelegramTemplate): ] -class WebhookAdmin(BaseView, model=Webhook): - name = "Webhook" - name_plural = "Webhooks" +class WebhookEndpointAdmin(BaseView, model=WebhookEndpoint): + name = "Webhook Endpoint" + name_plural = "Webhook Endpoints" icon = "fa-solid fa-globe" category = "Settings" column_list = [ - Webhook.id, - Webhook.app, - Webhook.url, - Webhook.created_at, - Webhook.updated_at, + WebhookEndpoint.id, + WebhookEndpoint.app, + WebhookEndpoint.url, + WebhookEndpoint.is_active, + WebhookEndpoint.disabled_reason, + WebhookEndpoint.created_at, + WebhookEndpoint.updated_at, ] column_filters = [ - OperationColumnFilter(Webhook.app_id), - AllUniqueStringValuesFilter(Webhook.type), - OperationColumnFilter(Webhook.url), + OperationColumnFilter(WebhookEndpoint.app_id), + AllUniqueStringValuesFilter(WebhookEndpoint.type), + OperationColumnFilter(WebhookEndpoint.url), + BooleanFilter(WebhookEndpoint.is_active), ] -class WebhookEventAdmin(BaseView, model=WebhookEvent): +class WebhookDeliveryAdmin(BaseView, model=WebhookDelivery): can_create = False can_edit = False can_delete = False - name = "Webhook Event" - name_plural = "Webhook Events" + name = "Webhook Delivery" + name_plural = "Webhook Deliveries" icon = "fa-solid fa-book-atlas" category = "Settings" column_list = [ - WebhookEvent.id, - WebhookEvent.webhook, - "webhook.type", - WebhookEvent.status_code, - WebhookEvent.request, - WebhookEvent.response, - WebhookEvent.created_at, - WebhookEvent.updated_at, + WebhookDelivery.id, + WebhookDelivery.event_id, + WebhookDelivery.endpoint, + "endpoint.type", + WebhookDelivery.status, + WebhookDelivery.attempt_count, + WebhookDelivery.status_code, + WebhookDelivery.response, + WebhookDelivery.next_attempt_at, + WebhookDelivery.completed_at, + WebhookDelivery.created_at, ] column_filters = [ - OperationColumnFilter(WebhookEvent.webhook_id), - OperationColumnFilter(WebhookEvent.status_code), + OperationColumnFilter(WebhookDelivery.endpoint_id), + OperationColumnFilter(WebhookDelivery.status_code), + AllUniqueStringValuesFilter(WebhookDelivery.status), ] - column_formatters = {WebhookEvent.response: json_format, WebhookEvent.request: json_format} + column_formatters = {WebhookDelivery.response: json_format, WebhookDelivery.request: json_format} + + +class WebhookAttemptAdmin(BaseView, model=WebhookAttempt): + can_create = False + can_edit = False + can_delete = False + + name = "Webhook Attempt" + name_plural = "Webhook Attempts" + icon = "fa-solid fa-clock-rotate-left" + category = "Settings" + + column_list = [ + WebhookAttempt.id, + WebhookAttempt.delivery, + WebhookAttempt.attempt_number, + WebhookAttempt.status_code, + WebhookAttempt.error, + WebhookAttempt.duration_ms, + WebhookAttempt.created_at, + ] + column_filters = [OperationColumnFilter(WebhookAttempt.delivery_id)] + column_formatters = {WebhookAttempt.response: json_format, WebhookAttempt.request: json_format} diff --git a/fastid/admin/views/versioning.py b/fastid/admin/views/versioning.py index 42f6e42..fa32282 100644 --- a/fastid/admin/views/versioning.py +++ b/fastid/admin/views/versioning.py @@ -10,7 +10,7 @@ TelegramTemplateVersion, Transaction, UserVersion, - WebhookVersion, + WebhookEndpointVersion, ) @@ -93,6 +93,6 @@ class TelegramTemplateVersionAdmin(BaseVersionView, model=TelegramTemplateVersio name_plural = "Telegram Template Versions" -class WebhookVersionAdmin(BaseVersionView, model=WebhookVersion): - name = "Webhook Version" - name_plural = "Webhook Versions" +class WebhookEndpointVersionAdmin(BaseVersionView, model=WebhookEndpointVersion): + name = "Webhook Endpoint Version" + name_plural = "Webhook Endpoint Versions" diff --git a/fastid/apps/models.py b/fastid/apps/models.py index 3fe668f..a20d1ad 100644 --- a/fastid/apps/models.py +++ b/fastid/apps/models.py @@ -5,7 +5,7 @@ from fastid.apps.exceptions import InvalidClientCredentialsError, InvalidRedirectURIError from fastid.database.base import VersionedEntity from fastid.database.utils import uuid_hex -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint class App(VersionedEntity): @@ -18,7 +18,7 @@ class App(VersionedEntity): redirect_uris: Mapped[str] = mapped_column(default="") is_active: Mapped[bool] = mapped_column(default=True) - webhooks: Mapped[list[Webhook]] = relationship(back_populates="app", cascade="delete") + webhook_endpoints: Mapped[list[WebhookEndpoint]] = relationship(back_populates="app", cascade="delete") def verify_redirect_uri(self, redirect_uri: str) -> None: if redirect_uri not in self.redirect_uris.split(";"): diff --git a/fastid/auth/router.py b/fastid/auth/router.py index 8bc9409..9721a9b 100644 --- a/fastid/auth/router.py +++ b/fastid/auth/router.py @@ -43,10 +43,9 @@ async def register( user = await service.register(dto) notification = PushNotificationRequest(template="welcome") background.add_task(notify.push, user, notification) # pragma: nocover - webhook = SendWebhookRequest( - type=WebhookType.user_registration, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_registration, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user # pragma: nocover @@ -59,7 +58,6 @@ async def login( form: Annotated[OAuth2PasswordRequest, Form()], password_grant: Annotated[PasswordGrant, Depends()], webhooks: WebhooksDep, - background: BackgroundTasks, ) -> Any: match form.grant_type: case OAuth2Grant.password: @@ -68,7 +66,7 @@ async def login( raise NotSupportedGrantError assert auth_data.user is not None webhook = SendWebhookRequest(type=WebhookType.user_login, payload={"user": auth_data.user.model_dump(mode="json")}) - background.add_task(webhooks.send, webhook) # pragma: nocover + await webhooks.enqueue(webhook) return cookie_transport.get_login_response(auth_data.token) @@ -84,7 +82,6 @@ async def authorize( # noqa: PLR0913 refresh_token_grant: Annotated[RefreshTokenGrant, Depends()], client_credentials_grant: Annotated[ClientCredentialsGrant, Depends()], webhooks: WebhooksDep, - background: BackgroundTasks, ) -> Any: match form.grant_type: case OAuth2Grant.password: @@ -101,7 +98,7 @@ async def authorize( # noqa: PLR0913 webhook = SendWebhookRequest( type=WebhookType.user_login, payload={"user": auth_data.user.model_dump(mode="json")} ) - background.add_task(webhooks.send, webhook) # pragma: nocover + await webhooks.enqueue(webhook) return auth_data.token diff --git a/fastid/auth/use_cases.py b/fastid/auth/use_cases.py index 25628dc..3e178de 100644 --- a/fastid/auth/use_cases.py +++ b/fastid/auth/use_cases.py @@ -25,7 +25,7 @@ async def register(self, dto: UserCreate) -> User: raise UserAlreadyExistsError user = User.from_create(dto) user = await self.uow.users.add(user) - await self.uow.commit() + await self.uow.flush() return user async def get_userinfo(self, token: str, *, token_type: str = "access") -> User: # noqa: S107 diff --git a/fastid/core/lifespan.py b/fastid/core/lifespan.py index a21811d..72c9725 100644 --- a/fastid/core/lifespan.py +++ b/fastid/core/lifespan.py @@ -13,7 +13,6 @@ from fastid.notify.utils import collect_email_templates, collect_telegram_templates from fastid.oauth.models import OAUTH_PROVIDER_NAMES, OAuthProvider from fastid.oauth.repositories import OAuthProviderNameSpecification -from fastid.webhooks.senders.dependencies import client as webhooks_client class LifespanTasks: @@ -69,7 +68,6 @@ async def create_oauth_providers(self) -> None: async def on_shutdown(self) -> None: await self.cache.client.aclose() - await webhooks_client.aclose() async def __aenter__(self) -> Self: await self.uow.__aenter__() diff --git a/fastid/database/models.py b/fastid/database/models.py index e977e0a..324e50c 100644 --- a/fastid/database/models.py +++ b/fastid/database/models.py @@ -6,7 +6,7 @@ from fastid.database.base import BaseOrm from fastid.notify.models import EmailTemplate, Notification, TelegramTemplate from fastid.oauth.models import OAuthAccount, OAuthProvider -from fastid.webhooks.models import Webhook, WebhookEvent +from fastid.webhooks.models import WebhookAttempt, WebhookDelivery, WebhookEndpoint configure_mappers() @@ -19,7 +19,7 @@ TelegramTemplateVersion, Transaction, UserVersion, - WebhookVersion, + WebhookEndpointVersion, ) __all__ = [ @@ -36,9 +36,10 @@ "EmailTemplateVersion", "TelegramTemplateVersion", "Transaction", - "WebhookVersion", - "Webhook", - "WebhookEvent", + "WebhookEndpointVersion", + "WebhookEndpoint", + "WebhookAttempt", + "WebhookDelivery", "OAuthAccountVersion", "OAuthProviderVersion", ] diff --git a/fastid/database/uow.py b/fastid/database/uow.py index 3df0eb7..56fb371 100644 --- a/fastid/database/uow.py +++ b/fastid/database/uow.py @@ -9,7 +9,7 @@ from fastid.auth.repositories import UserRepository from fastid.notify.repositories import EmailTemplateRepository, NotificationRepository, TelegramTemplateRepository from fastid.oauth.repositories import OAuthAccountRepository, OAuthProviderRepository -from fastid.webhooks.repositories import WebhookEventRepository, WebhookRepository +from fastid.webhooks.repositories import WebhookAttemptRepository, WebhookDeliveryRepository, WebhookEndpointRepository if TYPE_CHECKING: from types import TracebackType @@ -28,8 +28,9 @@ class SQLAlchemyUOW: email_templates: EmailTemplateRepository telegram_templates: TelegramTemplateRepository notifications: NotificationRepository - webhooks: WebhookRepository - webhook_events: WebhookEventRepository + webhook_endpoints: WebhookEndpointRepository + webhook_deliveries: WebhookDeliveryRepository + webhook_attempts: WebhookAttemptRepository session_factory: async_sessionmaker[AsyncSession] session: AsyncSession @@ -46,8 +47,9 @@ async def begin(self) -> None: self.email_templates = EmailTemplateRepository(self.session) self.telegram_templates = TelegramTemplateRepository(self.session) self.notifications = NotificationRepository(self.session) - self.webhooks = WebhookRepository(self.session) - self.webhook_events = WebhookEventRepository(self.session) + self.webhook_endpoints = WebhookEndpointRepository(self.session) + self.webhook_deliveries = WebhookDeliveryRepository(self.session) + self.webhook_attempts = WebhookAttemptRepository(self.session) @property def is_active(self) -> bool: diff --git a/fastid/database/versioning.py b/fastid/database/versioning.py index 5868dcb..c0a8bba 100644 --- a/fastid/database/versioning.py +++ b/fastid/database/versioning.py @@ -4,7 +4,7 @@ from fastid.auth.models import User from fastid.notify.models import EmailTemplate, TelegramTemplate from fastid.oauth.models import OAuthAccount, OAuthProvider -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint Transaction = transaction_class(User) @@ -14,4 +14,4 @@ OAuthProviderVersion = version_class(OAuthProvider) EmailTemplateVersion = version_class(EmailTemplate) TelegramTemplateVersion = version_class(TelegramTemplate) -WebhookVersion = version_class(Webhook) +WebhookEndpointVersion = version_class(WebhookEndpoint) diff --git a/fastid/profile/router.py b/fastid/profile/router.py index 8ad2eff..7e5825a 100644 --- a/fastid/profile/router.py +++ b/fastid/profile/router.py @@ -1,6 +1,6 @@ from typing import Any -from fastapi import APIRouter, BackgroundTasks, status +from fastapi import APIRouter, status from fastid.auth.dependencies import UserDep, UserVTDep from fastid.auth.schemas import ( @@ -19,14 +19,11 @@ @router.patch("/users/me/profile", response_model=UserDTO, status_code=status.HTTP_200_OK) -async def patch( - service: ProfilesDep, user: UserDep, dto: UserUpdate, webhooks: WebhooksDep, background: BackgroundTasks -) -> Any: +async def patch(service: ProfilesDep, user: UserDep, dto: UserUpdate, webhooks: WebhooksDep) -> Any: user = await service.update_profile(user, dto) - webhook = SendWebhookRequest( - type=WebhookType.user_update_profile, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_update_profile, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user @@ -35,20 +32,18 @@ async def patch( response_model=UserDTO, status_code=status.HTTP_200_OK, ) -async def change_email( # noqa: PLR0913 +async def change_email( service: ProfilesDep, notify_service: NotifyDep, user: UserVTDep, dto: UserChangeEmail, webhooks: WebhooksDep, - background: BackgroundTasks, ) -> Any: await notify_service.validate_otp(user, dto.code) user = await service.change_email(user, dto) - webhook = SendWebhookRequest( - type=WebhookType.user_change_email, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_change_email, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user @@ -57,14 +52,11 @@ async def change_email( # noqa: PLR0913 response_model=UserDTO, status_code=status.HTTP_200_OK, ) -async def change_password( - service: ProfilesDep, user: UserVTDep, dto: UserChangePassword, webhooks: WebhooksDep, background: BackgroundTasks -) -> Any: +async def change_password(service: ProfilesDep, user: UserVTDep, dto: UserChangePassword, webhooks: WebhooksDep) -> Any: user = await service.change_password(user, dto) - webhook = SendWebhookRequest( - type=WebhookType.user_change_password, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_change_password, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user @@ -73,10 +65,9 @@ async def change_password( response_model=UserDTO, status_code=status.HTTP_200_OK, ) -async def delete(service: ProfilesDep, user: UserVTDep, webhooks: WebhooksDep, background: BackgroundTasks) -> Any: +async def delete(service: ProfilesDep, user: UserVTDep, webhooks: WebhooksDep) -> Any: user = await service.delete_account(user) - webhook = SendWebhookRequest( - type=WebhookType.user_delete, payload=UserDTO.model_validate(user).model_dump(mode="json") - ) - background.add_task(webhooks.send, webhook) # pragma: nocover + user_data = UserDTO.model_validate(user).model_dump(mode="json") + webhook = SendWebhookRequest(type=WebhookType.user_delete, payload=user_data | {"user": user_data}) + await webhooks.enqueue(webhook) return user diff --git a/fastid/profile/use_cases.py b/fastid/profile/use_cases.py index 9517fc2..e504ba7 100644 --- a/fastid/profile/use_cases.py +++ b/fastid/profile/use_cases.py @@ -32,7 +32,7 @@ async def update_profile( dto: UserUpdate, ) -> User: user.merge_model(dto) - await self.uow.commit() + await self.uow.flush() return user async def change_email(self, user: User, dto: UserChangeEmail) -> User: @@ -43,17 +43,17 @@ async def change_email(self, user: User, dto: UserChangeEmail) -> User: else: raise UserAlreadyExistsError user.change_email(dto.new_email) - await self.uow.commit() + await self.uow.flush() return user async def change_password(self, user: User, dto: UserChangePassword) -> User: user.set_password(dto.password) - await self.uow.commit() + await self.uow.flush() return user async def delete_account(self, user: User) -> User: user = await self.uow.users.delete(user) - await self.uow.commit() + await self.uow.flush() return user async def get_many(self, pagination: Pagination, sorting: Sorting) -> Page[User]: diff --git a/fastid/security/webhooks.py b/fastid/security/webhooks.py index 06de90c..7352c05 100644 --- a/fastid/security/webhooks.py +++ b/fastid/security/webhooks.py @@ -1,72 +1,71 @@ +import base64 import hashlib import hmac import json import time +from collections.abc import Mapping from typing import Any from fastid.database.utils import UUIDv7, uuid from fastid.webhooks.config import webhook_settings -from fastid.webhooks.schemas import SignatureAlgorithm +from fastid.webhooks.models import generate_webhook_secret -HASH_FUNCTIONS = { - SignatureAlgorithm.sha256: hashlib.sha256, - SignatureAlgorithm.sha512: hashlib.sha512, - SignatureAlgorithm.sha1: hashlib.sha1, -} +STANDARD_ID_HEADER = "webhook-id" +STANDARD_TIMESTAMP_HEADER = "webhook-timestamp" +STANDARD_SIGNATURE_HEADER = "webhook-signature" -def generate_headers( - payload: dict[str, Any], - timestamp: int, - webhook_id: str, - secret_key: str, - *, - algorithm: SignatureAlgorithm = webhook_settings.signature_algorithm, -) -> dict[str, str]: - signature = generate_signature(payload, webhook_id, timestamp, secret_key, algorithm=algorithm) +def serialize_payload(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def generate_secret() -> str: + return generate_webhook_secret() + + +def _secret_bytes(secret_key: str) -> bytes: + if not secret_key.startswith("whsec_"): + return secret_key.encode() + try: + return base64.b64decode(secret_key.removeprefix("whsec_"), validate=True) + except ValueError as exc: + msg = "Invalid whsec_ webhook secret" + raise ValueError(msg) from exc + + +def generate_standard_signature(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> str: + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + digest = hmac.new(_secret_bytes(secret_key), signed, hashlib.sha256).digest() + return f"v1,{base64.b64encode(digest).decode()}" + + +def generate_delivery_headers(body: bytes, webhook_id: str, timestamp: int, secret_key: str) -> dict[str, str]: return { - webhook_settings.id_header: webhook_id, - webhook_settings.timestamp_header: str(timestamp), - webhook_settings.signature_header: signature, + STANDARD_ID_HEADER: webhook_id, + STANDARD_TIMESTAMP_HEADER: str(timestamp), + STANDARD_SIGNATURE_HEADER: generate_standard_signature(body, webhook_id, timestamp, secret_key), + "Content-Type": "application/json", + "User-Agent": webhook_settings.user_agent, } -def generate_signature( - payload: dict[str, Any], - webhook_id: str, - timestamp: int, - secret_key: str, - *, - algorithm: SignatureAlgorithm = webhook_settings.signature_algorithm, -) -> str: - payload_str = json.dumps(payload, sort_keys=True, separators=(",", ":")) - payload_str = f"{webhook_id}.{timestamp}.{payload_str}" - hash_func = HASH_FUNCTIONS[algorithm] - hmac_obj = hmac.new(key=secret_key.encode(), msg=payload_str.encode(), digestmod=hash_func) - return hmac_obj.hexdigest() - - -def verify_headers( - payload: dict[str, Any], - headers: dict[str, str], +def verify_standard_headers( + body: bytes, + headers: Mapping[str, str], secret_key: str, tolerance_seconds: int = webhook_settings.tolerance_seconds, ) -> bool: + normalized = {key.lower(): value for key, value in headers.items()} try: - timestamp = int(headers[webhook_settings.timestamp_header]) - webhook_id = headers[webhook_settings.id_header] - received_signature = headers[webhook_settings.signature_header] - except (KeyError, ValueError): # pragma: nocover + timestamp = int(normalized[STANDARD_TIMESTAMP_HEADER]) + webhook_id = normalized[STANDARD_ID_HEADER] + signatures = normalized[STANDARD_SIGNATURE_HEADER].split() + except (KeyError, ValueError): return False - - if not all([timestamp, webhook_id, received_signature]): + if not webhook_id or not signatures or not is_timestamp_valid(timestamp, tolerance_seconds): return False - - if not is_timestamp_valid(timestamp, tolerance_seconds): - return False - - expected_signature = generate_signature(payload, webhook_id, timestamp, secret_key) - return hmac.compare_digest(received_signature, expected_signature) + expected = generate_standard_signature(body, webhook_id, timestamp, secret_key) + return any(hmac.compare_digest(signature, expected) for signature in signatures) def get_event_id() -> UUIDv7: diff --git a/fastid/webhooks/config.py b/fastid/webhooks/config.py index c49fa7e..4ced759 100644 --- a/fastid/webhooks/config.py +++ b/fastid/webhooks/config.py @@ -1,16 +1,28 @@ from pydantic_settings import SettingsConfigDict from fastid.core.schemas import ENV_PREFIX, BaseSettings -from fastid.webhooks.schemas import SignatureAlgorithm class WebhookSettings(BaseSettings): - signature_algorithm: SignatureAlgorithm = SignatureAlgorithm.sha256 - signature_header: str = "X-Webhook-Signature" - timestamp_header: str = "X-Webhook-Timestamp" - id_header: str = "X-Webhook-Id" tolerance_seconds: int = 300 page_expires_in_seconds: int = 60 + retry_delays_seconds: tuple[int, ...] = (0, 5, 300, 1800, 7200, 18000, 36000, 50400, 72000, 86400) + retry_jitter_ratio: float = 0.2 + retry_after_max_seconds: int = 86400 + worker_batch_size: int = 100 + worker_concurrency: int = 20 + worker_poll_seconds: float = 1.0 + worker_lease_seconds: int = 60 + worker_metrics_port: int = 9101 + connect_timeout_seconds: float = 5.0 + read_timeout_seconds: float = 20.0 + write_timeout_seconds: float = 5.0 + pool_timeout_seconds: float = 5.0 + max_connections: int = 100 + max_keepalive_connections: int = 20 + max_response_bytes: int = 65536 + allow_insecure_urls: bool = False + user_agent: str = "FastID-Webhooks/0.1" model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX}webhook_") diff --git a/fastid/webhooks/metrics.py b/fastid/webhooks/metrics.py new file mode 100644 index 0000000..e972b6c --- /dev/null +++ b/fastid/webhooks/metrics.py @@ -0,0 +1,21 @@ +from prometheus_client import Counter, Gauge, Histogram + +ATTEMPTS = Counter( + "fastid_webhook_attempts_total", + "Webhook delivery attempts by event type and outcome.", + ["event_type", "outcome"], +) +ATTEMPT_DURATION = Histogram( + "fastid_webhook_attempt_duration_seconds", + "Webhook delivery attempt duration.", + ["event_type"], +) +DUE_DELIVERIES = Gauge( + "fastid_webhook_due_deliveries", + "Webhook deliveries currently due for processing.", +) +DISABLED_ENDPOINTS = Counter( + "fastid_webhook_disabled_endpoints_total", + "Webhook endpoints disabled by reason.", + ["reason"], +) diff --git a/fastid/webhooks/models.py b/fastid/webhooks/models.py index a6f5185..2073f14 100644 --- a/fastid/webhooks/models.py +++ b/fastid/webhooks/models.py @@ -1,10 +1,13 @@ from __future__ import annotations +import base64 +import datetime # noqa: TCH003 - SQLAlchemy resolves mapped annotations at runtime +import secrets from enum import auto from typing import TYPE_CHECKING, Any from uuid import UUID # noqa: TCH003 -from sqlalchemy import ForeignKey +from sqlalchemy import ForeignKey, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from fastid.core.schemas import BaseEnum @@ -14,6 +17,10 @@ from fastid.apps.models import App +def generate_webhook_secret() -> str: + return f"whsec_{base64.b64encode(secrets.token_bytes(32)).decode()}" + + class WebhookType(BaseEnum): user_registration = auto() user_login = auto() @@ -23,24 +30,61 @@ class WebhookType(BaseEnum): user_delete = auto() -class Webhook(VersionedEntity): - __tablename__ = "webhooks" +class WebhookDeliveryStatus(BaseEnum): + pending = auto() + processing = auto() + succeeded = auto() + exhausted = auto() + cancelled = auto() + + +class WebhookEndpoint(VersionedEntity): + __tablename__ = "webhook_endpoints" app_id: Mapped[UUID] = mapped_column(ForeignKey("apps.id"), index=True) type: Mapped[WebhookType] - secret: Mapped[str] + secret: Mapped[str] = mapped_column(default=generate_webhook_secret) url: Mapped[str] + is_active: Mapped[bool] = mapped_column(default=True) + disabled_at: Mapped[datetime.datetime | None] + disabled_reason: Mapped[str | None] + + app: Mapped[App] = relationship(back_populates="webhook_endpoints") + deliveries: Mapped[list[WebhookDelivery]] = relationship(back_populates="endpoint", cascade="delete") + + +class WebhookDelivery(Entity): + __tablename__ = "webhook_deliveries" + + endpoint_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_endpoints.id"), index=True) + event_id: Mapped[UUID] = mapped_column(index=True) + event_type: Mapped[WebhookType] + payload: Mapped[dict[str, Any]] + status: Mapped[WebhookDeliveryStatus] = mapped_column(default=WebhookDeliveryStatus.pending, index=True) + attempt_count: Mapped[int] = mapped_column(default=0) + next_attempt_at: Mapped[datetime.datetime] = mapped_column(index=True) + leased_until: Mapped[datetime.datetime | None] = mapped_column(index=True) + completed_at: Mapped[datetime.datetime | None] + request: Mapped[dict[str, Any] | None] + status_code: Mapped[int | None] + response: Mapped[dict[str, Any] | None] + error: Mapped[str | None] - app: Mapped[App] = relationship(back_populates="webhooks") - webhook_events: Mapped[list[WebhookEvent]] = relationship(back_populates="webhook", cascade="delete") + endpoint: Mapped[WebhookEndpoint] = relationship(back_populates="deliveries") + attempts: Mapped[list[WebhookAttempt]] = relationship(back_populates="delivery", cascade="delete") -class WebhookEvent(Entity): - __tablename__ = "webhook_events" +class WebhookAttempt(Entity): + __tablename__ = "webhook_attempts" + __table_args__ = (UniqueConstraint("delivery_id", "attempt_number"),) - webhook_id: Mapped[UUID] = mapped_column(ForeignKey("webhooks.id"), index=True) + delivery_id: Mapped[UUID] = mapped_column(ForeignKey("webhook_deliveries.id"), index=True) + attempt_number: Mapped[int] + timestamp: Mapped[int] request: Mapped[dict[str, Any]] - status_code: Mapped[int] - response: Mapped[dict[str, Any]] + status_code: Mapped[int | None] + response: Mapped[dict[str, Any] | None] + error: Mapped[str | None] + duration_ms: Mapped[int] - webhook: Mapped[Webhook] = relationship(back_populates="webhook_events") + delivery: Mapped[WebhookDelivery] = relationship(back_populates="attempts") diff --git a/fastid/webhooks/repositories.py b/fastid/webhooks/repositories.py index 2986275..bf7e030 100644 --- a/fastid/webhooks/repositories.py +++ b/fastid/webhooks/repositories.py @@ -3,28 +3,32 @@ from fastid.database.repository import SQLAlchemyRepository from fastid.database.specification import Specification from fastid.database.utils import UUIDv7 -from fastid.webhooks.models import Webhook, WebhookEvent, WebhookType +from fastid.webhooks.models import WebhookAttempt, WebhookDelivery, WebhookEndpoint, WebhookType -class WebhookRepository(SQLAlchemyRepository[Webhook]): - model_type = Webhook +class WebhookEndpointRepository(SQLAlchemyRepository[WebhookEndpoint]): + model_type = WebhookEndpoint -class WebhookTypeSpecification(Specification): +class WebhookEndpointTypeSpecification(Specification): def __init__(self, webhook_type: WebhookType) -> None: self.type = webhook_type def apply(self, stmt: Any) -> Any: - return stmt.where(Webhook.type == self.type) + return stmt.where(WebhookEndpoint.type == self.type, WebhookEndpoint.is_active.is_(True)) -class WebhookEventRepository(SQLAlchemyRepository[WebhookEvent]): - model_type = WebhookEvent +class WebhookDeliveryRepository(SQLAlchemyRepository[WebhookDelivery]): + model_type = WebhookDelivery -class WebhookEventWebhookIDSpecification(Specification): - def __init__(self, webhook_id: UUIDv7) -> None: - self.webhook_id = webhook_id +class WebhookAttemptRepository(SQLAlchemyRepository[WebhookAttempt]): + model_type = WebhookAttempt + + +class WebhookDeliveryEndpointIDSpecification(Specification): + def __init__(self, endpoint_id: UUIDv7) -> None: + self.endpoint_id = endpoint_id def apply(self, stmt: Any) -> Any: - return stmt.where(WebhookEvent.webhook_id == self.webhook_id) + return stmt.where(WebhookDelivery.endpoint_id == self.endpoint_id) diff --git a/fastid/webhooks/schemas.py b/fastid/webhooks/schemas.py index 91cf953..60f6862 100644 --- a/fastid/webhooks/schemas.py +++ b/fastid/webhooks/schemas.py @@ -1,20 +1,13 @@ -from enum import auto from typing import Any from uuid import UUID from pydantic import Field from fastid.auth.schemas import UserDTO -from fastid.core.schemas import BaseEnum, BaseModel +from fastid.core.schemas import BaseModel from fastid.webhooks.models import WebhookType -class SignatureAlgorithm(BaseEnum): - sha256 = auto() - sha512 = auto() - sha1 = auto() - - class SendWebhookRequest(BaseModel): type: WebhookType payload: dict[str, Any] = Field(default_factory=dict) diff --git a/fastid/webhooks/senders/dependencies.py b/fastid/webhooks/senders/dependencies.py index 964a445..342c109 100644 --- a/fastid/webhooks/senders/dependencies.py +++ b/fastid/webhooks/senders/dependencies.py @@ -3,9 +3,20 @@ import httpx from fastapi import Depends +from fastid.webhooks.config import webhook_settings from fastid.webhooks.senders.httpx import WebhookSender -client = httpx.AsyncClient() +timeout = httpx.Timeout( + webhook_settings.read_timeout_seconds, + connect=webhook_settings.connect_timeout_seconds, + write=webhook_settings.write_timeout_seconds, + pool=webhook_settings.pool_timeout_seconds, +) +limits = httpx.Limits( + max_connections=webhook_settings.max_connections, + max_keepalive_connections=webhook_settings.max_keepalive_connections, +) +client = httpx.AsyncClient(timeout=timeout, limits=limits, follow_redirects=False, trust_env=False) def get_sender() -> WebhookSender: diff --git a/fastid/webhooks/senders/httpx.py b/fastid/webhooks/senders/httpx.py index 5afb33a..6d0e30c 100644 --- a/fastid/webhooks/senders/httpx.py +++ b/fastid/webhooks/senders/httpx.py @@ -1,19 +1,163 @@ -from typing import Any +import asyncio +import ipaddress +import json +import socket +import time +from dataclasses import dataclass +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime import httpx +from fastid.webhooks.config import webhook_settings + +HTTPS_PORT = 443 + + +class UnsafeWebhookURLError(ValueError): + pass + + +@dataclass(frozen=True) +class ResolvedTarget: + url: httpx.URL + host_header: str + sni_hostname: str | None + + +@dataclass(frozen=True) +class WebhookResponse: + status_code: int + content: dict[str, object] + error: str | None + retry_after_seconds: int | None + duration_ms: int + + +async def resolve_targets(value: str) -> list[ResolvedTarget]: + url = httpx.URL(value) + if not url.host: + msg = "Webhook URL must include a host" + raise UnsafeWebhookURLError(msg) + if url.username or url.password: + msg = "Webhook URL must not include credentials" + raise UnsafeWebhookURLError(msg) + if webhook_settings.allow_insecure_urls: + return [ResolvedTarget(url=url, host_header=_host_header(url), sni_hostname=url.host)] + if url.scheme != "https": + msg = "Webhook URL must use HTTPS" + raise UnsafeWebhookURLError(msg) + + port = url.port or HTTPS_PORT + loop = asyncio.get_running_loop() + try: + records = await loop.getaddrinfo(url.host, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + msg = f"Webhook host could not be resolved: {exc}" + raise UnsafeWebhookURLError(msg) from exc + addresses = list(dict.fromkeys(record[4][0].split("%", 1)[0] for record in records)) + if not addresses: + msg = "Webhook host did not resolve to an address" + raise UnsafeWebhookURLError(msg) + parsed = [ipaddress.ip_address(address) for address in addresses] + if any(not address.is_global for address in parsed): + msg = "Webhook host resolves to a non-public address" + raise UnsafeWebhookURLError(msg) + return [ + ResolvedTarget(url=url.copy_with(host=str(address)), host_header=_host_header(url), sni_hostname=url.host) + for address in parsed + ] + + +def _host_header(url: httpx.URL) -> str: + host = url.host + if ":" in host and not host.startswith("["): + host = f"[{host}]" + if url.port and url.port != HTTPS_PORT: + return f"{host}:{url.port}" + return host + + +def _retry_after(value: str | None) -> int | None: + if value is None: + return None + try: + return max(0, int(value)) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=UTC) + return max(0, int((retry_at - datetime.now(UTC)).total_seconds())) + + +def _content(raw: bytes, *, truncated: bool) -> dict[str, object]: + text = raw.decode(errors="replace") + try: + value = json.loads(text) if text else None + except json.JSONDecodeError: + value = text + content: dict[str, object] = value if isinstance(value, dict) else {"value": value} + if truncated: + content["truncated"] = True + return content + class WebhookSender: def __init__(self, client: httpx.AsyncClient) -> None: self.client = client - async def send(self, url: str, payload: dict[str, Any], headers: dict[str, str]) -> dict[str, Any]: - try: - response = await self.client.post(url, json=payload, headers=headers) - except httpx.RequestError as e: - return {"status_code": 0, "content": {"error": str(e)}} + async def send(self, url: str, body: bytes, headers: dict[str, str]) -> WebhookResponse: + started = time.monotonic() try: - content = response.json() - except Exception: # noqa: BLE001 - content = {"text": response.text} - return {"status_code": response.status_code, "content": content} + targets = await resolve_targets(url) + except UnsafeWebhookURLError as exc: + return self._error(exc, started) + + last_error: httpx.RequestError | None = None + for target in targets: + request_headers = headers | {"Host": target.host_header} + extensions = {"sni_hostname": target.sni_hostname} if target.sni_hostname else None + request = self.client.build_request( + "POST", target.url, content=body, headers=request_headers, extensions=extensions + ) + try: + response = await self.client.send(request, stream=True, follow_redirects=False) + except httpx.RequestError as exc: + last_error = exc + continue + try: + raw = bytearray() + truncated = False + async for chunk in response.aiter_bytes(): + remaining = webhook_settings.max_response_bytes - len(raw) + if remaining <= 0: + truncated = True + break + raw.extend(chunk[:remaining]) + if len(chunk) > remaining: + truncated = True + break + finally: + await response.aclose() + return WebhookResponse( + status_code=response.status_code, + content=_content(bytes(raw), truncated=truncated), + error=None, + retry_after_seconds=_retry_after(response.headers.get("Retry-After")), + duration_ms=int((time.monotonic() - started) * 1000), + ) + assert last_error is not None + return self._error(last_error, started) + + @staticmethod + def _error(exc: Exception, started: float) -> WebhookResponse: + return WebhookResponse( + status_code=0, + content={"error": str(exc)}, + error=str(exc), + retry_after_seconds=None, + duration_ms=int((time.monotonic() - started) * 1000), + ) diff --git a/fastid/webhooks/use_cases.py b/fastid/webhooks/use_cases.py index 5a77f4d..523cb8e 100644 --- a/fastid/webhooks/use_cases.py +++ b/fastid/webhooks/use_cases.py @@ -1,56 +1,35 @@ -from fastid.cache.dependencies import CacheDep -from fastid.cache.exceptions import KeyNotFoundError from fastid.core.base import UseCase -from fastid.database.dependencies import UOWRawDep, transactional -from fastid.security.webhooks import ( - generate_headers, - get_event_id, - get_timestamp, - get_webhook_id, -) -from fastid.webhooks.config import webhook_settings -from fastid.webhooks.models import Webhook, WebhookEvent -from fastid.webhooks.repositories import WebhookTypeSpecification +from fastid.database.dependencies import UOWDep +from fastid.database.utils import naive_utc +from fastid.security.webhooks import get_event_id, get_timestamp, get_webhook_id +from fastid.webhooks.models import WebhookDelivery, WebhookDeliveryStatus +from fastid.webhooks.repositories import WebhookEndpointTypeSpecification from fastid.webhooks.schemas import Event, SendWebhookRequest, WebhookPayload -from fastid.webhooks.senders.dependencies import SenderDep class WebhookUseCases(UseCase): - def __init__(self, uow: UOWRawDep, cache: CacheDep, sender: SenderDep) -> None: - self.uow = uow # Due to background nature of notification use cases, use raw dependency - self.cache = cache - self.sender = sender + def __init__(self, uow: UOWDep) -> None: + self.uow = uow - @transactional - async def send(self, dto: SendWebhookRequest) -> None: - cache_key = f"webhooks:{dto.type}" - try: - cached = await self.cache.get(cache_key) - except KeyNotFoundError: - webhook_page = await self.uow.webhooks.get_many(WebhookTypeSpecification(dto.type)) - webhooks = webhook_page.items - await self.cache.set( - cache_key, - [{"id": str(w.id), "secret": w.secret, "url": w.url} for w in webhooks], - expire=webhook_settings.page_expires_in_seconds, - ) - else: - webhooks = [Webhook(**w) for w in cached] + async def enqueue(self, dto: SendWebhookRequest) -> None: + endpoint_page = await self.uow.webhook_endpoints.get_many(WebhookEndpointTypeSpecification(dto.type)) event_id = get_event_id() event_timestamp = get_timestamp() event_dto = Event(event_type=dto.type, event_id=event_id, timestamp=event_timestamp) payload = WebhookPayload(event=event_dto, data=dto.payload).model_dump(mode="json") - for webhook in webhooks: - webhook_id = get_webhook_id() - timestamp = get_timestamp() - headers = generate_headers(payload, timestamp, str(webhook_id), webhook.secret) - request = {"headers": headers, "body": payload} - data = await self.sender.send(webhook.url, payload, headers) - event = WebhookEvent( - id=webhook_id, - webhook_id=webhook.id, - request=request, - status_code=data["status_code"], - response=data["content"], + now = naive_utc() + for endpoint in endpoint_page.items: + delivery = WebhookDelivery( + id=get_webhook_id(), + endpoint_id=endpoint.id, + event_id=event_id, + event_type=dto.type, + payload=payload, + status=WebhookDeliveryStatus.pending, + next_attempt_at=now, ) - await self.uow.webhook_events.add(event) + await self.uow.webhook_deliveries.add(delivery) + + async def send(self, dto: SendWebhookRequest) -> None: + """Backward-compatible alias for enqueueing a durable delivery.""" + await self.enqueue(dto) diff --git a/fastid/webhooks/worker.py b/fastid/webhooks/worker.py new file mode 100644 index 0000000..74daf2e --- /dev/null +++ b/fastid/webhooks/worker.py @@ -0,0 +1,246 @@ +import asyncio +import logging +import secrets +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timedelta +from uuid import UUID + +from prometheus_client import start_http_server +from sqlalchemy import func, or_, select, update +from sqlalchemy.orm import joinedload + +from fastid.database.dependencies import get_uow_raw +from fastid.database.uow import SQLAlchemyUOW +from fastid.database.utils import naive_utc +from fastid.security.webhooks import generate_delivery_headers, get_timestamp, serialize_payload +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.metrics import ATTEMPT_DURATION, ATTEMPTS, DISABLED_ENDPOINTS, DUE_DELIVERIES +from fastid.webhooks.models import WebhookAttempt, WebhookDelivery, WebhookDeliveryStatus, WebhookEndpoint +from fastid.webhooks.senders.dependencies import client +from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender + +log = logging.getLogger(__name__) +HTTP_OK = 200 +HTTP_MULTIPLE_CHOICES = 300 +HTTP_GONE = 410 +RANDOM = secrets.SystemRandom() + + +def get_retry_delay(attempt_number: int, retry_after_seconds: int | None, *, jitter: float | None = None) -> int: + delay = webhook_settings.retry_delays_seconds[attempt_number] + if jitter is None: + jitter = RANDOM.uniform(1 - webhook_settings.retry_jitter_ratio, 1 + webhook_settings.retry_jitter_ratio) + delay = max(0, int(delay * jitter)) + if retry_after_seconds is not None: + retry_after = min(retry_after_seconds, webhook_settings.retry_after_max_seconds) + delay = max(delay, retry_after) + return delay + + +@dataclass(frozen=True) +class ClaimedDelivery: + webhook_id: UUID + event_id: UUID + event_type: str + payload: dict[str, object] + endpoint_url: str + endpoint_secret: str + + +class WebhookWorker: + def __init__(self, sender: WebhookSender | None = None) -> None: + self.sender = sender or WebhookSender(client) + self.stopping = asyncio.Event() + self.semaphore = asyncio.Semaphore(webhook_settings.worker_concurrency) + + async def run(self) -> None: + log.info("Webhook worker started") + while not self.stopping.is_set(): + count = await self.run_once() + if count == 0: + with suppress(TimeoutError): + await asyncio.wait_for(self.stopping.wait(), timeout=webhook_settings.worker_poll_seconds) + log.info("Webhook worker stopped") + + async def run_once(self) -> int: + deliveries = await self._claim() + await asyncio.gather(*(self._process_with_limit(delivery) for delivery in deliveries)) + return len(deliveries) + + async def _claim(self) -> list[ClaimedDelivery]: + now = naive_utc() + lease = now + timedelta(seconds=webhook_settings.worker_lease_seconds) + uow = get_uow_raw() + async with uow: + stmt = ( + select(WebhookDelivery) + .options(joinedload(WebhookDelivery.endpoint)) + .where( + WebhookDelivery.next_attempt_at <= now, + or_( + WebhookDelivery.status == WebhookDeliveryStatus.pending, + (WebhookDelivery.status == WebhookDeliveryStatus.processing) + & (WebhookDelivery.leased_until <= now), + ), + ) + .order_by(WebhookDelivery.next_attempt_at, WebhookDelivery.created_at) + .limit(webhook_settings.worker_batch_size) + .with_for_update(skip_locked=True, of=WebhookDelivery) + ) + rows = list((await uow.session.scalars(stmt)).all()) + due_count = await uow.session.scalar( + select(func.count()) + .select_from(WebhookDelivery) + .where( + WebhookDelivery.next_attempt_at <= now, + or_( + WebhookDelivery.status == WebhookDeliveryStatus.pending, + (WebhookDelivery.status == WebhookDeliveryStatus.processing) + & (WebhookDelivery.leased_until <= now), + ), + ) + ) + DUE_DELIVERIES.set(due_count or 0) + claimed: list[ClaimedDelivery] = [] + for delivery in rows: + if not delivery.endpoint.is_active: + delivery.status = WebhookDeliveryStatus.cancelled + delivery.completed_at = now + delivery.error = "endpoint disabled" + continue + delivery.status = WebhookDeliveryStatus.processing + delivery.leased_until = lease + claimed.append( + ClaimedDelivery( + webhook_id=delivery.id, + event_id=delivery.event_id, + event_type=str(delivery.event_type), + payload=delivery.payload, + endpoint_url=delivery.endpoint.url, + endpoint_secret=delivery.endpoint.secret, + ) + ) + return claimed + return [] # pragma: no cover - the unit of work context always enters + + async def _process_with_limit(self, delivery: ClaimedDelivery) -> None: + async with self.semaphore: + await self._process(delivery) + + async def _process(self, delivery: ClaimedDelivery) -> None: + timestamp = get_timestamp() + body = serialize_payload(delivery.payload) + headers = generate_delivery_headers( + body, + str(delivery.webhook_id), + timestamp, + delivery.endpoint_secret, + ) + response = await self.sender.send(delivery.endpoint_url, body, headers) + await self._record(delivery.webhook_id, delivery.event_type, timestamp, headers, response) + + async def _record( + self, + webhook_id: UUID, + event_type: str, + timestamp: int, + headers: dict[str, str], + response: WebhookResponse, + ) -> None: + now = naive_utc() + uow = get_uow_raw() + async with uow: + delivery = await uow.webhook_deliveries.get(webhook_id) + endpoint = await uow.webhook_endpoints.get(delivery.endpoint_id) + attempt_number = delivery.attempt_count + 1 + stored_headers = { + key: "[redacted]" if "signature" in key.lower() else value for key, value in headers.items() + } + request = {"headers": stored_headers, "body": delivery.payload} + attempt = WebhookAttempt( + delivery_id=delivery.id, + attempt_number=attempt_number, + timestamp=timestamp, + request=request, + status_code=response.status_code, + response=response.content, + error=response.error, + duration_ms=response.duration_ms, + ) + await uow.webhook_attempts.add(attempt) + delivery.attempt_count = attempt_number + delivery.request = request + delivery.status_code = response.status_code + delivery.response = response.content + delivery.error = response.error + delivery.leased_until = None + + if HTTP_OK <= response.status_code < HTTP_MULTIPLE_CHOICES: + delivery.status = WebhookDeliveryStatus.succeeded + delivery.completed_at = now + outcome = "succeeded" + elif response.status_code == HTTP_GONE: + await self._disable(uow, endpoint, delivery, now, "endpoint returned 410 Gone") + outcome = "gone" + DISABLED_ENDPOINTS.labels(reason="gone").inc() + elif attempt_number >= len(webhook_settings.retry_delays_seconds): + await self._disable(uow, endpoint, delivery, now, "delivery retries exhausted") + outcome = "exhausted" + DISABLED_ENDPOINTS.labels(reason="exhausted").inc() + else: + delay = get_retry_delay(attempt_number, response.retry_after_seconds) + delivery.status = WebhookDeliveryStatus.pending + delivery.next_attempt_at = now + timedelta(seconds=delay) + outcome = "retry" + ATTEMPTS.labels(event_type=event_type, outcome=outcome).inc() + ATTEMPT_DURATION.labels(event_type=event_type).observe(response.duration_ms / 1000) + log.info( + "Webhook attempt completed: webhook_id=%s event_id=%s attempt=%d status_code=%d state=%s duration_ms=%d", + delivery.id, + delivery.event_id, + attempt_number, + response.status_code, + delivery.status, + response.duration_ms, + ) + + @staticmethod + async def _disable( + uow: SQLAlchemyUOW, endpoint: WebhookEndpoint, delivery: WebhookDelivery, now: datetime, reason: str + ) -> None: + delivery.status = ( + WebhookDeliveryStatus.exhausted if delivery.status_code != HTTP_GONE else WebhookDeliveryStatus.cancelled + ) + delivery.completed_at = now + endpoint.is_active = False + endpoint.disabled_at = now + endpoint.disabled_reason = reason + await uow.session.execute( + update(WebhookDelivery) + .where( + WebhookDelivery.endpoint_id == endpoint.id, + WebhookDelivery.id != delivery.id, + WebhookDelivery.status.in_((WebhookDeliveryStatus.pending, WebhookDeliveryStatus.processing)), + ) + .values(status=WebhookDeliveryStatus.cancelled, completed_at=now, error="endpoint disabled") + ) + + +async def run_worker() -> None: + if webhook_settings.worker_metrics_port > 0: + start_http_server(webhook_settings.worker_metrics_port) + worker = WebhookWorker() + try: + await worker.run() + finally: + await client.aclose() + + +def main() -> None: + with suppress(KeyboardInterrupt): + asyncio.run(run_worker()) + + +if __name__ == "__main__": + main() diff --git a/migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py b/migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py new file mode 100644 index 0000000..bd30671 --- /dev/null +++ b/migrations/versions/2026_07_17_1700-7f3c0d2f18a1_durable_webhooks.py @@ -0,0 +1,154 @@ +"""durable webhooks + +Revision ID: 7f3c0d2f18a1 +Revises: bfa61bae871d +Create Date: 2026-07-17 17:00:00 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "7f3c0d2f18a1" +down_revision: str | None = "bfa61bae871d" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +webhook_type = sa.Enum( + "user_registration", + "user_login", + "user_update_profile", + "user_change_email", + "user_change_password", + "user_delete", + name="webhooktype", + native_enum=False, +) +delivery_status = sa.Enum( + "pending", + "processing", + "succeeded", + "exhausted", + "cancelled", + name="webhookdeliverystatus", + native_enum=False, +) + + +def upgrade() -> None: + op.add_column("webhooks", sa.Column("is_active", sa.Boolean(), server_default=sa.true(), nullable=False)) + op.add_column("webhooks", sa.Column("disabled_at", sa.DateTime(), nullable=True)) + op.add_column("webhooks", sa.Column("disabled_reason", sa.String(), nullable=True)) + op.add_column("webhooks_version", sa.Column("is_active", sa.Boolean(), nullable=True)) + op.add_column("webhooks_version", sa.Column("disabled_at", sa.DateTime(), nullable=True)) + op.add_column("webhooks_version", sa.Column("disabled_reason", sa.String(), nullable=True)) + + op.add_column("webhook_events", sa.Column("event_id", sa.Uuid(), nullable=True)) + op.add_column("webhook_events", sa.Column("event_type", webhook_type, nullable=True)) + op.add_column("webhook_events", sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column("webhook_events", sa.Column("status", delivery_status, nullable=True)) + op.add_column("webhook_events", sa.Column("attempt_count", sa.BigInteger(), server_default="0", nullable=False)) + op.add_column("webhook_events", sa.Column("next_attempt_at", sa.DateTime(), nullable=True)) + op.add_column("webhook_events", sa.Column("leased_until", sa.DateTime(), nullable=True)) + op.add_column("webhook_events", sa.Column("completed_at", sa.DateTime(), nullable=True)) + op.add_column("webhook_events", sa.Column("error", sa.String(), nullable=True)) + op.alter_column("webhook_events", "request", existing_type=postgresql.JSONB(), nullable=True) + op.alter_column("webhook_events", "status_code", existing_type=sa.BigInteger(), nullable=True) + op.alter_column("webhook_events", "response", existing_type=postgresql.JSONB(), nullable=True) + + op.execute( + """ + UPDATE webhook_events AS delivery + SET event_id = delivery.id, + event_type = endpoint.type, + payload = COALESCE(delivery.request -> 'body', '{}'::jsonb), + status = CASE + WHEN delivery.status_code BETWEEN 200 AND 299 THEN 'succeeded' + ELSE 'exhausted' + END, + attempt_count = 1, + next_attempt_at = delivery.created_at, + completed_at = delivery.updated_at, + error = CASE WHEN delivery.status_code = 0 THEN delivery.response ->> 'error' ELSE NULL END + FROM webhooks AS endpoint + WHERE endpoint.id = delivery.webhook_id + """ + ) + op.alter_column("webhook_events", "event_id", nullable=False) + op.alter_column("webhook_events", "event_type", nullable=False) + op.alter_column("webhook_events", "payload", nullable=False) + op.alter_column("webhook_events", "status", nullable=False) + op.alter_column("webhook_events", "next_attempt_at", nullable=False) + + op.create_index(op.f("webhook_events_event_id_idx"), "webhook_events", ["event_id"]) + op.create_index(op.f("webhook_events_status_idx"), "webhook_events", ["status"]) + op.create_index(op.f("webhook_events_next_attempt_at_idx"), "webhook_events", ["next_attempt_at"]) + op.create_index(op.f("webhook_events_leased_until_idx"), "webhook_events", ["leased_until"]) + + op.create_table( + "webhook_attempts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("delivery_id", sa.Uuid(), nullable=False), + sa.Column("attempt_number", sa.BigInteger(), nullable=False), + sa.Column("timestamp", sa.BigInteger(), nullable=False), + sa.Column("request", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("status_code", sa.BigInteger(), nullable=True), + sa.Column("response", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("error", sa.String(), nullable=True), + sa.Column("duration_ms", sa.BigInteger(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["delivery_id"], ["webhook_events.id"], name=op.f("webhook_attempts_delivery_id_fkey")), + sa.PrimaryKeyConstraint("id", name=op.f("webhook_attempts_pkey")), + sa.UniqueConstraint("delivery_id", "attempt_number", name=op.f("webhook_attempts_delivery_id_key")), + ) + op.create_index(op.f("webhook_attempts_delivery_id_idx"), "webhook_attempts", ["delivery_id"]) + op.execute( + """ + INSERT INTO webhook_attempts ( + id, delivery_id, attempt_number, timestamp, request, status_code, + response, error, duration_ms, created_at, updated_at + ) + SELECT id, id, 1, EXTRACT(EPOCH FROM created_at)::bigint, request, + status_code, response, error, 0, created_at, updated_at + FROM webhook_events + """ + ) + + +def downgrade() -> None: + op.drop_index(op.f("webhook_attempts_delivery_id_idx"), table_name="webhook_attempts") + op.drop_table("webhook_attempts") + op.execute( + """ + UPDATE webhook_events + SET request = COALESCE(request, jsonb_build_object('headers', '{}'::jsonb, 'body', payload)), + status_code = COALESCE(status_code, 0), + response = COALESCE(response, jsonb_build_object('error', COALESCE(error, 'not delivered'))) + """ + ) + op.alter_column("webhook_events", "response", existing_type=postgresql.JSONB(), nullable=False) + op.alter_column("webhook_events", "status_code", existing_type=sa.BigInteger(), nullable=False) + op.alter_column("webhook_events", "request", existing_type=postgresql.JSONB(), nullable=False) + op.drop_index(op.f("webhook_events_leased_until_idx"), table_name="webhook_events") + op.drop_index(op.f("webhook_events_next_attempt_at_idx"), table_name="webhook_events") + op.drop_index(op.f("webhook_events_status_idx"), table_name="webhook_events") + op.drop_index(op.f("webhook_events_event_id_idx"), table_name="webhook_events") + for column in ( + "error", + "completed_at", + "leased_until", + "next_attempt_at", + "attempt_count", + "status", + "payload", + "event_type", + "event_id", + ): + op.drop_column("webhook_events", column) + for table in ("webhooks_version", "webhooks"): + op.drop_column(table, "disabled_reason") + op.drop_column(table, "disabled_at") + op.drop_column(table, "is_active") diff --git a/migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py b/migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py new file mode 100644 index 0000000..af940e9 --- /dev/null +++ b/migrations/versions/2026_07_17_1800-a4e1d2c3b5f6_rename_webhook_events_to_deliveries.py @@ -0,0 +1,39 @@ +"""rename webhook events to deliveries + +Revision ID: a4e1d2c3b5f6 +Revises: 7f3c0d2f18a1 +Create Date: 2026-07-17 18:00:00 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "a4e1d2c3b5f6" +down_revision: str | None = "7f3c0d2f18a1" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +OLD_TABLE = "webhook_events" +NEW_TABLE = "webhook_deliveries" +INDEX_COLUMNS = ("webhook_id", "event_id", "status", "next_attempt_at", "leased_until") + + +def upgrade() -> None: + op.rename_table(OLD_TABLE, NEW_TABLE) + op.execute(f"ALTER TABLE {NEW_TABLE} RENAME CONSTRAINT {OLD_TABLE}_pkey TO {NEW_TABLE}_pkey") + op.execute( + f"ALTER TABLE {NEW_TABLE} " f"RENAME CONSTRAINT {OLD_TABLE}_webhook_id_fkey TO {NEW_TABLE}_webhook_id_fkey" + ) + for column in INDEX_COLUMNS: + op.execute(f"ALTER INDEX {OLD_TABLE}_{column}_idx RENAME TO {NEW_TABLE}_{column}_idx") + + +def downgrade() -> None: + op.rename_table(NEW_TABLE, OLD_TABLE) + op.execute(f"ALTER TABLE {OLD_TABLE} RENAME CONSTRAINT {NEW_TABLE}_pkey TO {OLD_TABLE}_pkey") + op.execute( + f"ALTER TABLE {OLD_TABLE} " f"RENAME CONSTRAINT {NEW_TABLE}_webhook_id_fkey TO {OLD_TABLE}_webhook_id_fkey" + ) + for column in INDEX_COLUMNS: + op.execute(f"ALTER INDEX {NEW_TABLE}_{column}_idx RENAME TO {OLD_TABLE}_{column}_idx") diff --git a/migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py b/migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py new file mode 100644 index 0000000..7aea5ba --- /dev/null +++ b/migrations/versions/2026_07_20_1200-5b2c8d7e9f10_rename_webhooks_to_endpoints.py @@ -0,0 +1,59 @@ +"""rename webhooks to endpoints + +Revision ID: 5b2c8d7e9f10 +Revises: a4e1d2c3b5f6 +Create Date: 2026-07-20 12:00:00 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "5b2c8d7e9f10" +down_revision: str | None = "a4e1d2c3b5f6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +UPGRADE_RENAMES = ( + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_pkey TO webhook_endpoints_pkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhooks_app_id_fkey TO webhook_endpoints_app_id_fkey", + "ALTER INDEX webhooks_app_id_idx RENAME TO webhook_endpoints_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhooks_version_pkey TO webhook_endpoints_version_pkey", + "ALTER INDEX webhooks_version_app_id_idx RENAME TO webhook_endpoints_version_app_id_idx", + "ALTER INDEX webhooks_version_end_transaction_id_idx RENAME TO webhook_endpoints_version_end_transaction_id_idx", + "ALTER INDEX webhooks_version_operation_type_idx RENAME TO webhook_endpoints_version_operation_type_idx", + "ALTER INDEX webhooks_version_transaction_id_idx RENAME TO webhook_endpoints_version_transaction_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_webhook_id_fkey " + "TO webhook_deliveries_endpoint_id_fkey", + "ALTER INDEX webhook_deliveries_webhook_id_idx RENAME TO webhook_deliveries_endpoint_id_idx", +) + +DOWNGRADE_RENAMES = ( + "ALTER INDEX webhook_deliveries_endpoint_id_idx RENAME TO webhook_deliveries_webhook_id_idx", + "ALTER TABLE webhook_deliveries RENAME CONSTRAINT webhook_deliveries_endpoint_id_fkey " + "TO webhook_deliveries_webhook_id_fkey", + "ALTER INDEX webhook_endpoints_version_transaction_id_idx RENAME TO webhooks_version_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_operation_type_idx RENAME TO webhooks_version_operation_type_idx", + "ALTER INDEX webhook_endpoints_version_end_transaction_id_idx RENAME TO webhooks_version_end_transaction_id_idx", + "ALTER INDEX webhook_endpoints_version_app_id_idx RENAME TO webhooks_version_app_id_idx", + "ALTER TABLE webhook_endpoints_version RENAME CONSTRAINT webhook_endpoints_version_pkey TO webhooks_version_pkey", + "ALTER INDEX webhook_endpoints_app_id_idx RENAME TO webhooks_app_id_idx", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_app_id_fkey TO webhooks_app_id_fkey", + "ALTER TABLE webhook_endpoints RENAME CONSTRAINT webhook_endpoints_pkey TO webhooks_pkey", +) + + +def upgrade() -> None: + op.rename_table("webhooks", "webhook_endpoints") + op.rename_table("webhooks_version", "webhook_endpoints_version") + op.alter_column("webhook_deliveries", "webhook_id", new_column_name="endpoint_id") + for statement in UPGRADE_RENAMES: + op.execute(statement) + + +def downgrade() -> None: + for statement in DOWNGRADE_RENAMES: + op.execute(statement) + op.alter_column("webhook_deliveries", "endpoint_id", new_column_name="webhook_id") + op.rename_table("webhook_endpoints_version", "webhooks_version") + op.rename_table("webhook_endpoints", "webhooks") diff --git a/poetry.lock b/poetry.lock index 0158551..750e947 100644 --- a/poetry.lock +++ b/poetry.lock @@ -370,6 +370,25 @@ files = [ frozenlist = ">=1.1.0" typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} +[[package]] +name = "aiosqlite" +version = "0.21.0" +description = "asyncio bridge to the standard sqlite3 module" +optional = false +python-versions = ">=3.9" +groups = ["examples"] +files = [ + {file = "aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0"}, + {file = "aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3"}, +] + +[package.dependencies] +typing_extensions = ">=4.0" + +[package.extras] +dev = ["attribution (==1.7.1)", "black (==24.3.0)", "build (>=1.2)", "coverage[toml] (==7.6.10)", "flake8 (==7.0.0)", "flake8-bugbear (==24.12.12)", "flit (==3.10.1)", "mypy (==1.14.1)", "ufmt (==2.5.1)", "usort (==1.0.8.post1)"] +docs = ["sphinx (==8.1.3)", "sphinx-mdinclude (==0.6.1)"] + [[package]] name = "alembic" version = "1.18.0" @@ -4251,7 +4270,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "examples"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -5111,4 +5130,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "b1791a972d4445d49ce544f1c02ade34f45b590888d4c6e2d13b46f04ea078cf" +content-hash = "2306b9aa5f13f9d437b892c20eef3debb146485b53e2cf0778067ea592a3e673" diff --git a/pyproject.toml b/pyproject.toml index 16d867d..c5b8884 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,13 @@ mdx-include = "^1.4.2" mkdocs-markdownextradata-plugin = "^0.2.6" markdown-include-variants = "^0.0.4" + +[tool.poetry.group.examples] +optional = true + +[tool.poetry.group.examples.dependencies] +aiosqlite = "^0.21.0" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/tests/api/auth/test_authorize_password_grant.py b/tests/api/auth/test_authorize_password_grant.py index 4a2b4dc..160a86e 100644 --- a/tests/api/auth/test_authorize_password_grant.py +++ b/tests/api/auth/test_authorize_password_grant.py @@ -7,8 +7,8 @@ from fastid.core.config import core_settings from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests import mocks from tests.mocks import faker from tests.utils.auth import ( @@ -17,7 +17,7 @@ async def test_authorize_password_grant( - client: AsyncClient, user: UserDTO, webhook_login: Webhook, uow: SQLAlchemyUOW + client: AsyncClient, user: UserDTO, webhook_login: WebhookEndpoint, uow: SQLAlchemyUOW ) -> None: assert user.email is not None token = await authorize_password_grant(client, user.email, mocks.USER_CREATE.password) @@ -25,9 +25,9 @@ async def test_authorize_password_grant( assert token.refresh_token is not None try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_login.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_login.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_dynamic_server_url_is_used_as_jwt_issuer( diff --git a/tests/api/auth/test_register.py b/tests/api/auth/test_register.py index af047cc..4ff5236 100644 --- a/tests/api/auth/test_register.py +++ b/tests/api/auth/test_register.py @@ -5,12 +5,12 @@ from fastid.auth.schemas import UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import USER_CREATE -async def test_register(client: AsyncClient, webhook_registration: Webhook, uow: SQLAlchemyUOW) -> None: +async def test_register(client: AsyncClient, webhook_registration: WebhookEndpoint, uow: SQLAlchemyUOW) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) assert response.status_code == status.HTTP_201_CREATED user = UserDTO.model_validate_json(response.content) @@ -20,9 +20,9 @@ async def test_register(client: AsyncClient, webhook_registration: Webhook, uow: assert user.email == USER_CREATE.email try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_registration.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_registration.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_register_existent(client: AsyncClient, user: UserDTO) -> None: diff --git a/tests/api/conftest.py b/tests/api/conftest.py index ec3d949..e4323b8 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -14,7 +14,7 @@ from fastid.database.uow import SQLAlchemyUOW from fastid.notify.schemas import UserAction from fastid.security.crypto import generate_otp -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint from tests import mocks from tests.dependencies import get_test_cache, get_test_uow from tests.utils.auth import authorize_password_grant, register_user @@ -107,35 +107,35 @@ async def oauth_app(client: AsyncClient, user_su_token: TokenResponse) -> AppDTO @pytest.fixture -async def webhook_registration(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_registration(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_REGISTRATION_RECORD) @pytest.fixture -async def webhook_login(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_login(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_LOGIN_RECORD) @pytest.fixture -async def webhook_delete(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_delete(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_DELETE_RECORD) @pytest.fixture -async def webhook_profile_update(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_profile_update(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_UPDATE_PROFILE_RECORD) @pytest.fixture -async def webhook_change_password(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_change_password(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_CHANGE_PASSWORD) @pytest.fixture -async def webhook_change_email(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_change_email(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_CHANGE_EMAIL) @pytest.fixture -async def webhook_wrong_url(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> Webhook: +async def webhook_wrong_url(uow: SQLAlchemyUOW, oauth_app: AppDTO) -> WebhookEndpoint: return await create_webhook(uow, oauth_app, mocks.WEBHOOK_WRONG_URL) diff --git a/tests/api/profile/test_delete_user.py b/tests/api/profile/test_delete_user.py index 00367fa..6294782 100644 --- a/tests/api/profile/test_delete_user.py +++ b/tests/api/profile/test_delete_user.py @@ -6,8 +6,8 @@ from fastid.auth.schemas import TokenResponse, UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification async def test_delete_user( # noqa: PLR0913 @@ -15,7 +15,7 @@ async def test_delete_user( # noqa: PLR0913 user: UserDTO, user_token: TokenResponse, verify_token: str, - webhook_delete: Webhook, + webhook_delete: WebhookEndpoint, uow: SQLAlchemyUOW, ) -> None: client.cookies.set(vt_transport.name, verify_token) @@ -28,9 +28,9 @@ async def test_delete_user( # noqa: PLR0913 UserDTO.model_validate_json(response.content) try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_delete.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_delete.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_delete_user_not_verified(client: AsyncClient, user: UserDTO, user_token: TokenResponse) -> None: diff --git a/tests/api/profile/test_update_user_email.py b/tests/api/profile/test_update_user_email.py index 64d1645..8ffff07 100644 --- a/tests/api/profile/test_update_user_email.py +++ b/tests/api/profile/test_update_user_email.py @@ -8,8 +8,8 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW from fastid.security.crypto import generate_otp -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import faker @@ -19,7 +19,7 @@ async def test_update_user_email( # noqa: PLR0913 user: UserDTO, user_token: TokenResponse, verify_token: str, - webhook_change_email: Webhook, + webhook_change_email: WebhookEndpoint, uow: SQLAlchemyUOW, ) -> None: code = generate_otp() @@ -38,9 +38,9 @@ async def test_update_user_email( # noqa: PLR0913 assert user.email == new_email try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_change_email.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_change_email.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_update_user_email_already_exists( # noqa: PLR0913 diff --git a/tests/api/profile/test_update_user_password.py b/tests/api/profile/test_update_user_password.py index 922770a..69fba5b 100644 --- a/tests/api/profile/test_update_user_password.py +++ b/tests/api/profile/test_update_user_password.py @@ -6,8 +6,8 @@ from fastid.auth.schemas import TokenResponse, UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import faker @@ -16,7 +16,7 @@ async def test_update_user_password( # noqa: PLR0913 user: UserDTO, user_token: TokenResponse, verify_token: str, - webhook_change_password: Webhook, + webhook_change_password: WebhookEndpoint, uow: SQLAlchemyUOW, ) -> None: new_password = faker.password() @@ -31,9 +31,9 @@ async def test_update_user_password( # noqa: PLR0913 UserDTO.model_validate_json(response.content) try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_change_password.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_change_password.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") async def test_update_user_password_not_verified(client: AsyncClient, user: UserDTO, user_token: TokenResponse) -> None: diff --git a/tests/api/profile/test_update_user_profile.py b/tests/api/profile/test_update_user_profile.py index 47fa3e2..7af6f67 100644 --- a/tests/api/profile/test_update_user_profile.py +++ b/tests/api/profile/test_update_user_profile.py @@ -5,13 +5,17 @@ from fastid.auth.schemas import TokenResponse, UserDTO from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests import mocks async def test_update_user_profile( - client: AsyncClient, user: UserDTO, user_token: TokenResponse, webhook_profile_update: Webhook, uow: SQLAlchemyUOW + client: AsyncClient, + user: UserDTO, + user_token: TokenResponse, + webhook_profile_update: WebhookEndpoint, + uow: SQLAlchemyUOW, ) -> None: response = await client.patch( "/users/me/profile", @@ -24,6 +28,6 @@ async def test_update_user_profile( assert user.last_name == mocks.USER_UPDATE.last_name try: - await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_profile_update.id)) + await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_profile_update.id)) except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") diff --git a/tests/api/webhooks/test_worker_delivery.py b/tests/api/webhooks/test_worker_delivery.py new file mode 100644 index 0000000..6b353bf --- /dev/null +++ b/tests/api/webhooks/test_worker_delivery.py @@ -0,0 +1,111 @@ +import asyncio +from datetime import timedelta + +import pytest +from httpx import AsyncClient +from starlette import status + +import fastid.webhooks.worker as worker_module +from fastid.database.uow import SQLAlchemyUOW +from fastid.database.utils import naive_utc +from fastid.security.webhooks import verify_standard_headers +from fastid.webhooks.models import WebhookDeliveryStatus, WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification +from fastid.webhooks.senders.httpx import WebhookResponse, WebhookSender +from fastid.webhooks.worker import WebhookWorker +from tests.dependencies import get_test_uow +from tests.mocks import USER_CREATE + +HTTP_MULTIPLE_CHOICES = 300 + + +class StubSender(WebhookSender): + def __init__(self, status_code: int) -> None: + self.status_code = status_code + self.calls: list[tuple[bytes, dict[str, str]]] = [] + + async def send(self, url: str, body: bytes, headers: dict[str, str]) -> WebhookResponse: + assert url + self.calls.append((body, headers)) + return WebhookResponse( + status_code=self.status_code, + content={"accepted": self.status_code < HTTP_MULTIPLE_CHOICES}, + error=None, + retry_after_seconds=None, + duration_ms=2, + ) + + +@pytest.mark.parametrize( + ("status_code", "expected_status", "active"), + [ + (204, WebhookDeliveryStatus.succeeded, True), + (410, WebhookDeliveryStatus.cancelled, False), + (500, WebhookDeliveryStatus.pending, True), + ], +) +async def test_worker_records_delivery_outcome( # noqa: PLR0913 + client: AsyncClient, + webhook_registration: WebhookEndpoint, + uow: SQLAlchemyUOW, + monkeypatch: pytest.MonkeyPatch, + status_code: int, + expected_status: WebhookDeliveryStatus, + *, + active: bool, +) -> None: + response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) + assert response.status_code == status.HTTP_201_CREATED + monkeypatch.setattr(worker_module, "get_uow_raw", get_test_uow) + sender = StubSender(status_code) + + assert await WebhookWorker(sender=sender).run_once() == 1 + + delivery = await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_registration.id)) + assert delivery.status == expected_status + assert delivery.attempt_count == 1 + assert delivery.status_code == status_code + attempts = (await uow.webhook_attempts.get_many()).items + assert len(attempts) == 1 + body, headers = sender.calls[0] + assert headers["webhook-id"] == str(delivery.id) + assert headers["webhook-id"] != str(delivery.event_id) + assert verify_standard_headers(body, headers, webhook_registration.secret) + await uow.session.refresh(webhook_registration) + assert webhook_registration.is_active is active + + +async def test_workers_do_not_claim_the_same_delivery( + client: AsyncClient, + webhook_registration: WebhookEndpoint, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) + assert response.status_code == status.HTTP_201_CREATED + monkeypatch.setattr(worker_module, "get_uow_raw", get_test_uow) + senders = [StubSender(204), StubSender(204)] + + counts = await asyncio.gather(*(WebhookWorker(sender=sender).run_once() for sender in senders)) + + assert sum(counts) == 1 + assert sum(len(sender.calls) for sender in senders) == 1 + + +async def test_worker_recovers_an_expired_lease( + client: AsyncClient, + webhook_registration: WebhookEndpoint, + uow: SQLAlchemyUOW, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) + assert response.status_code == status.HTTP_201_CREATED + delivery = await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_registration.id)) + delivery.status = WebhookDeliveryStatus.processing + delivery.leased_until = naive_utc() - timedelta(seconds=1) + await uow.commit() + monkeypatch.setattr(worker_module, "get_uow_raw", get_test_uow) + + assert await WebhookWorker(sender=StubSender(204)).run_once() == 1 + + await uow.session.refresh(delivery) + assert delivery.status == WebhookDeliveryStatus.succeeded diff --git a/tests/api/webhooks/test_wrong_url.py b/tests/api/webhooks/test_wrong_url.py index a712634..75e0ac4 100644 --- a/tests/api/webhooks/test_wrong_url.py +++ b/tests/api/webhooks/test_wrong_url.py @@ -4,17 +4,19 @@ from fastid.database.exceptions import NoResultFoundError from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook -from fastid.webhooks.repositories import WebhookEventWebhookIDSpecification +from fastid.webhooks.models import WebhookDeliveryStatus, WebhookEndpoint +from fastid.webhooks.repositories import WebhookDeliveryEndpointIDSpecification from tests.mocks import USER_CREATE -async def test_wrong_url(client: AsyncClient, webhook_wrong_url: Webhook, uow: SQLAlchemyUOW) -> None: +async def test_wrong_url(client: AsyncClient, webhook_wrong_url: WebhookEndpoint, uow: SQLAlchemyUOW) -> None: response = await client.post("/register", json=USER_CREATE.model_dump(mode="json")) assert response.status_code == status.HTTP_201_CREATED try: - event = await uow.webhook_events.find(WebhookEventWebhookIDSpecification(webhook_wrong_url.id)) - assert event.status_code == 0 + delivery = await uow.webhook_deliveries.find(WebhookDeliveryEndpointIDSpecification(webhook_wrong_url.id)) + assert delivery.status == WebhookDeliveryStatus.pending + assert delivery.status_code is None + assert delivery.attempt_count == 0 except NoResultFoundError: - pytest.fail("No webhook event created") + pytest.fail("No webhook delivery created") diff --git a/tests/examples/__init__.py b/tests/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/examples/test_webhook_advanced.py b/tests/examples/test_webhook_advanced.py new file mode 100644 index 0000000..e05ce33 --- /dev/null +++ b/tests/examples/test_webhook_advanced.py @@ -0,0 +1,153 @@ +import asyncio +import time +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker +from starlette import status + +from examples.webhook_advanced import ( + MAX_BODY_BYTES, + SQLAlchemyIdempotencyStore, + WebhookEnvelope, + create_app, + create_schema, + create_store_engine, +) +from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request + +CLAIM_COUNT = 10 +EXPECTED_DUPLICATE_CLAIMS = CLAIM_COUNT - 1 +EXPECTED_PROCESS_CALLS = 2 + + +def database_url(tmp_path: Path) -> str: + return f"sqlite+aiosqlite:///{tmp_path / 'webhooks.sqlite3'}" + + +def store_for(engine: AsyncEngine) -> SQLAlchemyIdempotencyStore: + return SQLAlchemyIdempotencyStore(async_sessionmaker(engine, expire_on_commit=False)) + + +async def test_claim_persists_across_store_instances(tmp_path: Path) -> None: + url = database_url(tmp_path) + first_engine = create_store_engine(url) + second_engine = create_store_engine(url) + first = store_for(first_engine) + second = store_for(second_engine) + await create_schema(first_engine) + + assert await first.claim("webhook-1") + await first.complete("webhook-1") + assert not await second.claim("webhook-1") + + await first_engine.dispose() + await second_engine.dispose() + + +async def test_concurrent_claim_has_one_winner(tmp_path: Path) -> None: + engine = create_store_engine(database_url(tmp_path)) + store = store_for(engine) + await create_schema(engine) + + results = await asyncio.gather(*(store.claim("webhook-1") for _ in range(CLAIM_COUNT))) + + assert results.count(True) == 1 + assert results.count(False) == EXPECTED_DUPLICATE_CLAIMS + await engine.dispose() + + +async def test_release_allows_retry(tmp_path: Path) -> None: + engine = create_store_engine(database_url(tmp_path)) + store = store_for(engine) + await create_schema(engine) + assert await store.claim("webhook-1") + + await store.release("webhook-1") + + assert await store.claim("webhook-1") + await engine.dispose() + + +def test_duplicate_is_acknowledged_once(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + processed: list[str] = [] + + async def record(event: WebhookEnvelope) -> None: + processed.append(str(event.event.event_id)) + + value = payload() + body, headers = signed_request(value) + with TestClient(create_app(database_url(tmp_path), processor=record)) as client: + first = client.post("/fastid-webhooks", content=body, headers=headers) + duplicate = client.post("/fastid-webhooks", content=body, headers=headers) + + assert first.status_code == status.HTTP_204_NO_CONTENT + assert duplicate.status_code == status.HTTP_204_NO_CONTENT + assert processed == [value["event"]["event_id"]] + assert headers["webhook-id"] != value["event"]["event_id"] + + +def test_processing_failure_releases_claim(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + calls = 0 + + async def fail_once(event: WebhookEnvelope) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError(str(event.event.event_id)) + + body, headers = signed_request(payload()) + with TestClient(create_app(database_url(tmp_path), processor=fail_once)) as client: + failed = client.post("/fastid-webhooks", content=body, headers=headers) + retried = client.post("/fastid-webhooks", content=body, headers=headers) + + assert failed.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert retried.status_code == status.HTTP_204_NO_CONTENT + assert calls == EXPECTED_PROCESS_CALLS + + +def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"{}" + headers = headers_for(body, "webhook-1") + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_invalid_signature(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + headers["webhook-signature"] = "v1,invalid" + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +def test_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, _ = signed_request(payload()) + headers = headers_for(body, "webhook-1", timestamp=int(time.time()) - 301) + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_oversized_body(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"x" * (MAX_BODY_BYTES + 1) + headers = headers_for(body, "webhook-1") + + with TestClient(create_app(database_url(tmp_path))) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE diff --git a/tests/examples/test_webhook_quickstart.py b/tests/examples/test_webhook_quickstart.py new file mode 100644 index 0000000..38ff71f --- /dev/null +++ b/tests/examples/test_webhook_quickstart.py @@ -0,0 +1,100 @@ +import time + +import pytest +from fastapi.testclient import TestClient +from starlette import status + +from examples.webhook_quickstart import app +from tests.examples.webhook_helpers import SECRET, headers_for, payload, signed_request + + +def test_accepts_valid_webhook(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_204_NO_CONTENT + assert response.content == b"" + + +def test_requires_secret_at_startup(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FASTID_WEBHOOK_SECRET", raising=False) + + with pytest.raises(RuntimeError, match="FASTID_WEBHOOK_SECRET is required"), TestClient(app): + pass + + +def test_rejects_tampered_body(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body + b" ", headers=headers) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +@pytest.mark.parametrize("header", ["webhook-id", "webhook-timestamp", "webhook-signature"]) +def test_rejects_missing_header(monkeypatch: pytest.MonkeyPatch, header: str) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body, headers = signed_request(payload()) + headers.pop(header) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_malformed_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "opaque-webhook-id", timestamp="opaque-timestamp") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Invalid webhook-timestamp header"} + + +@pytest.mark.parametrize("offset", [-3600, 3600]) +def test_rejects_timestamp_outside_tolerance( + monkeypatch: pytest.MonkeyPatch, + offset: int, +) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + value = payload() + body, _ = signed_request(value) + headers = headers_for(body, "event-1", timestamp=int(time.time()) + offset) + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == {"detail": "Webhook timestamp is outside the allowed tolerance"} + + +def test_rejects_malformed_json(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"not-json" + headers = headers_for(body, "event-1") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +def test_rejects_invalid_envelope(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FASTID_WEBHOOK_SECRET", SECRET) + body = b"{}" + headers = headers_for(body, "event-1") + + with TestClient(app) as client: + response = client.post("/fastid-webhooks", content=body, headers=headers) + + assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/tests/examples/webhook_helpers.py b/tests/examples/webhook_helpers.py new file mode 100644 index 0000000..8906720 --- /dev/null +++ b/tests/examples/webhook_helpers.py @@ -0,0 +1,41 @@ +import base64 +import hashlib +import hmac +import json +import time +from typing import Any +from uuid import uuid4 + +SECRET = f"whsec_{base64.b64encode(b'test-webhook-secret').decode()}" + + +def payload(event_id: str | None = None) -> dict[str, Any]: + event_id = event_id or str(uuid4()) + return { + "event": {"event_id": event_id, "event_type": "user_registration", "timestamp": int(time.time())}, + "data": {"user": {"id": str(uuid4()), "email": "person@example.com"}}, + } + + +def headers_for( + body: bytes, + webhook_id: str, + *, + secret: str = SECRET, + timestamp: int | str | None = None, +) -> dict[str, str]: + timestamp = timestamp or int(time.time()) + key = base64.b64decode(secret.removeprefix("whsec_"), validate=True) + signed = b".".join((webhook_id.encode(), str(timestamp).encode(), body)) + signature = base64.b64encode(hmac.new(key, signed, hashlib.sha256).digest()).decode() + return { + "webhook-id": webhook_id, + "webhook-timestamp": str(timestamp), + "webhook-signature": f"v1,{signature}", + "content-type": "application/json", + } + + +def signed_request(value: dict[str, Any], secret: str = SECRET) -> tuple[bytes, dict[str, str]]: + body = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() + return body, headers_for(body, str(uuid4()), secret=secret) diff --git a/tests/mocks.py b/tests/mocks.py index 987d58c..df9dc10 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -11,7 +11,6 @@ from fastid.oauth.schemas import OpenIDBearer from fastid.security.crypto import crypt_ctx from fastid.security.webhooks import get_timestamp, get_webhook_id -from fastid.webhooks.config import webhook_settings from fastid.webhooks.models import WebhookType from tests.utils.auth import generate_random_state @@ -141,16 +140,6 @@ def webhook_record_factory(webhook_type: WebhookType, **kwargs: Any) -> dict[str WEBHOOK_PAYLOAD = {"test": {"test1": 1, "test2": "hello", "hello3": True}} WEBHOOK_TIMESTAMP = get_timestamp() WEBHOOK_ID = str(get_webhook_id()) -WEBHOOK_SECRET_KEY = faker.pystr() -WEBHOOK_EXPIRED_TIMESTAMP = WEBHOOK_TIMESTAMP - webhook_settings.tolerance_seconds -WEBHOOK_WRONG_SECRET_KEY = faker.pystr() -WEBHOOK_TEST_DATA = [ - (WEBHOOK_TIMESTAMP, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, True), - (0, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, False), - (WEBHOOK_TIMESTAMP, "", WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, False), - (WEBHOOK_EXPIRED_TIMESTAMP, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_SECRET_KEY, False), - (WEBHOOK_TIMESTAMP, WEBHOOK_ID, WEBHOOK_SECRET_KEY, WEBHOOK_WRONG_SECRET_KEY, False), -] class MockError(Exception): diff --git a/tests/security/test_webhooks.py b/tests/security/test_webhooks.py index 6579ce4..454c708 100644 --- a/tests/security/test_webhooks.py +++ b/tests/security/test_webhooks.py @@ -1,14 +1,61 @@ import pytest -from fastid.security.webhooks import generate_headers, verify_headers -from tests.mocks import WEBHOOK_PAYLOAD, WEBHOOK_TEST_DATA +from fastid.security.webhooks import ( + generate_delivery_headers, + generate_secret, + serialize_payload, + verify_standard_headers, +) +from tests.mocks import WEBHOOK_ID, WEBHOOK_PAYLOAD, WEBHOOK_TIMESTAMP + + +@pytest.mark.parametrize("secret", ["plain-secret", generate_secret()]) +def test_verify_standard_headers(secret: str) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, secret) + + assert set(headers) == { + "webhook-id", + "webhook-timestamp", + "webhook-signature", + "Content-Type", + "User-Agent", + } + assert verify_standard_headers(body, headers, secret) + assert not verify_standard_headers(body + b" ", headers, secret) + assert not verify_standard_headers(body, headers, "wrong-secret") + + +def test_standard_signature_uses_exact_utf8_body() -> None: + payload = {"message": "Привет"} + body = serialize_payload(payload) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + + assert b"\\u" not in body + assert verify_standard_headers(body, {key.upper(): value for key, value in headers.items()}, "secret") @pytest.mark.parametrize( - ("timestamp", "webhook_id", "generate_secret_key", "verify_secret_key", "expected"), WEBHOOK_TEST_DATA + "headers", + [ + {}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": "invalid", "webhook-signature": "v1,value"}, + {"webhook-id": "", "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": "v1,value"}, + {"webhook-id": WEBHOOK_ID, "webhook-timestamp": str(WEBHOOK_TIMESTAMP), "webhook-signature": ""}, + ], ) -def test_verify_headers( - timestamp: int, webhook_id: str, generate_secret_key: str, verify_secret_key: str, *, expected: bool -) -> None: - headers = generate_headers(WEBHOOK_PAYLOAD, timestamp, webhook_id, generate_secret_key) - assert verify_headers(WEBHOOK_PAYLOAD, headers, verify_secret_key) == expected +def test_verify_standard_headers_rejects_malformed_headers(headers: dict[str, str]) -> None: + assert not verify_standard_headers(b"{}", headers, "secret") + + +def test_verify_standard_headers_rejects_stale_timestamp(monkeypatch: pytest.MonkeyPatch) -> None: + body = serialize_payload(WEBHOOK_PAYLOAD) + headers = generate_delivery_headers(body, WEBHOOK_ID, WEBHOOK_TIMESTAMP, "secret") + monkeypatch.setattr("fastid.security.webhooks.time.time", lambda: WEBHOOK_TIMESTAMP + 301) + + assert not verify_standard_headers(body, headers, "secret") + + +def test_invalid_prefixed_secret_is_a_configuration_error() -> None: + with pytest.raises(ValueError, match="Invalid whsec_ webhook secret"): + generate_delivery_headers(b"{}", WEBHOOK_ID, WEBHOOK_TIMESTAMP, "whsec_not-base64") diff --git a/tests/utils/webhooks.py b/tests/utils/webhooks.py index b7de23c..c3a10f6 100644 --- a/tests/utils/webhooks.py +++ b/tests/utils/webhooks.py @@ -2,11 +2,11 @@ from fastid.apps.schemas import AppDTO from fastid.database.uow import SQLAlchemyUOW -from fastid.webhooks.models import Webhook +from fastid.webhooks.models import WebhookEndpoint -async def create_webhook(uow: SQLAlchemyUOW, oauth_app: AppDTO, data: dict[str, Any]) -> Webhook: - record = Webhook(app_id=oauth_app.id, **data) - await uow.webhooks.add(record) +async def create_webhook(uow: SQLAlchemyUOW, oauth_app: AppDTO, data: dict[str, Any]) -> WebhookEndpoint: + record = WebhookEndpoint(app_id=oauth_app.id, **data) + await uow.webhook_endpoints.add(record) await uow.commit() return record diff --git a/tests/webhooks/__init__.py b/tests/webhooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/webhooks/test_endpoint_models.py b/tests/webhooks/test_endpoint_models.py new file mode 100644 index 0000000..5fe7a31 --- /dev/null +++ b/tests/webhooks/test_endpoint_models.py @@ -0,0 +1,11 @@ +from fastid.apps.models import App +from fastid.webhooks.models import WebhookDelivery, WebhookEndpoint + + +def test_webhook_endpoint_and_delivery_mapping_names() -> None: + assert WebhookEndpoint.__tablename__ == "webhook_endpoints" + assert WebhookDelivery.__table__.c.endpoint_id.name == "endpoint_id" + foreign_key = next(iter(WebhookDelivery.__table__.c.endpoint_id.foreign_keys)) + assert foreign_key.target_fullname == "webhook_endpoints.id" + assert WebhookDelivery.endpoint.property.mapper.class_ is WebhookEndpoint + assert App.webhook_endpoints.property.mapper.class_ is WebhookEndpoint diff --git a/tests/webhooks/test_sender.py b/tests/webhooks/test_sender.py new file mode 100644 index 0000000..cf9b43d --- /dev/null +++ b/tests/webhooks/test_sender.py @@ -0,0 +1,51 @@ +import httpx +import pytest +from starlette import status + +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.senders.httpx import WebhookSender, resolve_targets +from tests.mocks import faker + + +async def test_sender_does_not_follow_redirects(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", True) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.content == b'{"ok":true}' + return httpx.Response(302, headers={"Location": "http://127.0.0.1/private"}, text="redirect") + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + result = await WebhookSender(client).send("http://example.test/hook", b'{"ok":true}', {}) + + assert result.status_code == status.HTTP_302_FOUND + assert result.content == {"value": "redirect"} + + +async def test_sender_bounds_response(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", True) + monkeypatch.setattr(webhook_settings, "max_response_bytes", 4) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _request: httpx.Response(status.HTTP_200_OK, text="123456")) + ) as client: + result = await WebhookSender(client).send("http://example.test/hook", b"{}", {}) + + assert result.content == {"value": 1234, "truncated": True} + + +@pytest.mark.parametrize( + ("url", "message"), + [("http://example.com/hook", "HTTPS"), (f"https://user:{faker.password()}@example.com/hook", "credentials")], +) +async def test_resolve_targets_rejects_unsafe_url(monkeypatch: pytest.MonkeyPatch, url: str, message: str) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", False) + + with pytest.raises(ValueError, match=message): + await resolve_targets(url) + + +async def test_resolve_targets_rejects_loopback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(webhook_settings, "allow_insecure_urls", False) + + with pytest.raises(ValueError, match="non-public"): + await resolve_targets("https://127.0.0.1/hook") diff --git a/tests/webhooks/test_worker.py b/tests/webhooks/test_worker.py new file mode 100644 index 0000000..fe7e8a8 --- /dev/null +++ b/tests/webhooks/test_worker.py @@ -0,0 +1,18 @@ +from fastid.webhooks.config import webhook_settings +from fastid.webhooks.worker import get_retry_delay + +FIVE_SECONDS = 5 +FIVE_MINUTES = 300 +ONE_MINUTE = 60 +ONE_DAY = 86400 + + +def test_retry_schedule_without_jitter() -> None: + assert get_retry_delay(1, None, jitter=1) == FIVE_SECONDS + assert get_retry_delay(2, None, jitter=1) == FIVE_MINUTES + assert get_retry_delay(9, None, jitter=1) == ONE_DAY + + +def test_retry_after_overrides_schedule_and_is_capped() -> None: + assert get_retry_delay(1, ONE_MINUTE, jitter=1) == ONE_MINUTE + assert get_retry_delay(1, webhook_settings.retry_after_max_seconds * 2, jitter=1) == ONE_DAY