From 0e9091d05efc6f59737ac1d862844c53b3258f4c Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:23:05 +0000 Subject: [PATCH 01/29] Bump the version to 1.1.2 for delayed events Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 19 +++++++++++++++++++ meson.build | 2 +- packaging/freebsd/+MANIFEST | 2 +- packaging/netbsd/Makefile | 2 +- packaging/opensuse/merovingian.spec | 5 ++++- packaging/rhel/merovingian.spec | 5 ++++- packaging/rpm/merovingian.spec | 5 ++++- scripts/build-deb.sh | 4 ++-- scripts/build-freebsd-pkg.sh | 4 ++-- scripts/build-netbsd-pkg.sh | 4 ++-- scripts/build-openbsd-pkg.sh | 4 ++-- scripts/build-opensuse-rpm.sh | 4 ++-- scripts/build-rhel-rpm.sh | 4 ++-- scripts/build-rpm.sh | 4 ++-- scripts/build-static-linux.sh | 2 +- src/db_migrate.cpp | 2 +- src/main.cpp | 2 +- 17 files changed, 51 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c5925ae0..a6745a339 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ Changes from 1.0.0, the first production release, onwards; the format is set out Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md), which is no longer added to ([ADR-0740](docs/adr/0740-the-changelog-starts-at-1-0-0-and-the-pre-release-entries-are-a-frozen-record.md)). +# Version 1.1.2 + +## Features + +* Delayed events (MSC4140): a client can schedule an event to be sent after a delay, and restart, cancel or send it, + so a MatrixRTC call membership ends when its client vanishes. + +## Security Fixes + +* None. + +## Bug fixes + +* None. + +## Release notes + +Delayed events for MatrixRTC, on top of 1.1.1. Upgrading runs migration 003, which adds one table. + # Version 1.1.1 ## Features diff --git a/meson.build b/meson.build index 90070cb6a..813a15102 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'merovingian', 'cpp', 'c', - version: '1.1.1', + version: '1.1.2', license: 'GPL-3.0-or-later', meson_version: '>=1.1.0', default_options: [ diff --git a/packaging/freebsd/+MANIFEST b/packaging/freebsd/+MANIFEST index 2ba83013d..8ec24b40a 100644 --- a/packaging/freebsd/+MANIFEST +++ b/packaging/freebsd/+MANIFEST @@ -1,5 +1,5 @@ name: "merovingian" -version: "1.1.1" +version: "1.1.2" origin: net/merovingian comment: Secure Matrix Protocol homeserver desc: "Merovingian is a Matrix Protocol homeserver." diff --git a/packaging/netbsd/Makefile b/packaging/netbsd/Makefile index ccdfd53dc..ee3d41614 100644 --- a/packaging/netbsd/Makefile +++ b/packaging/netbsd/Makefile @@ -1,4 +1,4 @@ -DISTNAME= merovingian-1.1.1 +DISTNAME= merovingian-1.1.2 CATEGORIES= net MASTER_SITES= ${MASTER_SITE_GITHUB:=OMG-Software/} GITHUB_PROJECT= Merovingian diff --git a/packaging/opensuse/merovingian.spec b/packaging/opensuse/merovingian.spec index 9cbf6bb40..4c5cc7c95 100644 --- a/packaging/opensuse/merovingian.spec +++ b/packaging/opensuse/merovingian.spec @@ -1,5 +1,5 @@ Name: merovingian -Version: 1.1.1 +Version: 1.1.2 Release: 1%{?dist} Summary: Secure Matrix Protocol homeserver @@ -115,6 +115,9 @@ fi %{_sysconfdir}/merovingian/merovingian.conf.example %changelog +* Tue Nov 17 2026 James Chapman - 1.1.2-1 +- Merovingian 1.1.2 + * Mon Nov 16 2026 James Chapman - 1.1.1-1 - Merovingian 1.1.1 diff --git a/packaging/rhel/merovingian.spec b/packaging/rhel/merovingian.spec index dcad8c654..218c2842d 100644 --- a/packaging/rhel/merovingian.spec +++ b/packaging/rhel/merovingian.spec @@ -1,5 +1,5 @@ Name: merovingian -Version: 1.1.1 +Version: 1.1.2 Release: 1%{?dist} Summary: Secure Matrix Protocol homeserver @@ -100,6 +100,9 @@ fi %{_sysconfdir}/merovingian/merovingian.conf.example %changelog +* Tue Nov 17 2026 James Chapman - 1.1.2-1 +- Merovingian 1.1.2 + * Mon Nov 16 2026 James Chapman - 1.1.1-1 - Merovingian 1.1.1 diff --git a/packaging/rpm/merovingian.spec b/packaging/rpm/merovingian.spec index 861d50dfe..d6dacd37c 100644 --- a/packaging/rpm/merovingian.spec +++ b/packaging/rpm/merovingian.spec @@ -1,5 +1,5 @@ Name: merovingian -Version: 1.1.1 +Version: 1.1.2 Release: 1%{?dist} Summary: Secure Matrix Protocol homeserver @@ -104,6 +104,9 @@ fi %{_sysconfdir}/merovingian/merovingian.conf.example %changelog +* Tue Nov 17 2026 James Chapman - 1.1.2-1 +- Merovingian 1.1.2 + * Mon Nov 16 2026 James Chapman - 1.1.1-1 - Merovingian 1.1.1 diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh index 222b53984..810712faf 100755 --- a/scripts/build-deb.sh +++ b/scripts/build-deb.sh @@ -1,9 +1,9 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.1. +# Build for merovingian 1.1.2. set -e -VERSION="1.1.1" +VERSION="1.1.2" PKG_NAME="merovingian" STAGING="staging-deb" diff --git a/scripts/build-freebsd-pkg.sh b/scripts/build-freebsd-pkg.sh index 0c4e6fd97..999397b0e 100755 --- a/scripts/build-freebsd-pkg.sh +++ b/scripts/build-freebsd-pkg.sh @@ -1,9 +1,9 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.1. +# Build for merovingian 1.1.2. set -e -VERSION="1.1.1" +VERSION="1.1.2" STAGING="staging-fbsd" # Clean any state (staged files, build dir) from cached FreeBSD VM runs. diff --git a/scripts/build-netbsd-pkg.sh b/scripts/build-netbsd-pkg.sh index 736a85ab9..5d3651758 100644 --- a/scripts/build-netbsd-pkg.sh +++ b/scripts/build-netbsd-pkg.sh @@ -1,6 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.1. +# Build for merovingian 1.1.2. # # The checked-in packaging/netbsd/Makefile is the pkgsrc recipe kept for # downstream porters. This script generates a framework-free package so CI @@ -11,7 +11,7 @@ # a staged tree using tar, which produces a package that pkg_add can install. set -e -VERSION="1.1.1" +VERSION="1.1.2" STAGE="staging-netbsd" PREFIX=/usr/pkg diff --git a/scripts/build-openbsd-pkg.sh b/scripts/build-openbsd-pkg.sh index 4779ebb99..a05d22805 100644 --- a/scripts/build-openbsd-pkg.sh +++ b/scripts/build-openbsd-pkg.sh @@ -1,6 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.1. +# Build for merovingian 1.1.2. # # Standalone pkg_create(1), no ports tree. The checked-in packaging/openbsd/PLIST # is the ports-framework packing list kept for downstream porters; this script @@ -8,7 +8,7 @@ # produce and validate an installable package on every run. set -e -VERSION="1.1.1" +VERSION="1.1.2" STAGE="staging-openbsd" PREFIX=/usr/local diff --git a/scripts/build-opensuse-rpm.sh b/scripts/build-opensuse-rpm.sh index c58981dbb..cd3cd82d3 100644 --- a/scripts/build-opensuse-rpm.sh +++ b/scripts/build-opensuse-rpm.sh @@ -1,6 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build an RPM package for merovingian 1.1.1 on OpenSUSE Tumbleweed or Leap. +# Build an RPM package for merovingian 1.1.2 on OpenSUSE Tumbleweed or Leap. # # Uses packaging/opensuse/merovingian.spec which adapts BuildRequires to # OpenSUSE package names (libopenssl-devel, postgresql-devel, libpng16-devel, @@ -12,7 +12,7 @@ # (.leap16, ADR-0650). set -e -VERSION="1.1.1" +VERSION="1.1.2" DIST="${MEROVINGIAN_RPM_DIST:-.tumbleweed}" case "${DIST}" in .[a-z]*) ;; diff --git a/scripts/build-rhel-rpm.sh b/scripts/build-rhel-rpm.sh index b9fac8782..a6b668a76 100644 --- a/scripts/build-rhel-rpm.sh +++ b/scripts/build-rhel-rpm.sh @@ -1,13 +1,13 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build an RPM package for merovingian 1.1.1 targeting RHEL 10 / AlmaLinux 10. +# Build an RPM package for merovingian 1.1.2 targeting RHEL 10 / AlmaLinux 10. # # Differs from build-rpm.sh only in the spec file used: packaging/rhel/merovingian.spec # omits catch-devel from BuildRequires (Catch2 is not reliably available in EPEL 10 # and is not needed when -Dbuild_tests=false). set -e -VERSION="1.1.1" +VERSION="1.1.2" mkdir -p "${HOME}/rpmbuild/BUILD" mkdir -p "${HOME}/rpmbuild/BUILDROOT" diff --git a/scripts/build-rpm.sh b/scripts/build-rpm.sh index 5fd3f46ea..ac4f9b3f7 100755 --- a/scripts/build-rpm.sh +++ b/scripts/build-rpm.sh @@ -1,9 +1,9 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build an RPM package for merovingian 1.1.1 using rpmbuild. +# Build an RPM package for merovingian 1.1.2 using rpmbuild. set -e -VERSION="1.1.1" +VERSION="1.1.2" # 1. Create rpmbuild directory tree mkdir -p "${HOME}/rpmbuild/BUILD" diff --git a/scripts/build-static-linux.sh b/scripts/build-static-linux.sh index bacfb77c5..a83d86c1e 100755 --- a/scripts/build-static-linux.sh +++ b/scripts/build-static-linux.sh @@ -3,7 +3,7 @@ # Build a portable Linux fallback tarball with musl-linked static PIE binaries. set -eu -VERSION="${MEROVINGIAN_VERSION:-1.1.1}" +VERSION="${MEROVINGIAN_VERSION:-1.1.2}" BUILD_DIR="${BUILD_DIR:-build-static-linux}" STAGING="staging-static-linux" PACKAGE_ROOT="merovingian-${VERSION}-linux-static-x86_64" diff --git a/src/db_migrate.cpp b/src/db_migrate.cpp index 2cb7fb5d9..1e38bb17f 100644 --- a/src/db_migrate.cpp +++ b/src/db_migrate.cpp @@ -23,7 +23,7 @@ namespace { -constexpr auto version = std::string_view{"1.1.1"}; +constexpr auto version = std::string_view{"1.1.2"}; auto print_help() -> void { diff --git a/src/main.cpp b/src/main.cpp index da3e7cfd6..2606ec74b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -81,7 +81,7 @@ namespace { -constexpr auto version = std::string_view{"1.1.1"}; +constexpr auto version = std::string_view{"1.1.2"}; struct BootstrapConfigResult final { From affbc57ba3e23afb233c51378914475092d94620 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:55:42 +0000 Subject: [PATCH 02/29] Record the delayed-events decisions (ADR-0823 to ADR-0826) Co-Authored-By: Claude Fable 5.2 --- ...ameter-forms-that-released-clients-send.md | 138 ++++++++++++++++++ ...d-per-user-and-deleted-with-the-account.md | 95 ++++++++++++ ...send-path-never-early-and-at-least-once.md | 114 +++++++++++++++ ...-per-user-caps-and-per-user-rate-limits.md | 78 ++++++++++ docs/adr/index.md | 4 + 5 files changed, 429 insertions(+) create mode 100644 docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md create mode 100644 docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md create mode 100644 docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md create mode 100644 docs/adr/0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md diff --git a/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md b/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md new file mode 100644 index 000000000..0fd2d0ebb --- /dev/null +++ b/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md @@ -0,0 +1,138 @@ +# Delayed events are served on the stable, unstable and query-parameter forms that released clients send + +* Status: accepted +* Date: 2026-11-17 + +## Context and Problem Statement + +Delayed events (MSC4140) let a client ask the server to send an event later, and to cancel or postpone it. MatrixRTC +uses them to schedule a call membership leave, so that a participant whose client vanished is removed from the call. +The MSC merged on 2026-09-08 and the feature is in matrix-spec `main` (commit `7198bf6724`, 2026-11-13) as v1.20. It +is not in the vendored v1.19 specification, so it has no conformance scenarios here. Which request forms does the +server serve? + +The released clients, read at their release tags on 2026-11-17, do not send the spec's stable paths: + +* Element X iOS and Android 26.11.0 (matrix-rust-sdk `3ff03638`, ruma `0ed7fba3`) and Element Call 0.28.2 running + standalone send only the query form, `?org.matrix.msc4140.delay=` on the ordinary `/send` and `/state` routes. + Element X manages an event with `POST .../org.matrix.msc4140/delayed_events/{delayId}` and the action in the body. +* Element Web 1.12.35 (matrix-js-sdk 45.1.0) tries the unstable dedicated path + `.../org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}` first and falls back to the query form on + `M_UNRECOGNIZED`. It manages with `POST .../org.matrix.msc4140/delayed_events/{delayId}/{send|cancel|restart}`. + Element Call 0.28.2 pins matrix-js-sdk commit `3f05dc4d`, which does the same. +* lk-jwt-service 0.10.0 manages a delegated leave as an application service acting for the user, on the same + unstable management path. +* Every client reads only the `org.matrix.msc4140` flag in `unstable_features`; matrix-js-sdk refuses to use the + feature without it. + +## Decision Outcome + +Chosen option: "serve every form a released client sends, plus the spec's stable forms", because a server that serves +only the stable paths schedules nothing for the clients that exist. + +| # | Method and path | Who sends it | +|---|---|---| +| S1 | `PUT /_matrix/client/v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}` | spec, stable; no released client yet | +| S2 | `PUT /_matrix/client/unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}` | Element Web 1.12.35 | +| L1 | `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}?org.matrix.msc4140.delay=` | Element X, standalone Element Call | +| L2 | `PUT /_matrix/client/v3/rooms/{roomId}/state/{eventType}/{stateKey}?org.matrix.msc4140.delay=`, and the no-state-key and empty-state-key forms the ordinary state route accepts | Element X, standalone Element Call; the path a call's leave is scheduled on today | +| M1 | `POST /_matrix/client/v1/delayed_events/{delayId}/{send\|cancel\|restart}` | spec, stable | +| M2 | `POST /_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}/{send\|cancel\|restart}` | matrix-js-sdk, lk-jwt-service | +| M3 | `POST /_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}` with body `{"action":"send"\|"cancel"\|"restart"}` | Element X; the js-sdk fallback | +| G1 | `GET /_matrix/client/v1/delayed_events/{delayId}` | spec, stable | +| G2 | `GET /_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}` | a matrix-js-sdk method | + +M3 exists only under the unstable prefix; `POST /_matrix/client/v1/delayed_events/{delayId}` is not a route. A last +segment other than `send`, `cancel` or `restart` on M1 or M2 is not a route (`404 M_UNRECOGNIZED`). + +Every route needs an access token and goes through the standard gate (`M_MISSING_TOKEN`, `M_UNKNOWN_TOKEN`, the +moderation gate for locked and suspended accounts). Application-service masquerade (`?user_id=`) works exactly as on +other routes; lk-jwt-service depends on it. + +A request that carries `org.matrix.msc4140.delay` is always taken down the delayed branch, before the ordinary +route's idempotency lookup and before any send. It never falls through to an immediate send, including when the +feature is disabled (then it is `403 M_FORBIDDEN`). A delay that is not a decimal integer of at least 1, or is +repeated, is `400 M_INVALID_PARAM`. + +Management looks an event up by the authenticated user and the delay ID. A delay ID that does not exist and one that +belongs to another user get the same answer, byte for byte: `404 M_NOT_FOUND`, `Delayed event not found.` + +Management outcomes, all success bodies `{}`: + +| State of the delayed event | `restart` | `cancel` | `send` | +|---|---|---|---| +| scheduled | 200; the delay starts again from now | 200; finalised as cancelled | sends now; 200 and finalised as sent; on failure the status and error an ordinary send gives, and the event stays scheduled with its timer unchanged | +| sent | 409 | 409 | 200 | +| cancelled by the user | 409 | 200 | 409 | +| cancelled because sending failed | 409 | 200 | 409 | +| being sent right now | 409 | 409 | 409 | + +The 409 errcode is `M_UNKNOWN`. Released clients treat 404 `M_NOT_FOUND` and any 409 alike as "gone" +(matrix-js-sdk `isDelayedEventGoneError`, lk-jwt-service). + +`/versions` carries `org.matrix.msc4140` and `org.matrix.msc4140.stable`, both set to whether the feature is enabled. +`/capabilities` carries `m.delayed_events` and `org.matrix.msc4140.delayed_events`, each +`{"max_delay_ms", "max_scheduled"}`, zeros when disabled. The event a delayed event became carries +`unsigned.delay_id` and `unsigned.org.matrix.msc4140.delay_id` for the user who scheduled it only, and no +`transaction_id`. + +## Pros and Cons of the Options + +### Serve only the spec's stable paths + +* Bad, because no released client calls them: Element X and standalone Element Call would schedule nothing. + +### Do not serve the dedicated path and rely on the js-sdk fallback + +* Bad, because the fallback needs exactly `M_UNRECOGNIZED`, is permanent for the client's lifetime, and the dedicated + path is where the spec and clients are going. + +### Answer 404 for every finalised delayed event, as Synapse 1.165 does by deleting the row + +* Bad, because it loses the spec's idempotent 200 for a repeated action and the stored error. Released clients treat + 409 and 404 alike, so nothing is gained. + +### Unauthenticated management, with the delay ID as a bearer capability + +Earlier MSC drafts did this, and one other server still accepts it on the path form. + +* Bad, because the final MSC and the spec require authentication, every released client sends a token, and + lk-jwt-service authenticates as an application service. The delay ID is an identifier, never a credential. + +### The pre-1.157 Synapse error for a too-large delay + +`M_UNKNOWN` with `org.matrix.msc4140.errcode`. It is the only form Element Call 0.28.2's pinned js-sdk recognises. + +* Bad, because it is gone from the MSC and from Synapse. The server answers `400` with `M_DELAY_TOO_LARGE` on S1 and + `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE` on S2, L1 and L2. The cost is confined to an operator who lowers the maximum + delay below what the client asks for (18 seconds; one hour with MSC4195 delegation): that client joins without a + delayed leave. + +### The listing endpoint + +`GET .../delayed_events` is MSC4486. + +* Bad, because it is unmerged and no released code calls it. It stays `404 M_UNRECOGNIZED`. + +### Cancel a pending delayed state event when another state event with the same type and state key is sent + +Earlier drafts did this. Synapse stopped in 1.127 and 1.164. + +* Bad, because the client sends its join immediately after scheduling its leave under the same key; the join would + cancel the leave. + +## Consequences + +* The unstable and query forms are retired only after released clients stop sending them. Check the client release + tags again when a spec release containing delayed events is vendored. +* The listing endpoint and sticky events (MSC4354) remain out of scope; the sticky member of the dedicated body is + ignored. +* Application-service masquerade on the management routes must keep working. +* The user manual tells operators to keep the maximum delay at one hour or more. + +## Links + +* Related: [ADR-0824](0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md) +* Related: [ADR-0825](0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md) +* Related: [ADR-0826](0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md) +* Related: [ADR-0822](0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md) diff --git a/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md b/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md new file mode 100644 index 000000000..e3f075948 --- /dev/null +++ b/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md @@ -0,0 +1,95 @@ +# A delayed event is a stored row that is kept after it is finalised, bounded per user, and deleted with the account + +* Status: accepted +* Date: 2026-11-17 + +## Context and Problem Statement + +A delayed event ([ADR-0823](0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md)) +must survive a restart, because a call's leave that is lost leaves a vanished participant listed in the call, the bug +the feature exists to fix. After it is sent or cancelled, clients still ask about it: the spec answers a repeated +action with 200 or 409, a lookup shows the outcome and any stored error, and the event it became carries +`unsigned.delay_id`. How is it stored, how long is it kept, and what bounds it? + +## Decision Outcome + +Chosen option: "one table, with finalised rows kept for a bounded time", because it gives the spec's answers and +survives restarts with one mechanism. + +* Migration 003 (`migrations/003_delayed_events.sql`, schema version 3, name `delayed_events`) adds the table + `delayed_events` and the index `idx_delayed_events_user_id` on `(user_id)`, following + [ADR-0778](0778-the-schema-is-built-from-per-version-tables-and-each-migration-step-from-what-its-version-adds.md) + exactly as migration 002 did. An earlier version's definitions, `001_initial_schema.sql` and + `002_oauth_tables.sql` are never edited. +* Every column is `TEXT`, with no NULLs, like every other table: numbers are decimal text, absent values are `''` or + `0`. The columns are `delay_id` (primary key), `user_id`, `device_id`, `room_id`, `event_type`, `is_state`, + `state_key`, `content_json`, `txn_id`, `via_appservice`, `delay_ms`, `delayed_since_ms`, `status` (`scheduled`, + `sent`, `cancelled` or `failed`), `finalised_at_ms`, `event_id` and `error_json`. +* The table is mirrored in memory in `PersistentStore` and hydrated at start-up on both backends. A table class in + the database module, `DelayedEventTable`, owns the rows and keeps its indexes right by construction; nothing else + touches the rows. Its indexes are by delay ID; per user, the scheduled count and the finalised rows oldest first; + scheduled rows ordered by due time (`delayed_since_ms + delay_ms`, then `delay_id`); by (user, device, room, event + type, transaction ID) for idempotency; by event ID for sent rows; and finalised rows by finalised time. The runtime + mutex serialises it, so it has no lock of its own. +* The mirror changes only after the transaction commits (one exception, in + [ADR-0825](0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md)). +* A user may hold at most `max_scheduled_per_user` scheduled events. At the cap a new one is refused with + `429 M_LIMIT_EXCEEDED`; a scheduled event is the user's state and is never evicted. +* Finalised rows are kept for the spec's idempotent answers, for `GET` and for `unsigned.delay_id`. They are bounded + per user by `max_finalised_per_user` (default 100): when a row is finalised and the user is at the bound, that + user's oldest finalised rows are deleted in the same transaction. They are bounded in age by a fixed 7 days, pruned + by the delayed-events job, at most 500 rows per transaction, oldest first. +* A restart of a scheduled event is persisted: it updates `delayed_since_ms`. +* Deactivating an account deletes all of the user's rows, scheduled and finalised, in one transaction, as a step of + `deactivate_user_account` beside the pusher cleanup. A failure is logged and deactivation continues; the check at + send time (ADR-0825) is the guarantee. +* Logout and device deletion do not cancel. A delayed event belongs to the user; any of their devices, or an + application service acting for them, may manage it. +* The federation worker never reads the table. It is classified in the never-reads set of + `tests/unit/test_worker_table_load_profile.cpp`. + +## Pros and Cons of the Options + +### Delete the row when it is sent or cancelled (Synapse) + +* Bad, because there is then no 200 and 409 distinction, no stored error, and no `unsigned.delay_id`. + +### An in-memory queue only + +* Bad, because it is lost on restart, which leaves a vanished participant listed in a call. + +### Not persisting each restart + +Persist only the schedule, not the postponement. + +* Bad, because after a server restart the persisted deadline is earlier than the one the client last set, and the + event would be sent early. The alternative of restarting every timer at start-up never sends for a server restarted + more often than the delay. The price accepted is one small write per restart, the same as Synapse. + +### The spec's example retention (7 days or 1000 per user) + +* Bad, because every row is mirrored in memory with its content (up to 65,536 bytes). The count is 100 by default and + configurable; the age is the spec's 7 days. + +### Cancelling on logout or device deletion + +* Bad, because a call's leave must still fire after its device is gone. + +### `INTEGER` or nullable columns + +* Bad, because every table in this schema stores numbers as text and has no NULLs. + +## Consequences + +* A new reader goes through the table class's indexes and never scans the table. +* The worst case held for one user is `max_scheduled_per_user` plus `max_finalised_per_user` rows of at most 65,536 + bytes of content. +* The federation worker does not read the table; a change that makes it read the table must change its + classification deliberately. +* A new code path that deletes an account's data must delete these rows too. + +## Links + +* Related: [ADR-0778](0778-the-schema-is-built-from-per-version-tables-and-each-migration-step-from-what-its-version-adds.md) +* Related: [ADR-0823](0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md) +* Related: [ADR-0826](0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md) diff --git a/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md b/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md new file mode 100644 index 000000000..cdc0f9447 --- /dev/null +++ b/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md @@ -0,0 +1,114 @@ +# A due delayed event is sent by one job through the ordinary send path, never early and at least once + +* Status: accepted +* Date: 2026-11-17 + +## Context and Problem Statement + +A delayed event ([ADR-0824](0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md)) +is stored with a deadline. Something must send it when the deadline passes, as the user, authorised against the room +as it is then. The spec requires that it is never sent early, asks for it within 30 seconds, requires the per-user +rate limit to apply, and says a failed send is cancelled with the error stored. Who sends it, with what checks, and +what happens when the server stops between sending and recording? + +## Decision Outcome + +Chosen option: "one `PeriodicJob` that sends through `send_event_as_user`", because it reuses the ordinary send path +and its authorisation, and the job machinery already sleeps until a deadline. + +* A `PeriodicJob` named `delayed-events` is created in `src/main.cpp` beside `typing-expiry`, started after process + hardening, stopped on shutdown in the same sequence as the other jobs, and refuses to serve if its thread cannot + start. `HomeserverRuntime::delayed_events_wake` (null in tests, installed and removed under the runtime lock like + `typing_expiry_wake`) is called when an event is scheduled. A restart only moves a deadline later and needs no wake. +* The body is a function taking the runtime, `now_ms` and the stop flag. Per run: take the runtime lock, ask the table + for the earliest-due scheduled row with due time at or before `now_ms`, send it, release the lock; one send per + lock hold, the stop flag checked between items, at most 100 sends per run (then it runs again at once). One age + prune batch, in its own lock hold. It returns the time until the next due row, clamped to 0 to 60 seconds, and 60 + seconds when nothing is scheduled. +* Time is wall-clock milliseconds, because the due time is persisted and must mean the same after a restart. Handlers + and the job take "now" from one function that honours a `HomeserverRuntime` test seam. Tests drive the job body with + a chosen `now_ms` and never sleep; no test depends on a wall-clock margin. +* Never early: the job sends only when `now_ms >= delayed_since_ms + delay_ms`, both read under the lock at the + moment of sending. Events that fell due while the server was down are sent by the first run after start-up. +* One send, under the runtime lock: re-read the row (still scheduled, not being sent); apply the account checks the + request gate would apply (unknown or deactivated `M_USER_DEACTIVATED`, locked `M_USER_LOCKED`, suspended + `M_USER_SUSPENDED`); apply the per-user rate limit with one `RateLimitEngine::check` with an empty address bucket + and the per-user key and policy of the ordinary `/send` or `/state` template; build the event body with the + ordinary route's helper; call `send_event_as_user`. The event is therefore authorised against the room state now, + signed, persisted, federated, pushed and published to `/sync` like any other. +* Success finalises the row as sent with the event ID. Failure finalises it as failed with the standard error object. + A failed send is never retried. +* The `send` action runs the same steps inside the request; on failure it answers with the send's own status and + error and leaves the row scheduled; a rate-limit denial there is `429 M_LIMIT_EXCEEDED` with `retry_after_ms`. +* Exactly one sender at a time: the job and the `send` action both run under the runtime mutex and re-read the status + under it. The row is also marked "being sent" (memory only) before `send_event_as_user` is called and unmarked when + it returns, and while it is marked the row is out of the due order and a second send, restart or cancel answers + 409. +* At least once: the order is send, then finalise. The event's commit and the row's update are two transactions; a + crash between them sends the event again after restart. For a call membership leave a duplicate is harmless and a + lost one is the bug being fixed. +* If the finalising commit fails after the send was attempted, the in-memory row is finalised anyway and the failure + is logged. Memory then says what happened, the database row still says `scheduled`, and the next start-up sends it + again. Leaving the memory row scheduled would send it again on every run. This is the only place the mirror changes + without a successful commit, and the code says why in a comment. + +## Pros and Cons of the Options + +### Record as sent, then send (at most once) + +* Bad, because a crash between the two loses a call's leave. + +### Put the row's update in the event's own transaction + +* Bad, because it would change how ordinary sending works. + +### Retry a failed send + +* Bad, because the spec says cancel and store the error, and an authorisation failure would retry for ever. + +### A grace period after start-up before sending overdue events + +* Bad, because a vanished client's membership would outlive the restart by the grace. The spec asks only for "never + early", and the reference server sends at once. + +### A steady (monotonic) clock + +* Bad, because the deadline must survive a restart. + +### A thread or timer per event + +* Bad, because `PeriodicJob` already sleeps until the earliest deadline and can be woken. + +### Sending without re-checking the account + +* Bad, because the lock, suspension and deactivation refusals live only at the request gate, so the job must apply + them itself, as the profile propagation job does. + +### No send-time rate limit + +* Bad, because the spec requires one. The engine's per-user bucket for the ordinary route is consulted with no client + address, which is a no-op unless an operator configured a per-user policy for sending. + +### A full dry run when an event is scheduled + +Compose and sign, then discard. + +* Bad, because it would need the ordinary send split in two. The schedule-time check is membership plus, for state, + `may_send_state_event`; the real authorisation is at send time. + +## Consequences + +* The delayed-events job is the only background sender of delayed events. +* Anything inside the send path that releases the runtime lock is covered by the "being sent" mark. A change that + makes the send path release the lock for a new event type must keep the mark set across it. +* A clock stepped backwards delays sends; stepped forwards, it brings them forward by the step. The server has no + other persistent clock. +* The one place an in-memory row changes without a successful commit is after a send attempt, for the reason above; + do not add another. +* Tests drive the job body with a chosen `now_ms` and never sleep. + +## Links + +* Related: [ADR-0824](0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md) +* Related: [ADR-0823](0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md) +* Related: [ADR-0353](0353-a-profile-change-holds-the-runtime-lock-for-a-fixed-share-of-the-users-rooms.md) diff --git a/docs/adr/0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md b/docs/adr/0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md new file mode 100644 index 000000000..88bfc92b0 --- /dev/null +++ b/docs/adr/0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md @@ -0,0 +1,78 @@ +# Delayed events are on by default, with a maximum delay, per-user caps and per-user rate limits + +* Status: accepted +* Date: 2026-11-17 + +## Context and Problem Statement + +Delayed events ([ADR-0823](0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md)) +hold server memory and database rows for each scheduled and finalised event, and a call restarts its delayed event +every 4 to 5 seconds. Should the feature be on, and what bounds it? + +## Decision Outcome + +Chosen option: "on by default, bounded per user, with the limits in a restart-required configuration section", because +MatrixRTC calls leave vanished participants listed without it and every limit has a safe default. + +| Key | Default | Range | +|---|---|---| +| `server.delayed_events.enabled` | `true` | boolean | +| `server.delayed_events.max_delay_ms` | `86400000` (24 hours) | 1000 to 2678400000 (31 days) | +| `server.delayed_events.max_scheduled_per_user` | `100` | 1 to 1000 | +| `server.delayed_events.max_finalised_per_user` | `100` | 1 to 1000 | + +* Every key is validated at parse time and is restart-required: the limits are snapshotted at start-up like + `server.client_api.*`. +* Disabled: scheduling is `403 M_FORBIDDEN` (`Delayed events are not enabled on this homeserver.`), `/versions` flags + are false and `/capabilities` carries zeros. Already-scheduled events are still sent and can still be managed. +* A delay over `max_delay_ms` is `400` (`M_DELAY_TOO_LARGE` on S1, `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE` on S2, L1 and + L2). A user at `max_scheduled_per_user` gets `429 M_LIMIT_EXCEEDED` with `retry_after_ms` and a `Retry-After` + header set to the time until the user's next scheduled send, rounded up to a whole second and at least 1. + `max_finalised_per_user` bounds the finalised rows kept per user, with the fixed 7-day age + ([ADR-0824](0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md)). +* `content` is held to the event size limit (65,536 bytes); a larger one is `413 M_TOO_LARGE`. +* Rate limits. The global limiter runs before dispatch as for every route, and each new path template is its own + bucket. Management (M1, M2, M3) and lookup (G1, G2) have a built-in per-address refinement of 3000 per 60 seconds + (many users behind one address, and lk-jwt-service manages every delegated leave from one address) and a built-in + per-user refinement of 120 per 60 seconds (a call restarts every 4 to 5 seconds, 12 to 15 a minute per device in a + call). The dedicated scheduling paths (S1, S2) use the generic per-address tier and a built-in per-user 60 per 60 + seconds. The legacy forms (L1, L2) share the ordinary `/send` and `/state` buckets. All are overridable through + `client_rate_limits.*`. + +## Pros and Cons of the Options + +### Off by default + +Synapse leaves it off: an unset `max_event_delay_duration` disables it. + +* Bad, because MatrixRTC calls leave vanished participants listed without it, and every limit has a safe default. + +### A value of `0` meaning disabled instead of an `enabled` key + +* Bad, because the range check on the other keys is simpler without it, and the capability's "0 means disabled" is + derived from `enabled`. + +### A server-wide cap on scheduled events + +* Bad, because one user could then exhaust it for everyone. The bound is per user. + +### A new rate-limit tier + +* Bad, because built-in per-user and per-address refinements on the new path templates are enough and stay + overridable through `client_rate_limits.*`. + +### Reloadable keys + +* Bad, because the limits are snapshotted at start-up like `server.client_api.*`. + +## Consequences + +* An operator running Element Call keeps `max_delay_ms` at one hour (3600000) or more; the user manual says so. +* Disabling the feature stops new scheduling; it does not cancel events already scheduled. +* A new delayed-events path template gets its own rate-limit bucket and an HTTP-3 scenario row. + +## Links + +* Related: [ADR-0823](0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md) +* Related: [ADR-0824](0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md) +* Related: [ADR-0825](0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md) diff --git a/docs/adr/index.md b/docs/adr/index.md index 346534385..d1afd2355 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -701,3 +701,7 @@ * [0820 - A left room's state_after stops at what the user may read, and always holds their own membership](0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md) * [0821 - server.turn.uris is added beside server.turn.server, which is kept](0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md) * [0822 - The RTC transport is advertised as type livekit with livekit_service_url](0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md) +* [0823 - Delayed events are served on the stable, unstable and query-parameter forms that released clients send](0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md) +* [0824 - A delayed event is a stored row that is kept after it is finalised, bounded per user, and deleted with the account](0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md) +* [0825 - A due delayed event is sent by one job through the ordinary send path, never early and at least once](0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md) +* [0826 - Delayed events are on by default, with a maximum delay, per-user caps and per-user rate limits](0826-delayed-events-are-on-by-default-with-a-maximum-delay-per-user-caps-and-per-user-rate-limits.md) From b6b5f0c40ea988d78d85e8889e14a1cfc7282367 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:12:38 +0000 Subject: [PATCH 03/29] Add migration 003 and the delayed-events table, store functions and hydration (ADR-0824, ADR-0825) Migration 003 creates delayed_events on SQLite and PostgreSQL, with DelayedEventTable holding the rows and their indexes in memory and DelayedEventStore.hpp writing them in one transaction each. Finalising evicts a user's oldest finalised rows in the same transaction, and the outcome of a send finalises memory even when its commit fails. Co-Authored-By: Claude Fable 5.2 --- .../database/DelayedEventOutcomeResult.hpp | 19 + .../database/DelayedEventStatus.hpp | 20 + .../database/DelayedEventStatusText.hpp | 18 + .../database/DelayedEventStore.hpp | 76 ++ .../database/DelayedEventTable.hpp | 138 +++ include/merovingian/database/Migration.hpp | 3 + .../database/PersistentDelayedEvent.hpp | 40 + .../merovingian/database/PersistentStore.hpp | 5 + include/merovingian/database/Schema.hpp | 2 +- migrations/003_delayed_events.sql | 5 + src/database/DelayedEventStatusText.cpp | 38 + src/database/DelayedEventStore.cpp | 266 ++++++ src/database/DelayedEventTable.cpp | 343 +++++++ src/database/Migration.cpp | 22 +- src/database/PostgresqlStore.cpp | 30 + src/database/Schema.cpp | 36 +- src/database/SqliteStore.cpp | 20 + src/database/meson.build | 3 + .../test_database_upgrade_flow.cpp | 41 +- .../test_persistent_homeserver_flow.cpp | 3 +- .../test_postgresql_store_behaviour_flow.cpp | 109 +++ tests/meson.build | 2 + tests/smoke/meson.build | 22 + tests/unit/test_database_persistence.cpp | 120 ++- tests/unit/test_delayed_event_store.cpp | 904 ++++++++++++++++++ tests/unit/test_delayed_event_table.cpp | 597 ++++++++++++ tests/unit/test_worker_table_load_profile.cpp | 4 + 27 files changed, 2847 insertions(+), 39 deletions(-) create mode 100644 include/merovingian/database/DelayedEventOutcomeResult.hpp create mode 100644 include/merovingian/database/DelayedEventStatus.hpp create mode 100644 include/merovingian/database/DelayedEventStatusText.hpp create mode 100644 include/merovingian/database/DelayedEventStore.hpp create mode 100644 include/merovingian/database/DelayedEventTable.hpp create mode 100644 include/merovingian/database/PersistentDelayedEvent.hpp create mode 100644 migrations/003_delayed_events.sql create mode 100644 src/database/DelayedEventStatusText.cpp create mode 100644 src/database/DelayedEventStore.cpp create mode 100644 src/database/DelayedEventTable.cpp create mode 100644 tests/unit/test_delayed_event_store.cpp create mode 100644 tests/unit/test_delayed_event_table.cpp diff --git a/include/merovingian/database/DelayedEventOutcomeResult.hpp b/include/merovingian/database/DelayedEventOutcomeResult.hpp new file mode 100644 index 000000000..588965412 --- /dev/null +++ b/include/merovingian/database/DelayedEventOutcomeResult.hpp @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +namespace merovingian::database +{ + +// What `record_delayed_event_outcome` did (ADR-0825). `recorded`: the row is finalised in the database and in memory. +// `refused`: nothing changed, because the row is absent or not scheduled or the arguments were not storable. +// `commit_failed`: the row IS finalised in memory but the database still holds it as scheduled, so the caller logs +// it and the next start-up sends the event again (at least once, ADR-0825). +enum class DelayedEventOutcomeResult +{ + recorded, + refused, + commit_failed, +}; + +} // namespace merovingian::database diff --git a/include/merovingian/database/DelayedEventStatus.hpp b/include/merovingian/database/DelayedEventStatus.hpp new file mode 100644 index 000000000..9fc48fcf4 --- /dev/null +++ b/include/merovingian/database/DelayedEventStatus.hpp @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +namespace merovingian::database +{ + +// Where a delayed event is in its life (migration 003, ADR-0824). `scheduled` waits for its time; the other three +// are final: `sent` became an event, `cancelled` was cancelled by its owner, `failed` was cancelled by the server +// because sending it failed (its row then holds the error). `DelayedEventStatusText.hpp` maps each to the text the +// `status` column holds. +enum class DelayedEventStatus +{ + scheduled, + sent, + cancelled, + failed, +}; + +} // namespace merovingian::database diff --git a/include/merovingian/database/DelayedEventStatusText.hpp b/include/merovingian/database/DelayedEventStatusText.hpp new file mode 100644 index 000000000..828ca09b1 --- /dev/null +++ b/include/merovingian/database/DelayedEventStatusText.hpp @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/database/DelayedEventStatus.hpp" + +#include +#include + +namespace merovingian::database +{ + +// The text the `delayed_events.status` column holds for `status`: `scheduled`, `sent`, `cancelled` or `failed`. +[[nodiscard]] auto delayed_event_status_text(DelayedEventStatus status) noexcept -> std::string_view; +// The status `text` names, byte for byte; nullopt for any other text, so a hydrating loader can refuse the row. +[[nodiscard]] auto parse_delayed_event_status(std::string_view text) noexcept -> std::optional; + +} // namespace merovingian::database diff --git a/include/merovingian/database/DelayedEventStore.hpp b/include/merovingian/database/DelayedEventStore.hpp new file mode 100644 index 000000000..a9625a9c8 --- /dev/null +++ b/include/merovingian/database/DelayedEventStore.hpp @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/database/DelayedEventOutcomeResult.hpp" +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/PersistentDelayedEvent.hpp" + +#include +#include +#include +#include +#include + +namespace merovingian::database +{ + +struct PersistentStore; + +// The delayed events of migration 003 (ADR-0824, ADR-0825). Every function here reads or writes +// `PersistentStore::delayed_events` and, for a write, the database in one transaction with bound parameters only. +// Memory changes only after the commit returns true, with the one stated exception of +// `record_delayed_event_outcome`. None takes a lock; in main the caller holds `runtime.mutex`. Nothing here logs a +// row's `content_json`. + +// Caps on what one row holds, enforced where the row is stored as well as where a request is parsed: an oversized row +// is refused, never truncated. +inline constexpr auto max_delayed_event_id_bytes = std::size_t{64U}; +// The event size limit (65,536 bytes serialised): content larger than this could never be sent. +inline constexpr auto max_delayed_event_content_bytes = std::size_t{65'536U}; +// A standard error object, which carries an errcode and a reason of at most 255 bytes. +inline constexpr auto max_delayed_event_error_bytes = std::size_t{1024U}; +// The user, device, room, event, event type, state key and transaction IDs. +inline constexpr auto max_delayed_event_field_bytes = std::size_t{255U}; +// The longest delay a row may hold: 31 days. The configured maximum is never above it. +inline constexpr auto max_delayed_event_delay_ms = std::uint64_t{2'678'400'000U}; + +// Stores a new scheduled row. False, writing nothing, when `delay_id`, `user_id`, `room_id`, `event_type` or +// `content_json` is empty, a field is over its cap, `delay_ms` is 0 or over `max_delayed_event_delay_ms`, the status is +// not `scheduled` (or the row carries a finalised time, event ID or error), the delay ID is already held, the +// transaction ID is already held for the same (user, device, room, event type), or the commit fails. It does NOT +// enforce the per-user cap: the caller does, because it must compute `Retry-After` from the user's rows. +[[nodiscard]] auto store_delayed_event(PersistentStore& store, PersistentDelayedEvent row) -> bool; + +// Sets `delayed_since_ms` of a scheduled row to `now_ms` (a restart). False when the row is absent, not scheduled, +// being sent, or the commit fails; then nothing changed. +[[nodiscard]] auto restart_delayed_event(PersistentStore& store, std::string_view delay_id, std::uint64_t now_ms) + -> bool; + +// Finalises a scheduled row as `cancelled` at `now_ms` (the owner's cancel). False when the row is absent, not +// scheduled, being sent, or the commit fails; then nothing changed. In the same transaction the user's oldest +// finalised rows are deleted so the user holds at most `max_finalised_per_user` finalised rows afterwards (the row +// finalised counts and is never the one evicted; callers pass 1 to 1000). +[[nodiscard]] auto cancel_delayed_event(PersistentStore& store, std::string_view delay_id, std::uint64_t now_ms, + std::size_t max_finalised_per_user) -> bool; + +// Records the outcome of an attempt to send a scheduled row: `outcome` is `sent` with the event's `event_id`, or +// `failed` with a standard error object `error_json` (the other field empty). Finalises at `now_ms` and evicts as +// `cancel_delayed_event` does, in one transaction. A row marked as being sent is the expected case. If the commit fails +// the in-memory row is STILL finalised and `commit_failed` is returned: this is the one place memory changes without +// a commit (ADR-0825). See the definition for why. +[[nodiscard]] auto record_delayed_event_outcome(PersistentStore& store, std::string_view delay_id, + DelayedEventStatus outcome, std::uint64_t now_ms, std::string event_id, + std::string error_json, std::size_t max_finalised_per_user) + -> DelayedEventOutcomeResult; + +// Deletes every row of `user_id`, scheduled and finalised, in one `DELETE ... WHERE user_id = $1`. True when the user +// has none (nothing is written) or the rows are gone; false when the commit fails, with memory unchanged. +[[nodiscard]] auto delete_delayed_events_of_user(PersistentStore& store, std::string_view user_id) -> bool; + +// Deletes finalised rows whose `finalised_at_ms` is below `cutoff_ms`, oldest first, at most `max_rows`, in one +// transaction. Returns how many were deleted, or nullopt when the commit failed (memory unchanged). +[[nodiscard]] auto prune_finalised_delayed_events(PersistentStore& store, std::uint64_t cutoff_ms, std::size_t max_rows) + -> std::optional; + +} // namespace merovingian::database diff --git a/include/merovingian/database/DelayedEventTable.hpp b/include/merovingian/database/DelayedEventTable.hpp new file mode 100644 index 000000000..49cbb601f --- /dev/null +++ b/include/merovingian/database/DelayedEventTable.hpp @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/ExaminedRows.hpp" +#include "merovingian/database/PersistentDelayedEvent.hpp" +#include "merovingian/database/StateGroupKeyHash.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace merovingian::database +{ + +// The in-memory copy of `delayed_events` (migration 003, ADR-0824). The class owns the rows and shows them read-only, +// so a writer that bypasses it does not compile and its indexes cannot drift from the rows: every mutation unindexes +// the row, changes it and indexes it again. Indexes: by delay ID (hash); per user, the scheduled rows by due time and +// the finalised rows by finalised time; every scheduled row that is not being sent by due time (`delayed_since_ms + +// delay_ms`, then delay ID), so the earliest due is O(1) and a restart O(log n); by (user, device, room, event type, +// transaction ID) for idempotency, only for rows with a transaction ID; by event ID for sent rows; and every +// finalised row by finalised time (the age prune). No reader scans the table. Nothing here is a secret, so keys are +// plain text. The table has no lock: a reader needs exclusion from writers (in main, `runtime.mutex`). A pointer it +// returns is valid until the next write. The defaulted copy and move keep every index valid: the indexes hold keys, +// never positions or pointers. +class DelayedEventTable final +{ +public: + // The row for `delay_id`, or nullptr. Does not check the owner: the management routes use `find_owned`. + [[nodiscard]] auto find(std::string_view delay_id) const -> PersistentDelayedEvent const*; + // The row for `delay_id` only when it belongs to `user_id`, else nullptr. The one place the ownership rule lives: + // another user's row and a row that does not exist are the same answer. + [[nodiscard]] auto find_owned(std::string_view user_id, std::string_view delay_id) const + -> PersistentDelayedEvent const*; + [[nodiscard]] auto size() const noexcept -> std::size_t + { + return m_rows.size(); + } + + // How many scheduled (not finalised) rows `user_id` holds, a being-sent row included. + [[nodiscard]] auto scheduled_count(std::string_view user_id) const -> std::size_t; + // The due time of the earliest-due scheduled row of `user_id` (a being-sent row included, for `Retry-After`), or + // nullopt when the user holds none. + [[nodiscard]] auto earliest_due_ms_of_user(std::string_view user_id) const -> std::optional; + // The scheduled row due first overall that is not marked as being sent, or nullptr. The order is `delayed_since_ms + // + delay_ms`, then delay ID. The caller compares its due time (`delayed_event_due_ms`) with the clock. + [[nodiscard]] auto earliest_due() const -> PersistentDelayedEvent const*; + // The row of (`user_id`, `device_id`, `room_id`, `event_type`, `txn_id`) of any status, or nullptr. Rows with no + // transaction ID are not in this index, so an empty `txn_id` finds nothing. + [[nodiscard]] auto find_by_transaction(std::string_view user_id, std::string_view device_id, + std::string_view room_id, std::string_view event_type, + std::string_view txn_id) const -> PersistentDelayedEvent const*; + // The sent row whose event is `event_id`, or nullptr. + [[nodiscard]] auto find_sent_by_event_id(std::string_view event_id) const -> PersistentDelayedEvent const*; + + // How many finalised rows `user_id` holds. + [[nodiscard]] auto finalised_count(std::string_view user_id) const -> std::size_t; + // At most `max` of `user_id`'s finalised rows, oldest first (by `finalised_at_ms`, then delay ID). + [[nodiscard]] auto finalised_of_user(std::string_view user_id, std::size_t max) const + -> std::vector; + // At most `max` finalised rows of every user whose `finalised_at_ms` is below `cutoff_ms`, oldest first (by + // `finalised_at_ms`, then delay ID). Costs the rows returned. + [[nodiscard]] auto finalised_before(std::uint64_t cutoff_ms, std::size_t max) const + -> std::vector; + // Whether `delay_id` is a row marked as being sent. + [[nodiscard]] auto is_being_sent(std::string_view delay_id) const -> bool; + + // Adds `row`, indexing it by its status. False, and nothing changes, when its delay ID is held, or it has a + // transaction ID and another row already holds the same (user, device, room, event type, transaction ID). + auto insert(PersistentDelayedEvent row) -> bool; + // Sets `delayed_since_ms` of a scheduled row, moving it in the due order. False when the row is absent or not + // scheduled. + auto restart(std::string_view delay_id, std::uint64_t delayed_since_ms) -> bool; + // Finalises a scheduled row: `status` (sent, cancelled or failed; scheduled is refused) with `finalised_at_ms`, + // `event_id` and `error_json`. Any being-sent mark of the row goes. False when the row is absent or not scheduled. + auto finalise(std::string_view delay_id, DelayedEventStatus status, std::uint64_t finalised_at_ms, + std::string event_id, std::string error_json) -> bool; + // Removes one row. False when absent. + auto erase(std::string_view delay_id) -> bool; + // Removes every row of `user_id`, returning how many. + auto erase_user(std::string_view user_id) -> std::size_t; + // Marks a scheduled row as being sent (memory only): it is not offered by `earliest_due`. False when the row is + // absent, not scheduled or already marked. + auto mark_being_sent(std::string_view delay_id) -> bool; + // Removes the mark; the row is offered again when still scheduled. False when it was not marked. + auto clear_being_sent(std::string_view delay_id) -> bool; + // Removes every row and mark, for hydration. + auto clear() noexcept -> void; + + // Rows every reader above looked at over the table's life, for tests that prove cost. + [[nodiscard]] auto rows_examined() const noexcept -> std::uint64_t + { + return m_rows_examined.value(); + } + +private: + template + using KeyedBy = std::unordered_map>; + using OrderKey = std::pair; + + // One user's rows: the scheduled ones by due time and the finalised ones by finalised time. + struct UserBook final + { + std::set scheduled{}; + std::set finalised{}; + }; + + auto index_row(PersistentDelayedEvent const& row) -> void; + auto unindex_row(PersistentDelayedEvent const& row) -> void; + + KeyedBy m_rows{}; + KeyedBy m_users{}; + // Every scheduled row not being sent, by (due time, delay ID). + std::set m_due{}; + // Every finalised row, by (finalised time, delay ID). + std::set m_finalised{}; + // (user NUL device NUL room NUL type NUL txn) -> delay ID, rows with a transaction ID only. + KeyedBy m_by_transaction{}; + // Event ID -> delay ID of the sent row. + KeyedBy m_by_event_id{}; + KeyedBy m_being_sent{}; + ExaminedRows m_rows_examined{}; +}; + +// The time a scheduled row is due: `delayed_since_ms + delay_ms`, saturating at the largest value. The store caps +// the delay at 31 days, so the sum cannot overflow for any real clock, but a row read from a database or built by a +// test is never trusted to be within it. +[[nodiscard]] auto delayed_event_due_ms(PersistentDelayedEvent const& row) noexcept -> std::uint64_t; + +} // namespace merovingian::database diff --git a/include/merovingian/database/Migration.hpp b/include/merovingian/database/Migration.hpp index 7ee8dd851..19c5ec693 100644 --- a/include/merovingian/database/Migration.hpp +++ b/include/merovingian/database/Migration.hpp @@ -26,6 +26,9 @@ namespace merovingian::database // Migration 002 (1.1.0): the OAuth 2.0 client and session tables (ADR-0776, ADR-0778). [[nodiscard]] auto oauth_tables_migration() -> MigrationStep; [[nodiscard]] auto downgrade_oauth_tables_migration() -> MigrationStep; +// Migration 003 (1.1.2): the delayed-events table (ADR-0824). +[[nodiscard]] auto delayed_events_migration() -> MigrationStep; +[[nodiscard]] auto downgrade_delayed_events_migration() -> MigrationStep; [[nodiscard]] auto upgrade_migration_catalog() -> std::vector; [[nodiscard]] auto downgrade_migration_catalog() -> std::vector; [[nodiscard]] auto migration_plan_for(SchemaState const& state) -> MigrationPlan; diff --git a/include/merovingian/database/PersistentDelayedEvent.hpp b/include/merovingian/database/PersistentDelayedEvent.hpp new file mode 100644 index 000000000..dc9a393d5 --- /dev/null +++ b/include/merovingian/database/PersistentDelayedEvent.hpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/database/DelayedEventStatus.hpp" + +#include +#include + +namespace merovingian::database +{ + +// Migration 003 (ADR-0824): one delayed event (`delayed_events`), an event a user scheduled to be sent later +// (MSC4140). `device_id` is the scheduling request's device, empty when its token had none; `txn_id` is empty for +// the legacy state form, which has none; `via_appservice` records that an application service scheduled it, which the +// send passes on. `delayed_since_ms` is when the timer last started: the event is due at `delayed_since_ms + +// delay_ms`. `finalised_at_ms` is 0 while scheduled; `event_id` is empty unless sent; `error_json` is empty unless +// failed and then holds the standard error object. Times are wall-clock milliseconds since the Unix epoch. A row +// never changes except through `DelayedEventTable`. +struct PersistentDelayedEvent final +{ + std::string delay_id{}; + std::string user_id{}; + std::string device_id{}; + std::string room_id{}; + std::string event_type{}; + bool is_state{false}; + std::string state_key{}; + std::string content_json{}; + std::string txn_id{}; + bool via_appservice{false}; + std::uint64_t delay_ms{0U}; + std::uint64_t delayed_since_ms{0U}; + DelayedEventStatus status{DelayedEventStatus::scheduled}; + std::uint64_t finalised_at_ms{0U}; + std::string event_id{}; + std::string error_json{}; +}; + +} // namespace merovingian::database diff --git a/include/merovingian/database/PersistentStore.hpp b/include/merovingian/database/PersistentStore.hpp index 72fa88c9f..91b8fe475 100644 --- a/include/merovingian/database/PersistentStore.hpp +++ b/include/merovingian/database/PersistentStore.hpp @@ -7,6 +7,7 @@ #include "merovingian/database/CredentialTable.hpp" #include "merovingian/database/DeferredPduAwaits.hpp" #include "merovingian/database/DeferredPduUpsert.hpp" +#include "merovingian/database/DelayedEventTable.hpp" #include "merovingian/database/EventStateGroupTable.hpp" #include "merovingian/database/EventTable.hpp" #include "merovingian/database/ExaminedRows.hpp" @@ -574,6 +575,10 @@ struct PersistentStore final // Migration 002 (ADR-0776, ADR-0777): `oauth_clients` and `oauth_sessions`, with their indexes. Read and write // through OAuthStore.hpp; `delete_device` removes a device's session in the transaction that deletes the device. OAuthTables oauth{}; + // Migration 003 (ADR-0824): `delayed_events`, MSC4140 events scheduled to be sent later. Read and write through + // DelayedEventStore.hpp; it has no lock of its own, so in main `runtime.mutex` serialises it. The federation worker + // never holds it. + DelayedEventTable delayed_events{}; std::vector appservice_txn_cursors{}; // Events queued for appservices, kept sorted by (appservice_id, stream_ordering). See // PersistentAppserviceOutboxEntry. diff --git a/include/merovingian/database/Schema.hpp b/include/merovingian/database/Schema.hpp index d88cc7b86..1b08d11d9 100644 --- a/include/merovingian/database/Schema.hpp +++ b/include/merovingian/database/Schema.hpp @@ -16,7 +16,7 @@ namespace merovingian::database [[nodiscard]] auto current_schema_version() noexcept -> std::uint32_t; // The schema split (ADR-0778). The tables and indexes migration `version` creates: version 1 is the frozen v1 -// schema, version 2 the OAuth 2.0 tables; any other version adds none. A migration step is built from these. +// schema, version 2 the OAuth 2.0 tables, version 3 the delayed-events table; any other version adds none. A migration step is built from these. [[nodiscard]] auto schema_definitions_added_by(std::uint32_t version) -> std::vector; [[nodiscard]] auto schema_indexes_added_by(std::uint32_t version) -> std::vector; // Exactly what v1 created, which the SQLite and PostgreSQL bootstraps build before any later step runs. diff --git a/migrations/003_delayed_events.sql b/migrations/003_delayed_events.sql new file mode 100644 index 000000000..dc8fd486f --- /dev/null +++ b/migrations/003_delayed_events.sql @@ -0,0 +1,5 @@ +-- merovingian-migration version=3 name=delayed_events direction=upgrade +-- statement create_delayed_events +CREATE TABLE "delayed_events" (delay_id TEXT PRIMARY KEY, user_id TEXT NOT NULL, device_id TEXT NOT NULL, room_id TEXT NOT NULL, event_type TEXT NOT NULL, is_state TEXT NOT NULL, state_key TEXT NOT NULL, content_json TEXT NOT NULL, txn_id TEXT NOT NULL, via_appservice TEXT NOT NULL, delay_ms TEXT NOT NULL, delayed_since_ms TEXT NOT NULL, status TEXT NOT NULL, finalised_at_ms TEXT NOT NULL, event_id TEXT NOT NULL, error_json TEXT NOT NULL) +-- statement create_idx_delayed_events_user_id +CREATE INDEX idx_delayed_events_user_id ON delayed_events (user_id) diff --git a/src/database/DelayedEventStatusText.cpp b/src/database/DelayedEventStatusText.cpp new file mode 100644 index 000000000..5081d1d08 --- /dev/null +++ b/src/database/DelayedEventStatusText.cpp @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "merovingian/database/DelayedEventStatusText.hpp" + +namespace merovingian::database +{ + +auto delayed_event_status_text(DelayedEventStatus status) noexcept -> std::string_view +{ + switch (status) + { + case DelayedEventStatus::scheduled: + return "scheduled"; + case DelayedEventStatus::sent: + return "sent"; + case DelayedEventStatus::cancelled: + return "cancelled"; + case DelayedEventStatus::failed: + return "failed"; + } + return "scheduled"; +} + +auto parse_delayed_event_status(std::string_view text) noexcept -> std::optional +{ + for (auto const status : {DelayedEventStatus::scheduled, DelayedEventStatus::sent, DelayedEventStatus::cancelled, + DelayedEventStatus::failed}) + { + if (text == delayed_event_status_text(status)) + { + return status; + } + } + return std::nullopt; +} + +} // namespace merovingian::database diff --git a/src/database/DelayedEventStore.cpp b/src/database/DelayedEventStore.cpp new file mode 100644 index 000000000..672f04e13 --- /dev/null +++ b/src/database/DelayedEventStore.cpp @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "merovingian/database/DelayedEventStore.hpp" + +#include "merovingian/database/BoundValue.hpp" +#include "merovingian/database/DelayedEventStatusText.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentStore.hpp" +#include "merovingian/database/PreparedStatement.hpp" + +#include +#include +#include +#include +#include + +namespace merovingian::database +{ +namespace +{ + + // A prepared statement takes at most 128 parameters, so a longer list of IDs is several DELETEs in the one + // transaction, never one statement per row. + constexpr auto delete_ids_per_statement = std::size_t{128U}; + + [[nodiscard]] auto text(std::string_view value) -> BoundValue + { + return {std::string{value}, false}; + } + + [[nodiscard]] auto number(std::uint64_t value) -> BoundValue + { + return text(std::to_string(value)); + } + + // Booleans are stored as the text "true" and "false", as users.deactivated is. + [[nodiscard]] auto flag(bool value) -> BoundValue + { + return text(value ? "true" : "false"); + } + + [[nodiscard]] auto within(std::string_view value, std::size_t cap) noexcept -> bool + { + return value.size() <= cap; + } + + [[nodiscard]] auto row_is_storable(PersistentDelayedEvent const& row) noexcept -> bool + { + return !row.delay_id.empty() && !row.user_id.empty() && !row.room_id.empty() && !row.event_type.empty() && + !row.content_json.empty() && within(row.delay_id, max_delayed_event_id_bytes) && + within(row.content_json, max_delayed_event_content_bytes) && + within(row.user_id, max_delayed_event_field_bytes) && + within(row.device_id, max_delayed_event_field_bytes) && + within(row.room_id, max_delayed_event_field_bytes) && + within(row.event_type, max_delayed_event_field_bytes) && + within(row.state_key, max_delayed_event_field_bytes) && + within(row.txn_id, max_delayed_event_field_bytes) && row.delay_ms != 0U && + row.delay_ms <= max_delayed_event_delay_ms && row.status == DelayedEventStatus::scheduled && + row.finalised_at_ms == 0U && row.event_id.empty() && row.error_json.empty(); + } + + [[nodiscard]] auto delete_rows_statement(std::vector const& delay_ids, std::size_t first, + std::size_t count) -> PreparedStatement + { + auto sql = std::string{"DELETE FROM delayed_events WHERE delay_id IN ("}; + auto parameters = std::vector{}; + parameters.reserve(count); + for (auto index = first; index < first + count; ++index) + { + sql += parameters.empty() ? "$" : ", $"; + sql += std::to_string(parameters.size() + 1U); + parameters.push_back(text(delay_ids[index])); + } + sql += ")"; + return {"delete_delayed_events", std::move(sql), std::move(parameters)}; + } + + // Appends the DELETEs for `delay_ids`, batched. + auto append_delete_statements(std::vector& statements, std::vector const& delay_ids) + -> void + { + for (auto first = std::size_t{0U}; first < delay_ids.size(); first += delete_ids_per_statement) + { + statements.push_back( + delete_rows_statement(delay_ids, first, std::min(delete_ids_per_statement, delay_ids.size() - first))); + } + } + + // The delay IDs of the oldest finalised rows of `user_id` to delete so that, once one more row is finalised, the + // user holds at most `bound` finalised rows. The row being finalised is still scheduled, so it is never among them. + [[nodiscard]] auto evictions_for(DelayedEventTable const& table, std::string_view user_id, std::size_t bound) + -> std::vector + { + auto doomed = std::vector{}; + auto const held = table.finalised_count(user_id); + if (held + 1U <= bound) + { + return doomed; + } + auto const excess = held + 1U - bound; + for (auto const* row : table.finalised_of_user(user_id, std::min(excess, held))) + { + doomed.push_back(row->delay_id); + } + return doomed; + } + + // The UPDATE that finalises one row. + [[nodiscard]] auto finalise_statement(std::string_view delay_id, DelayedEventStatus status, + std::uint64_t finalised_at_ms, std::string_view event_id, + std::string_view error_json) -> PreparedStatement + { + return { + "finalise_delayed_event", + "UPDATE delayed_events SET status = $1, finalised_at_ms = $2, event_id = $3, error_json = $4 WHERE " + "delay_id = $5", + {text(delayed_event_status_text(status)), number(finalised_at_ms), text(event_id), text(error_json), + text(delay_id)} + }; + } + +} // namespace + +auto store_delayed_event(PersistentStore& store, PersistentDelayedEvent row) -> bool +{ + if (!row_is_storable(row) || store.delayed_events.find(row.delay_id) != nullptr || + store.delayed_events.find_by_transaction(row.user_id, row.device_id, row.room_id, row.event_type, row.txn_id) != + nullptr) + { + return false; + } + auto const statement = PreparedStatement{ + "insert_delayed_event", + "INSERT INTO delayed_events (delay_id, user_id, device_id, room_id, event_type, is_state, state_key, " + "content_json, txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " + "error_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)", + {text(row.delay_id), text(row.user_id), text(row.device_id), text(row.room_id), text(row.event_type), + flag(row.is_state), text(row.state_key), text(row.content_json), text(row.txn_id), flag(row.via_appservice), + number(row.delay_ms), number(row.delayed_since_ms), text(delayed_event_status_text(row.status)), + number(row.finalised_at_ms), text(row.event_id), text(row.error_json)} + }; + if (!commit_persistent_transaction(store, {statement})) + { + return false; + } + return store.delayed_events.insert(std::move(row)); +} + +auto restart_delayed_event(PersistentStore& store, std::string_view delay_id, std::uint64_t now_ms) -> bool +{ + auto const* row = store.delayed_events.find(delay_id); + if (row == nullptr || row->status != DelayedEventStatus::scheduled || store.delayed_events.is_being_sent(delay_id)) + { + return false; + } + auto const statement = PreparedStatement{ + "restart_delayed_event", + "UPDATE delayed_events SET delayed_since_ms = $1 WHERE delay_id = $2", + {number(now_ms), text(delay_id)} + }; + if (!commit_persistent_transaction(store, {statement})) + { + return false; + } + return store.delayed_events.restart(delay_id, now_ms); +} + +auto cancel_delayed_event(PersistentStore& store, std::string_view delay_id, std::uint64_t now_ms, + std::size_t max_finalised_per_user) -> bool +{ + auto const* row = store.delayed_events.find(delay_id); + if (row == nullptr || row->status != DelayedEventStatus::scheduled || store.delayed_events.is_being_sent(delay_id)) + { + return false; + } + auto const doomed = evictions_for(store.delayed_events, row->user_id, max_finalised_per_user); + auto statements = std::vector{}; + statements.push_back(finalise_statement(delay_id, DelayedEventStatus::cancelled, now_ms, {}, {})); + append_delete_statements(statements, doomed); + if (!commit_persistent_transaction(store, statements)) + { + return false; + } + for (auto const& evicted : doomed) + { + std::ignore = store.delayed_events.erase(evicted); + } + return store.delayed_events.finalise(delay_id, DelayedEventStatus::cancelled, now_ms, {}, {}); +} + +auto record_delayed_event_outcome(PersistentStore& store, std::string_view delay_id, DelayedEventStatus outcome, + std::uint64_t now_ms, std::string event_id, std::string error_json, + std::size_t max_finalised_per_user) -> DelayedEventOutcomeResult +{ + auto const* row = store.delayed_events.find(delay_id); + auto const shape_is_right = (outcome == DelayedEventStatus::sent && !event_id.empty() && error_json.empty()) || + (outcome == DelayedEventStatus::failed && event_id.empty() && !error_json.empty()); + if (row == nullptr || row->status != DelayedEventStatus::scheduled || !shape_is_right || + !within(event_id, max_delayed_event_field_bytes) || !within(error_json, max_delayed_event_error_bytes)) + { + return DelayedEventOutcomeResult::refused; + } + auto const doomed = evictions_for(store.delayed_events, row->user_id, max_finalised_per_user); + auto statements = std::vector{}; + statements.push_back(finalise_statement(delay_id, outcome, now_ms, event_id, error_json)); + append_delete_statements(statements, doomed); + auto const committed = commit_persistent_transaction(store, statements); + // The one place memory changes without a commit (ADR-0825). By now the send was attempted, and memory must say + // what happened: leaving the row scheduled would send it again on every run of the job. When the commit failed, + // the database row still reads "scheduled", so the next start-up sends the event once more, which the + // at-least-once rule allows; the caller is told, so it can log it. The evictions follow the commit only: a + // failed commit deleted nothing, so no finalised row leaves memory either. + if (committed) + { + for (auto const& evicted : doomed) + { + std::ignore = store.delayed_events.erase(evicted); + } + } + std::ignore = store.delayed_events.finalise(delay_id, outcome, now_ms, std::move(event_id), std::move(error_json)); + return committed ? DelayedEventOutcomeResult::recorded : DelayedEventOutcomeResult::commit_failed; +} + +auto delete_delayed_events_of_user(PersistentStore& store, std::string_view user_id) -> bool +{ + if (store.delayed_events.scheduled_count(user_id) == 0U && store.delayed_events.finalised_count(user_id) == 0U) + { + return true; + } + auto const statement = PreparedStatement{ + "delete_delayed_events_of_user", "DELETE FROM delayed_events WHERE user_id = $1", {text(user_id)}}; + if (!commit_persistent_transaction(store, {statement})) + { + return false; + } + std::ignore = store.delayed_events.erase_user(user_id); + return true; +} + +auto prune_finalised_delayed_events(PersistentStore& store, std::uint64_t cutoff_ms, std::size_t max_rows) + -> std::optional +{ + auto doomed = std::vector{}; + for (auto const* row : store.delayed_events.finalised_before(cutoff_ms, max_rows)) + { + doomed.push_back(row->delay_id); + } + if (doomed.empty()) + { + return std::size_t{0U}; + } + auto statements = std::vector{}; + append_delete_statements(statements, doomed); + if (!commit_persistent_transaction(store, statements)) + { + return std::nullopt; + } + for (auto const& delay_id : doomed) + { + std::ignore = store.delayed_events.erase(delay_id); + } + return doomed.size(); +} + +} // namespace merovingian::database diff --git a/src/database/DelayedEventTable.cpp b/src/database/DelayedEventTable.cpp new file mode 100644 index 000000000..7973257ec --- /dev/null +++ b/src/database/DelayedEventTable.cpp @@ -0,0 +1,343 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "merovingian/database/DelayedEventTable.hpp" + +#include +#include + +namespace merovingian::database +{ +namespace +{ + + // A key made of the parts separated by NUL, which no Matrix identifier, event type or transaction ID may hold, so + // two different tuples never make the same key. + [[nodiscard]] auto transaction_key(std::string_view user_id, std::string_view device_id, std::string_view room_id, + std::string_view event_type, std::string_view txn_id) -> std::string + { + auto key = std::string{user_id}; + for (auto const part : {device_id, room_id, event_type, txn_id}) + { + key.push_back('\0'); + key.append(part); + } + return key; + } + + [[nodiscard]] auto transaction_key_of(PersistentDelayedEvent const& row) -> std::string + { + return transaction_key(row.user_id, row.device_id, row.room_id, row.event_type, row.txn_id); + } + +} // namespace + +auto delayed_event_due_ms(PersistentDelayedEvent const& row) noexcept -> std::uint64_t +{ + // Saturating add: a sum that would wrap is the largest value instead, so a corrupt row is due last, never first. + constexpr auto largest = std::numeric_limits::max(); + return row.delayed_since_ms > largest - row.delay_ms ? largest : row.delayed_since_ms + row.delay_ms; +} + +auto DelayedEventTable::find(std::string_view delay_id) const -> PersistentDelayedEvent const* +{ + auto const found = m_rows.find(delay_id); + return found == m_rows.end() ? nullptr : &found->second; +} + +auto DelayedEventTable::find_owned(std::string_view user_id, std::string_view delay_id) const + -> PersistentDelayedEvent const* +{ + auto const* row = find(delay_id); + return row != nullptr && row->user_id == user_id ? row : nullptr; +} + +auto DelayedEventTable::scheduled_count(std::string_view user_id) const -> std::size_t +{ + auto const found = m_users.find(user_id); + return found == m_users.end() ? 0U : found->second.scheduled.size(); +} + +auto DelayedEventTable::earliest_due_ms_of_user(std::string_view user_id) const -> std::optional +{ + auto const found = m_users.find(user_id); + if (found == m_users.end() || found->second.scheduled.empty()) + { + return std::nullopt; + } + m_rows_examined.add(1U); + return found->second.scheduled.begin()->first; +} + +auto DelayedEventTable::earliest_due() const -> PersistentDelayedEvent const* +{ + if (m_due.empty()) + { + return nullptr; + } + m_rows_examined.add(1U); + return find(m_due.begin()->second); +} + +auto DelayedEventTable::find_by_transaction(std::string_view user_id, std::string_view device_id, + std::string_view room_id, std::string_view event_type, + std::string_view txn_id) const -> PersistentDelayedEvent const* +{ + if (txn_id.empty()) + { + return nullptr; + } + auto const found = m_by_transaction.find(transaction_key(user_id, device_id, room_id, event_type, txn_id)); + if (found == m_by_transaction.end()) + { + return nullptr; + } + m_rows_examined.add(1U); + return find(found->second); +} + +auto DelayedEventTable::find_sent_by_event_id(std::string_view event_id) const -> PersistentDelayedEvent const* +{ + auto const found = m_by_event_id.find(event_id); + if (found == m_by_event_id.end()) + { + return nullptr; + } + m_rows_examined.add(1U); + return find(found->second); +} + +auto DelayedEventTable::finalised_count(std::string_view user_id) const -> std::size_t +{ + auto const found = m_users.find(user_id); + return found == m_users.end() ? 0U : found->second.finalised.size(); +} + +auto DelayedEventTable::finalised_of_user(std::string_view user_id, std::size_t max) const + -> std::vector +{ + auto rows = std::vector{}; + auto const found = m_users.find(user_id); + if (found == m_users.end()) + { + return rows; + } + for (auto const& [finalised_at_ms, delay_id] : found->second.finalised) + { + if (rows.size() >= max) + { + break; + } + rows.push_back(find(delay_id)); + } + m_rows_examined.add(rows.size()); + return rows; +} + +auto DelayedEventTable::finalised_before(std::uint64_t cutoff_ms, std::size_t max) const + -> std::vector +{ + auto rows = std::vector{}; + for (auto const& [finalised_at_ms, delay_id] : m_finalised) + { + if (rows.size() >= max || finalised_at_ms >= cutoff_ms) + { + break; + } + rows.push_back(find(delay_id)); + } + m_rows_examined.add(rows.size()); + return rows; +} + +auto DelayedEventTable::is_being_sent(std::string_view delay_id) const -> bool +{ + return m_being_sent.contains(delay_id); +} + +auto DelayedEventTable::index_row(PersistentDelayedEvent const& row) -> void +{ + auto& book = m_users[row.user_id]; + if (row.status == DelayedEventStatus::scheduled) + { + auto const due = OrderKey{delayed_event_due_ms(row), row.delay_id}; + book.scheduled.insert(due); + if (!m_being_sent.contains(row.delay_id)) + { + m_due.insert(due); + } + } + else + { + auto const finalised = OrderKey{row.finalised_at_ms, row.delay_id}; + book.finalised.insert(finalised); + m_finalised.insert(finalised); + } + if (!row.txn_id.empty()) + { + m_by_transaction.emplace(transaction_key_of(row), row.delay_id); + } + if (row.status == DelayedEventStatus::sent && !row.event_id.empty()) + { + // The first sent row to name an event keeps the index entry: an event ID belongs to one event, so a second + // claimant is a corrupt row and the lookup stays with the first. + m_by_event_id.emplace(row.event_id, row.delay_id); + } +} + +auto DelayedEventTable::unindex_row(PersistentDelayedEvent const& row) -> void +{ + if (auto const book = m_users.find(row.user_id); book != m_users.end()) + { + book->second.scheduled.erase({delayed_event_due_ms(row), row.delay_id}); + book->second.finalised.erase({row.finalised_at_ms, row.delay_id}); + if (book->second.scheduled.empty() && book->second.finalised.empty()) + { + m_users.erase(book); + } + } + m_due.erase({delayed_event_due_ms(row), row.delay_id}); + m_finalised.erase({row.finalised_at_ms, row.delay_id}); + if (!row.txn_id.empty()) + { + if (auto const found = m_by_transaction.find(transaction_key_of(row)); + found != m_by_transaction.end() && found->second == row.delay_id) + { + m_by_transaction.erase(found); + } + } + if (!row.event_id.empty()) + { + if (auto const found = m_by_event_id.find(row.event_id); + found != m_by_event_id.end() && found->second == row.delay_id) + { + m_by_event_id.erase(found); + } + } +} + +auto DelayedEventTable::insert(PersistentDelayedEvent row) -> bool +{ + if (m_rows.contains(row.delay_id) || (!row.txn_id.empty() && m_by_transaction.contains(transaction_key_of(row)))) + { + return false; + } + auto key = row.delay_id; + auto const [position, inserted] = m_rows.emplace(std::move(key), std::move(row)); + if (inserted) + { + index_row(position->second); + } + return inserted; +} + +auto DelayedEventTable::restart(std::string_view delay_id, std::uint64_t delayed_since_ms) -> bool +{ + auto const found = m_rows.find(delay_id); + if (found == m_rows.end() || found->second.status != DelayedEventStatus::scheduled) + { + return false; + } + unindex_row(found->second); + found->second.delayed_since_ms = delayed_since_ms; + index_row(found->second); + return true; +} + +auto DelayedEventTable::finalise(std::string_view delay_id, DelayedEventStatus status, std::uint64_t finalised_at_ms, + std::string event_id, std::string error_json) -> bool +{ + auto const found = m_rows.find(delay_id); + if (found == m_rows.end() || found->second.status != DelayedEventStatus::scheduled || + status == DelayedEventStatus::scheduled) + { + return false; + } + unindex_row(found->second); + m_being_sent.erase(found->first); + found->second.status = status; + found->second.finalised_at_ms = finalised_at_ms; + found->second.event_id = std::move(event_id); + found->second.error_json = std::move(error_json); + index_row(found->second); + return true; +} + +auto DelayedEventTable::erase(std::string_view delay_id) -> bool +{ + auto const found = m_rows.find(delay_id); + if (found == m_rows.end()) + { + return false; + } + unindex_row(found->second); + m_being_sent.erase(found->first); + m_rows.erase(found); + return true; +} + +auto DelayedEventTable::erase_user(std::string_view user_id) -> std::size_t +{ + auto const book = m_users.find(user_id); + if (book == m_users.end()) + { + return 0U; + } + auto ids = std::vector{}; + ids.reserve(book->second.scheduled.size() + book->second.finalised.size()); + for (auto const& [when, delay_id] : book->second.scheduled) + { + ids.push_back(delay_id); + } + for (auto const& [when, delay_id] : book->second.finalised) + { + ids.push_back(delay_id); + } + for (auto const& delay_id : ids) + { + std::ignore = erase(delay_id); + } + return ids.size(); +} + +auto DelayedEventTable::mark_being_sent(std::string_view delay_id) -> bool +{ + auto const found = m_rows.find(delay_id); + if (found == m_rows.end() || found->second.status != DelayedEventStatus::scheduled || + m_being_sent.contains(delay_id)) + { + return false; + } + m_due.erase({delayed_event_due_ms(found->second), found->second.delay_id}); + m_being_sent.emplace(found->second.delay_id, true); + return true; +} + +auto DelayedEventTable::clear_being_sent(std::string_view delay_id) -> bool +{ + auto const marked = m_being_sent.find(delay_id); + if (marked == m_being_sent.end()) + { + return false; + } + m_being_sent.erase(marked); + if (auto const found = m_rows.find(delay_id); + found != m_rows.end() && found->second.status == DelayedEventStatus::scheduled) + { + m_due.insert({delayed_event_due_ms(found->second), found->second.delay_id}); + } + return true; +} + +auto DelayedEventTable::clear() noexcept -> void +{ + m_rows.clear(); + m_users.clear(); + m_due.clear(); + m_finalised.clear(); + m_by_transaction.clear(); + m_by_event_id.clear(); + m_being_sent.clear(); +} + +} // namespace merovingian::database diff --git a/src/database/Migration.cpp b/src/database/Migration.cpp index 67dd06f39..b79af4514 100644 --- a/src/database/Migration.cpp +++ b/src/database/Migration.cpp @@ -38,6 +38,9 @@ namespace // Migration 002 (ADR-0776): migrations/002_oauth_tables.sql records it. constexpr auto oauth_tables_version = std::uint32_t{2U}; constexpr auto oauth_tables_name = std::string_view{"oauth_tables"}; + // Migration 003 (ADR-0824): migrations/003_delayed_events.sql records it. + constexpr auto delayed_events_version = std::uint32_t{3U}; + constexpr auto delayed_events_name = std::string_view{"delayed_events"}; [[nodiscard]] auto make_create_table_statement(SchemaTableDefinition const& table) -> std::optional @@ -333,17 +336,28 @@ auto downgrade_oauth_tables_migration() -> MigrationStep return schema_step_dropping(oauth_tables_version, "drop_oauth_tables"); } -// The next migration is v3: add its upgrade step at the end here, its downgrade -// (to v2) at the front of the downgrade catalog, and its tables to +auto delayed_events_migration() -> MigrationStep +{ + return schema_step_creating(delayed_events_version, delayed_events_name); +} + +auto downgrade_delayed_events_migration() -> MigrationStep +{ + return schema_step_dropping(delayed_events_version, "drop_delayed_events"); +} + +// The next migration is v4: add its upgrade step at the end here, its downgrade +// (to v3) at the front of the downgrade catalog, and its tables to // schema_definitions_added_by. Never change an earlier step. auto upgrade_migration_catalog() -> std::vector { - return {initial_schema_migration(), oauth_tables_migration()}; + return {initial_schema_migration(), oauth_tables_migration(), delayed_events_migration()}; } auto downgrade_migration_catalog() -> std::vector { - return {downgrade_oauth_tables_migration(), downgrade_initial_schema_migration()}; + return {downgrade_delayed_events_migration(), downgrade_oauth_tables_migration(), + downgrade_initial_schema_migration()}; } auto migration_plan_between(std::uint32_t current_version, std::uint32_t target_version) -> MigrationPlan diff --git a/src/database/PostgresqlStore.cpp b/src/database/PostgresqlStore.cpp index 292db041f..1eb80fbf0 100644 --- a/src/database/PostgresqlStore.cpp +++ b/src/database/PostgresqlStore.cpp @@ -8,6 +8,7 @@ #include "merovingian/canonicaljson/Parser.hpp" #include "merovingian/canonicaljson/Serializer.hpp" #include "merovingian/database/ConnectionPool.hpp" +#include "merovingian/database/DelayedEventStatusText.hpp" #include "merovingian/database/Migration.hpp" #include "merovingian/database/PostgresqlConnection.hpp" #include "merovingian/database/PostgresqlConnectionDeleter.hpp" @@ -1420,6 +1421,35 @@ namespace } } + // Migration 003 (ADR-0824): delayed events, main only (the federation worker sends no events). A row whose + // status text is not one of the four, or that duplicates a delay ID or a transaction key, is skipped. + if (table_load_profile_includes("delayed_events", profile)) + { + auto delayed = query_rows(connection, "postgresql_load_delayed_events", + "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, " + "content_json, txn_id, via_appservice, delay_ms, delayed_since_ms, status, " + "finalised_at_ms, event_id, error_json FROM delayed_events ORDER BY delay_id"); + if (!delayed.ok) + { + return false; + } + for (auto const& row : delayed.rows) + { + if (row.size() < 16U) + { + continue; + } + auto const status = parse_delayed_event_status(row[12]); + if (status.has_value()) + { + std::ignore = store.delayed_events.insert( + {row[0], row[1], row[2], row[3], row[4], text_is_true(row[5]), row[6], row[7], row[8], + text_is_true(row[9]), parse_u64(row[10]), parse_u64(row[11]), *status, parse_u64(row[13]), + row[14], row[15]}); + } + } + } + if (table_load_profile_includes("appservice_txn_cursor", profile)) { auto appservice_txn_cursor_result = diff --git a/src/database/Schema.cpp b/src/database/Schema.cpp index 48d5789f8..4256772ae 100644 --- a/src/database/Schema.cpp +++ b/src/database/Schema.cpp @@ -22,7 +22,7 @@ namespace // never an edit to core_tables that changes what v1 creates. Each later // version's tables and indexes are an array of their own, below core_indexes // (ADR-0778). - constexpr auto schema_version = std::uint32_t{2U}; + constexpr auto schema_version = std::uint32_t{3U}; constexpr auto core_tables = std::array{ SchemaTableDefinition{"schema_migrations", @@ -274,6 +274,25 @@ namespace SchemaIndexDefinition{"idx_oauth_sessions_client_id", "oauth_sessions", "client_id"}, }; + // Migration 003 (ADR-0824): MSC4140 delayed events. Every column is TEXT like the rest of the schema: numbers are + // decimal text and booleans are 'true' / 'false'. No foreign key: the store keeps its rules in code, and a row + // outlives its device (a delayed event belongs to the user, not to the device that scheduled it). + constexpr auto delayed_event_tables = std::array{ + SchemaTableDefinition{"delayed_events", + "delay_id TEXT PRIMARY KEY, user_id TEXT NOT NULL, device_id TEXT NOT NULL, " + "room_id TEXT NOT NULL, event_type TEXT NOT NULL, is_state TEXT NOT NULL, " + "state_key TEXT NOT NULL, content_json TEXT NOT NULL, txn_id TEXT NOT NULL, " + "via_appservice TEXT NOT NULL, delay_ms TEXT NOT NULL, delayed_since_ms TEXT NOT NULL, " + "status TEXT NOT NULL, finalised_at_ms TEXT NOT NULL, event_id TEXT NOT NULL, " + "error_json TEXT NOT NULL"}, + }; + + // The delete at account deactivation reads a user's rows by user ID; the store answers every other lookup from + // memory. + constexpr auto delayed_event_indexes = std::array{ + SchemaIndexDefinition{"idx_delayed_events_user_id", "delayed_events", "user_id"}, + }; + template auto append_all(std::vector& into, std::array const& from) -> void { @@ -285,7 +304,8 @@ namespace auto const same = [&index](SchemaIndexDefinition const& known) { return known.name == index.name && known.table == index.table && known.columns_sql == index.columns_sql; }; - return std::ranges::any_of(core_indexes, same) || std::ranges::any_of(oauth_indexes, same); + return std::ranges::any_of(core_indexes, same) || std::ranges::any_of(oauth_indexes, same) || + std::ranges::any_of(delayed_event_indexes, same); } [[nodiscard]] auto index_sql(SchemaIndexDefinition const& index, std::string_view prefix) @@ -327,6 +347,10 @@ auto schema_definitions_added_by(std::uint32_t version) -> std::vector std::vector std::optio { return *oauth; } + if (auto const delayed = std::ranges::find_if(delayed_event_tables, named); delayed != delayed_event_tables.end()) + { + return *delayed; + } return std::nullopt; } diff --git a/src/database/SqliteStore.cpp b/src/database/SqliteStore.cpp index 8d4aa3822..e69f866b0 100644 --- a/src/database/SqliteStore.cpp +++ b/src/database/SqliteStore.cpp @@ -7,6 +7,7 @@ #include "merovingian/canonicaljson/Serializer.hpp" #include "merovingian/core/FileDescriptor.hpp" #include "merovingian/database/ConnectionPool.hpp" +#include "merovingian/database/DelayedEventStatusText.hpp" #include "merovingian/database/ExaminedRows.hpp" #include "merovingian/database/Migration.hpp" #include "merovingian/database/MigrationStep.hpp" @@ -1182,6 +1183,25 @@ namespace column_text(row, 2), column_text(row, 3), parse_u64(column_text(row, 4))}); }) && + // Migration 003 (ADR-0824): delayed events. A row whose status text is not one of the four, or that + // duplicates a delay ID or a transaction key, is skipped, never repaired. + load_rows(connection, + "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, content_json, " + "txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " + "error_json FROM delayed_events ORDER BY delay_id", + [&store](sqlite3_stmt& row) { + auto const status = parse_delayed_event_status(column_text(row, 12)); + if (!status.has_value()) + { + return; + } + std::ignore = store.delayed_events.insert( + {column_text(row, 0), column_text(row, 1), column_text(row, 2), column_text(row, 3), + column_text(row, 4), text_is_true(column_text(row, 5)), column_text(row, 6), + column_text(row, 7), column_text(row, 8), text_is_true(column_text(row, 9)), + parse_u64(column_text(row, 10)), parse_u64(column_text(row, 11)), *status, + parse_u64(column_text(row, 13)), column_text(row, 14), column_text(row, 15)}); + }) && // Application service delivery state: transaction cursors, the outbox and the directory mapping. load_rows(connection, "SELECT appservice_id, next_txn_id, delivered_stream_ordering, pending_txn_id, " diff --git a/src/database/meson.build b/src/database/meson.build index bd7d74e0c..e00f7b85f 100644 --- a/src/database/meson.build +++ b/src/database/meson.build @@ -1,5 +1,8 @@ database_sources = files( 'Connection.cpp', + 'DelayedEventStatusText.cpp', + 'DelayedEventStore.cpp', + 'DelayedEventTable.cpp', 'EventTable.cpp', 'EventStateGroupTable.cpp', 'ForwardExtremityTable.cpp', diff --git a/tests/integration/test_database_upgrade_flow.cpp b/tests/integration/test_database_upgrade_flow.cpp index 7379a1e7e..6ea453e73 100644 --- a/tests/integration/test_database_upgrade_flow.cpp +++ b/tests/integration/test_database_upgrade_flow.cpp @@ -826,6 +826,7 @@ SCENARIO("A SQLite schema is built at v1 and stops there", "[database][upgrade][ } REQUIRE_FALSE(has_table(migrated.state, "oauth_clients")); REQUIRE_FALSE(has_table(migrated.state, "oauth_sessions")); + REQUIRE_FALSE(has_table(migrated.state, "delayed_events")); REQUIRE(sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE name LIKE 'oauth%'") == 0); } } @@ -991,8 +992,10 @@ SCENARIO("A SQLite database migrated from v1 holds exactly the schema a fresh on REQUIRE(migrated.ok); REQUIRE(fresh.ok); REQUIRE(migrated.store.schema.version == merovingian::database::current_schema_version()); - REQUIRE(migrated.store.schema.applied_migrations.back().name == "oauth_tables"); - REQUIRE(sqlite_count(migrated_path, "SELECT count(*) FROM schema_migrations") == 2); + REQUIRE(migrated.store.schema.applied_migrations.back().name == "delayed_events"); + REQUIRE(sqlite_count(migrated_path, "SELECT count(*) FROM schema_migrations") == 3); + REQUIRE(sqlite_count(migrated_path, "SELECT count(*) FROM delayed_events") == 0); + REQUIRE(migrated.store.delayed_events.size() == 0U); REQUIRE(sqlite_count(migrated_path, "SELECT count(*) FROM oauth_clients") == 0); REQUIRE(sqlite_count(migrated_path, "SELECT count(*) FROM oauth_sessions") == 0); REQUIRE(migrated.store.oauth.client_count() == 0U); @@ -1106,6 +1109,7 @@ SCENARIO("A PostgreSQL schema is built at v1 and stops there", "[database][upgra } REQUIRE_FALSE(has_table(migrated.state, "oauth_clients")); REQUIRE_FALSE(has_table(migrated.state, "oauth_sessions")); + REQUIRE_FALSE(has_table(migrated.state, "delayed_events")); REQUIRE(pg_count(scratch.conninfo(), "SELECT count(*) FROM information_schema.tables WHERE " "table_schema = 'public' AND table_name LIKE 'oauth%'") == 0); } @@ -1240,7 +1244,8 @@ SCENARIO("A PostgreSQL database migrated from v1 holds exactly the schema a fres REQUIRE(migrated.ok); REQUIRE(fresh.ok); REQUIRE(migrated.store.schema.version == merovingian::database::current_schema_version()); - REQUIRE(pg_count(migrated_scratch.conninfo(), "SELECT count(*) FROM schema_migrations") == 2); + REQUIRE(pg_count(migrated_scratch.conninfo(), "SELECT count(*) FROM schema_migrations") == 3); + REQUIRE(pg_count(migrated_scratch.conninfo(), "SELECT count(*) FROM delayed_events") == 0); REQUIRE(pg_count(migrated_scratch.conninfo(), "SELECT count(*) FROM oauth_clients") == 0); REQUIRE(pg_count(migrated_scratch.conninfo(), "SELECT count(*) FROM oauth_sessions") == 0); auto const migrated_shape = pg_schema_shape(migrated_scratch.conninfo()); @@ -1352,6 +1357,8 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this auto const ledger_rows_before = sqlite_count(path, "SELECT count(*) FROM schema_migrations"); auto const oauth_objects_before = sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE name LIKE 'oauth%'"); + auto const delayed_objects_before = + sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE name LIKE '%delayed_events%'"); WHEN("the migration path runs on it and the current runtime starts on the result") { @@ -1363,6 +1370,13 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this sqlite_execute(path, "INSERT INTO oauth_clients VALUES ('upgrade-probe', '{}', '0', '0')") && sqlite_count(path, "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == 1 && sqlite_execute(path, "DELETE FROM oauth_clients WHERE client_id = 'upgrade-probe'"); + // Version 3's table takes a row and gives it back the same way. + auto const delayed_events_usable = + sqlite_execute(path, "INSERT INTO delayed_events VALUES ('upgrade-probe', '@u:x', '', '!r:x', " + "'m.room.message', 'false', '', '{}', '', 'false', '1000', '0', " + "'scheduled', '0', '', '')") && + sqlite_count(path, "SELECT count(*) FROM delayed_events WHERE delay_id = 'upgrade-probe'") == 1 && + sqlite_execute(path, "DELETE FROM delayed_events WHERE delay_id = 'upgrade-probe'"); auto started = merovingian::homeserver::start_client_server( sqlite_fixture_config(path, master_key, expected.server_name)); REQUIRE(started.started); @@ -1375,10 +1389,11 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this REQUIRE(expected.schema_version == 1); REQUIRE(ledger_rows_before == 1); REQUIRE(oauth_objects_before == 0); + REQUIRE(delayed_objects_before == 0); REQUIRE(migrated.state.version == merovingian::database::current_schema_version()); REQUIRE(static_cast(migrated.state.version) > expected.schema_version); REQUIRE(static_cast(migrated.state.applied_migrations.size()) > ledger_rows_before); - REQUIRE(migrated.state.applied_migrations.back().name == "oauth_tables"); + REQUIRE(migrated.state.applied_migrations.back().name == "delayed_events"); REQUIRE(sqlite_count(path, "SELECT count(*) FROM schema_migrations") == static_cast(migrated.state.applied_migrations.size())); REQUIRE(sqlite_count(path, "SELECT count(*) FROM oauth_clients") == 0); @@ -1386,6 +1401,10 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this REQUIRE(sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND " "name = 'idx_oauth_sessions_client_id'") == 1); REQUIRE(oauth_tables_usable); + REQUIRE(delayed_events_usable); + REQUIRE(sqlite_count(path, "SELECT count(*) FROM delayed_events") == 0); + REQUIRE(sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND " + "name = 'idx_delayed_events_user_id'") == 1); REQUIRE(started.runtime.homeserver.database.schema_version == merovingian::database::current_schema_version()); require_fixture_served(view, expected); @@ -1420,6 +1439,9 @@ SCENARIO("The 1.0.3 release's own PostgreSQL database is migrated and served by auto const oauth_tables_before = pg_count(scratch.conninfo(), "SELECT count(*) FROM information_schema.tables WHERE " "table_schema = 'public' AND table_name LIKE 'oauth%'"); + auto const delayed_tables_before = + pg_count(scratch.conninfo(), "SELECT count(*) FROM information_schema.tables WHERE " + "table_schema = 'public' AND table_name = 'delayed_events'"); WHEN("the migration path runs on it and the current runtime starts on the result") { @@ -1431,6 +1453,13 @@ SCENARIO("The 1.0.3 release's own PostgreSQL database is migrated and served by pg_count(scratch.conninfo(), "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == 1 && pg_execute(scratch.conninfo(), "DELETE FROM oauth_clients WHERE client_id = 'upgrade-probe'"); + auto const delayed_events_usable = + pg_execute(scratch.conninfo(), "INSERT INTO delayed_events VALUES ('upgrade-probe', '@u:x', '', " + "'!r:x', 'm.room.message', 'false', '', '{}', '', 'false', '1000', " + "'0', 'scheduled', '0', '', '')") && + pg_count(scratch.conninfo(), "SELECT count(*) FROM delayed_events WHERE delay_id = 'upgrade-probe'") == + 1 && + pg_execute(scratch.conninfo(), "DELETE FROM delayed_events WHERE delay_id = 'upgrade-probe'"); auto started = merovingian::homeserver::start_client_server( postgresql_fixture_config(uri_file, master_key, expected.server_name)); REQUIRE(started.started); @@ -1443,13 +1472,15 @@ SCENARIO("The 1.0.3 release's own PostgreSQL database is migrated and served by REQUIRE(expected.schema_version == 1); REQUIRE(ledger_rows_before == 1); REQUIRE(oauth_tables_before == 0); + REQUIRE(delayed_tables_before == 0); REQUIRE(migrated.state.version == merovingian::database::current_schema_version()); REQUIRE(static_cast(migrated.state.version) > expected.schema_version); REQUIRE(static_cast(migrated.state.applied_migrations.size()) > ledger_rows_before); - REQUIRE(migrated.state.applied_migrations.back().name == "oauth_tables"); + REQUIRE(migrated.state.applied_migrations.back().name == "delayed_events"); REQUIRE(pg_count(scratch.conninfo(), "SELECT count(*) FROM schema_migrations") == static_cast(migrated.state.applied_migrations.size())); REQUIRE(oauth_tables_usable); + REQUIRE(delayed_events_usable); REQUIRE(started.runtime.homeserver.database.schema_version == merovingian::database::current_schema_version()); require_fixture_served(view, expected); diff --git a/tests/integration/test_persistent_homeserver_flow.cpp b/tests/integration/test_persistent_homeserver_flow.cpp index 9eb593c5a..e77a99ddf 100644 --- a/tests/integration/test_persistent_homeserver_flow.cpp +++ b/tests/integration/test_persistent_homeserver_flow.cpp @@ -788,8 +788,9 @@ SCENARIO("Persistent homeserver runtime bootstraps a fresh migrated schema", "[d REQUIRE(applied.size() == merovingian::database::current_schema_version()); REQUIRE(applied.front().direction == merovingian::database::MigrationDirection::upgrade); REQUIRE(applied.front().name == "initial_schema"); - REQUIRE(applied.back().name == "oauth_tables"); + REQUIRE(applied.back().name == "delayed_events"); REQUIRE(merovingian::homeserver::database_has_table(started.runtime.database, "oauth_clients")); + REQUIRE(merovingian::homeserver::database_has_table(started.runtime.database, "delayed_events")); REQUIRE(merovingian::homeserver::database_has_table(started.runtime.database, "oauth_sessions")); } } diff --git a/tests/integration/test_postgresql_store_behaviour_flow.cpp b/tests/integration/test_postgresql_store_behaviour_flow.cpp index 14c574375..ab9bebc1a 100644 --- a/tests/integration/test_postgresql_store_behaviour_flow.cpp +++ b/tests/integration/test_postgresql_store_behaviour_flow.cpp @@ -7,6 +7,9 @@ // MEROVINGIAN_TEST_POSTGRESQL_URI is not set; a skip is not a pass. #include +#include +#include +#include #include #include #include @@ -1680,3 +1683,109 @@ SCENARIO("PostgreSQL OAuth clients and sessions survive a reopen and a session g } } } + +SCENARIO("PostgreSQL delayed events of every status survive a reopen with every column and index", + "[database][postgresql][integration][pg-behaviour][delayed-events]") +{ + GIVEN("a scheduled state event, a sent row, a cancelled row and a failed row of two users") + { + auto const uri = postgresql_uri(); + if (uri.empty()) + { + SUCCEED("skipped: MEROVINGIAN_TEST_POSTGRESQL_URI is not set"); + return; + } + auto opened = reopen(uri); + auto const suffix = unique_suffix(); + auto const alice = "@delayed-a-" + suffix + ":example.org"; + auto const bob = "@delayed-b-" + suffix + ":example.org"; + auto const room_id = "!delayed-" + suffix + ":example.org"; + auto const make = [&](std::string id, std::string const& user) { + auto row = db::PersistentDelayedEvent{}; + row.delay_id = "d-" + id + "-" + suffix; + row.user_id = user; + row.device_id = "DEVICE"; + row.room_id = room_id; + row.event_type = "m.room.message"; + row.content_json = R"({"body":"two words \"quoted\""})"; + row.delay_ms = 60'000U; + row.delayed_since_ms = 1'000U; + return row; + }; + auto state = make("state", alice); + state.is_state = true; + state.state_key = alice; + state.via_appservice = true; + state.delay_ms = db::max_delayed_event_delay_ms; + state.delayed_since_ms = 123'456'789U; + auto message = make("message", alice); + message.txn_id = "txn-" + suffix; + REQUIRE(db::store_delayed_event(opened.store, state)); + REQUIRE(db::store_delayed_event(opened.store, message)); + REQUIRE(db::store_delayed_event(opened.store, make("sent", alice))); + REQUIRE(db::store_delayed_event(opened.store, make("cancelled", alice))); + REQUIRE(db::store_delayed_event(opened.store, make("failed", bob))); + REQUIRE(db::record_delayed_event_outcome(opened.store, "d-sent-" + suffix, db::DelayedEventStatus::sent, 2000U, + "$sent-" + suffix, {}, + 10U) == db::DelayedEventOutcomeResult::recorded); + REQUIRE(db::cancel_delayed_event(opened.store, "d-cancelled-" + suffix, 2001U, 10U)); + REQUIRE(db::record_delayed_event_outcome(opened.store, "d-failed-" + suffix, db::DelayedEventStatus::failed, + 2002U, {}, R"({"errcode":"M_FORBIDDEN","error":"nope"})", + 10U) == db::DelayedEventOutcomeResult::recorded); + + WHEN("the store is reopened") + { + opened = {}; + auto reopened = reopen(uri); + auto const& table = reopened.store.delayed_events; + auto const* held_state = table.find("d-state-" + suffix); + auto const* held_message = table.find("d-message-" + suffix); + + THEN("every column is back and the indexes answer") + { + REQUIRE(held_state != nullptr); + REQUIRE(held_state->user_id == alice); + REQUIRE(held_state->device_id == "DEVICE"); + REQUIRE(held_state->room_id == room_id); + REQUIRE(held_state->event_type == "m.room.message"); + REQUIRE(held_state->is_state); + REQUIRE(held_state->state_key == alice); + REQUIRE(held_state->content_json == R"({"body":"two words \"quoted\""})"); + REQUIRE(held_state->txn_id.empty()); + REQUIRE(held_state->via_appservice); + REQUIRE(held_state->delay_ms == db::max_delayed_event_delay_ms); + REQUIRE(held_state->delayed_since_ms == 123'456'789U); + REQUIRE(held_state->status == db::DelayedEventStatus::scheduled); + REQUIRE(held_state->finalised_at_ms == 0U); + REQUIRE(held_message != nullptr); + REQUIRE_FALSE(held_message->is_state); + REQUIRE_FALSE(held_message->via_appservice); + REQUIRE(table.find("d-sent-" + suffix)->status == db::DelayedEventStatus::sent); + REQUIRE(table.find("d-sent-" + suffix)->event_id == "$sent-" + suffix); + REQUIRE(table.find("d-sent-" + suffix)->finalised_at_ms == 2000U); + REQUIRE(table.find("d-cancelled-" + suffix)->status == db::DelayedEventStatus::cancelled); + REQUIRE(table.find("d-failed-" + suffix)->status == db::DelayedEventStatus::failed); + REQUIRE(table.find("d-failed-" + suffix)->error_json == R"({"errcode":"M_FORBIDDEN","error":"nope"})"); + REQUIRE(table.find_owned(bob, "d-failed-" + suffix) != nullptr); + REQUIRE(table.find_owned(alice, "d-failed-" + suffix) == nullptr); + REQUIRE(table.scheduled_count(alice) == 2U); + REQUIRE(table.finalised_count(alice) == 2U); + REQUIRE(table.finalised_count(bob) == 1U); + REQUIRE(table.earliest_due_ms_of_user(alice) == std::optional{61'000U}); + REQUIRE(table.find_by_transaction(alice, "DEVICE", room_id, "m.room.message", "txn-" + suffix) == + held_message); + REQUIRE(table.find_sent_by_event_id("$sent-" + suffix) == table.find("d-sent-" + suffix)); + } + + AND_THEN("deleting both users' rows removes them from the database as well") + { + REQUIRE(db::delete_delayed_events_of_user(reopened.store, alice)); + REQUIRE(db::delete_delayed_events_of_user(reopened.store, bob)); + auto again = reopen(uri); + REQUIRE(again.store.delayed_events.find("d-state-" + suffix) == nullptr); + REQUIRE(again.store.delayed_events.find("d-failed-" + suffix) == nullptr); + REQUIRE(again.store.delayed_events.scheduled_count(alice) == 0U); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index ecd15dfcd..4994e74b4 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -353,6 +353,8 @@ shell = find_program('sh') 'unit/test_observability_bounded_text.cpp', 'unit/test_database_persistence.cpp', 'unit/test_database_oauth_store.cpp', + 'unit/test_delayed_event_store.cpp', + 'unit/test_delayed_event_table.cpp', 'unit/test_media_reservation_sweep.cpp', 'unit/test_credential_tables.cpp', 'unit/test_bundled_aggregations.cpp', diff --git a/tests/smoke/meson.build b/tests/smoke/meson.build index 7081fedea..1271c6330 100644 --- a/tests/smoke/meson.build +++ b/tests/smoke/meson.build @@ -600,3 +600,25 @@ test( + '" | grep -q "current_version=0 target_version=2 steps=2"', ], ) + +# Migration 003 (1.1.2): a plan from v2 takes the checked-in 003 file alone, and a plan from an empty database takes +# all three (ADR-0778). +test( + 'db-migrate-plans-v2-to-v3-from-files-smoke-test', + smoke_test, + args: [ + '-c', + db_migrate_executable + ' --plan 2 3 --migrations "' + migrations_directory + + '" | grep -q "current_version=2 target_version=3 steps=1"', + ], +) + +test( + 'db-migrate-plans-empty-to-v3-from-files-smoke-test', + smoke_test, + args: [ + '-c', + db_migrate_executable + ' --plan 0 3 --migrations "' + migrations_directory + + '" | grep -q "current_version=0 target_version=3 steps=3"', + ], +) diff --git a/tests/unit/test_database_persistence.cpp b/tests/unit/test_database_persistence.cpp index e844729cb..05d542d66 100644 --- a/tests/unit/test_database_persistence.cpp +++ b/tests/unit/test_database_persistence.cpp @@ -1332,7 +1332,7 @@ SCENARIO("Database migration plans are contiguous and direction-aware", "[databa } } -SCENARIO("Database migration runner applies the v1 and v2 schema steps and their explicit downgrades", +SCENARIO("Database migration runner applies the v1, v2 and v3 schema steps and their explicit downgrades", "[database][migration]") { GIVEN("an empty schema state") @@ -1348,30 +1348,34 @@ SCENARIO("Database migration runner applies the v1 and v2 schema steps and their auto const downgrade_plan = merovingian::database::migration_plan_between(upgraded.state.version, 0U); auto const downgraded = merovingian::database::apply_migration_plan(upgraded.state, downgrade_plan); - THEN("v1 then v2 build the whole current schema, and two steps take it away newest first") + THEN("v1, v2 then v3 build the whole current schema, and three steps take it away newest first") { - REQUIRE(merovingian::database::current_schema_version() == 2U); + REQUIRE(merovingian::database::current_schema_version() == 3U); REQUIRE(upgrade_plan.direction == merovingian::database::MigrationDirection::upgrade); REQUIRE(upgrade_plan.current_version == 0U); - REQUIRE(upgrade_plan.target_version == 2U); - REQUIRE(upgrade_plan.steps.size() == 2U); + REQUIRE(upgrade_plan.target_version == 3U); + REQUIRE(upgrade_plan.steps.size() == 3U); REQUIRE(upgrade_plan.steps[0].version == 1U); REQUIRE(upgrade_plan.steps[0].name == "initial_schema"); REQUIRE(upgrade_plan.steps[1].version == 2U); REQUIRE(upgrade_plan.steps[1].name == "oauth_tables"); + REQUIRE(upgrade_plan.steps[2].version == 3U); + REQUIRE(upgrade_plan.steps[2].name == "delayed_events"); REQUIRE(upgraded.ok); - REQUIRE(upgraded.state.version == 2U); - REQUIRE(upgraded.state.applied_migrations.size() == 2U); + REQUIRE(upgraded.state.version == 3U); + REQUIRE(upgraded.state.applied_migrations.size() == 3U); REQUIRE(upgraded.state.applied_migrations.front().name == "initial_schema"); - REQUIRE(upgraded.state.applied_migrations.back().name == "oauth_tables"); + REQUIRE(upgraded.state.applied_migrations.back().name == "delayed_events"); REQUIRE(upgraded.state.tables.size() == merovingian::database::current_schema_tables().size()); REQUIRE(compatible.valid); REQUIRE(second_plan.steps.empty()); REQUIRE(downgrade_plan.direction == merovingian::database::MigrationDirection::downgrade); - REQUIRE(downgrade_plan.steps.size() == 2U); - REQUIRE(downgrade_plan.steps[0].name == "drop_oauth_tables"); - REQUIRE(downgrade_plan.steps[0].version == 1U); - REQUIRE(downgrade_plan.steps[1].name == "drop_initial_schema"); + REQUIRE(downgrade_plan.steps.size() == 3U); + REQUIRE(downgrade_plan.steps[0].name == "drop_delayed_events"); + REQUIRE(downgrade_plan.steps[0].version == 2U); + REQUIRE(downgrade_plan.steps[1].name == "drop_oauth_tables"); + REQUIRE(downgrade_plan.steps[1].version == 1U); + REQUIRE(downgrade_plan.steps[2].name == "drop_initial_schema"); REQUIRE(downgraded.ok); REQUIRE(downgraded.state.version == 0U); REQUIRE(downgraded.state.tables.empty()); @@ -1398,8 +1402,8 @@ SCENARIO("The v1 schema step creates every table in its final shape and every in THEN("each v1 table is created whole, by exactly the DDL the registry describes") { - REQUIRE(merovingian::database::upgrade_migration_catalog().size() == 2U); - REQUIRE(merovingian::database::downgrade_migration_catalog().size() == 2U); + REQUIRE(merovingian::database::upgrade_migration_catalog().size() == 3U); + REQUIRE(merovingian::database::downgrade_migration_catalog().size() == 3U); for (auto const& table : tables) { INFO("table " << table.name); @@ -1410,6 +1414,8 @@ SCENARIO("The v1 schema step creates every table in its final shape and every in REQUIRE(tables.size() == merovingian::database::schema_definitions_added_by(1U).size()); REQUIRE(statement_sql("create_oauth_clients").empty()); REQUIRE(statement_sql("create_oauth_sessions").empty()); + REQUIRE(statement_sql("create_delayed_events").empty()); + REQUIRE(statement_sql("create_idx_delayed_events_user_id").empty()); } AND_THEN("every column the pre-1.0 chain ALTERed on is part of its table's CREATE") @@ -1508,7 +1514,8 @@ SCENARIO("The v2 schema step adds only the OAuth 2.0 tables and their index", "[ "device_id))"); REQUIRE(step.statements[2].sql == "CREATE INDEX idx_oauth_sessions_client_id ON oauth_sessions (client_id)"); - REQUIRE(current.size() == initial.size() + 2U); + // Version 2 adds two tables; version 3 adds one more. + REQUIRE(current.size() == initial.size() + 3U); REQUIRE(std::ranges::find(initial, std::string_view{"oauth_clients"}) == initial.end()); REQUIRE(std::ranges::find(current, std::string_view{"oauth_clients"}) != current.end()); REQUIRE(std::ranges::find(current, std::string_view{"oauth_sessions"}) != current.end()); @@ -1526,6 +1533,61 @@ SCENARIO("The v2 schema step adds only the OAuth 2.0 tables and their index", "[ } } +SCENARIO("The v3 schema step adds only the delayed-events table and its user index", + "[database][migration][delayed-events]") +{ + GIVEN("the v3 schema step, its downgrade and the current table inventory") + { + auto const step = merovingian::database::delayed_events_migration(); + auto const downgrade = merovingian::database::downgrade_delayed_events_migration(); + auto const initial = merovingian::database::initial_schema_tables(); + auto const current = merovingian::database::current_schema_tables(); + + WHEN("its statements are inspected") + { + auto const names = [&step]() { + auto collected = std::vector{}; + for (auto const& statement : step.statements) + { + collected.push_back(statement.name); + } + return collected; + }(); + + THEN("it creates delayed_events with sixteen TEXT columns and the user index, and nothing else") + { + REQUIRE(step.version == 3U); + REQUIRE(step.name == "delayed_events"); + REQUIRE(step.direction == merovingian::database::MigrationDirection::upgrade); + REQUIRE(names == + std::vector{"create_delayed_events", "create_idx_delayed_events_user_id"}); + REQUIRE(step.statements[0].sql == + "CREATE TABLE \"delayed_events\" (delay_id TEXT PRIMARY KEY, user_id TEXT NOT NULL, " + "device_id TEXT NOT NULL, room_id TEXT NOT NULL, event_type TEXT NOT NULL, is_state TEXT " + "NOT NULL, state_key TEXT NOT NULL, content_json TEXT NOT NULL, txn_id TEXT NOT NULL, " + "via_appservice TEXT NOT NULL, delay_ms TEXT NOT NULL, delayed_since_ms TEXT NOT NULL, " + "status TEXT NOT NULL, finalised_at_ms TEXT NOT NULL, event_id TEXT NOT NULL, error_json " + "TEXT NOT NULL)"); + REQUIRE(step.statements[1].sql == + "CREATE INDEX idx_delayed_events_user_id ON delayed_events (user_id)"); + REQUIRE(current.size() == initial.size() + 3U); + REQUIRE(std::ranges::find(initial, std::string_view{"delayed_events"}) == initial.end()); + REQUIRE(std::ranges::find(current, std::string_view{"delayed_events"}) != current.end()); + REQUIRE(merovingian::database::schema_definitions_added_by(2U).size() == 2U); + REQUIRE(merovingian::database::schema_definitions_added_by(3U).size() == 1U); + } + + AND_THEN("its downgrade drops only that table, back to version 2") + { + REQUIRE(downgrade.version == 2U); + REQUIRE(downgrade.name == "drop_delayed_events"); + REQUIRE(downgrade.statements.size() == 1U); + REQUIRE(downgrade.statements[0].sql.find("delayed_events") != std::string::npos); + } + } + } +} + SCENARIO("A schema ledger naming a migration this build does not have is refused", "[database][migration][collapse][security]") { @@ -2420,7 +2482,7 @@ SCENARIO("Checked-in migrations hold exactly the steps this build runs", "[datab } SCENARIO("An offline plan takes only the migration files between its two versions", - "[database][migration][files][oauth]") + "[database][migration][files][oauth][delayed-events]") { GIVEN("the repository migration directory and the migration role") { @@ -2431,22 +2493,28 @@ SCENARIO("An offline plan takes only the migration files between its two version auto migration_config = merovingian::config::DatabaseConfig{}; migration_config.role = merovingian::config::DatabaseRole::migration; - WHEN("plans from v1 to v2 and from nothing to v2 are built from the files") + WHEN("plans from v1, from v2 and from nothing to v3 are built from the files") { auto const from_v1 = - merovingian::database::build_offline_migration_plan(migration_config, 1U, 2U, loaded.steps); + merovingian::database::build_offline_migration_plan(migration_config, 1U, 3U, loaded.steps); + auto const from_v2 = + merovingian::database::build_offline_migration_plan(migration_config, 2U, 3U, loaded.steps); auto const from_empty = - merovingian::database::build_offline_migration_plan(migration_config, 0U, 2U, loaded.steps); + merovingian::database::build_offline_migration_plan(migration_config, 0U, 3U, loaded.steps); auto const no_op = - merovingian::database::build_offline_migration_plan(migration_config, 2U, 2U, loaded.steps); + merovingian::database::build_offline_migration_plan(migration_config, 3U, 3U, loaded.steps); - THEN("a v1 database gets 002 alone, an empty one gets 001 then 002, and a v2 one gets nothing") + THEN("a v1 database gets 002 then 003, a v2 one 003 alone, an empty one all three, a v3 one nothing") { REQUIRE(from_v1.ok); - REQUIRE(from_v1.plan.steps.size() == 1U); + REQUIRE(from_v1.plan.steps.size() == 2U); REQUIRE(from_v1.plan.steps.front().name == "oauth_tables"); + REQUIRE(from_v1.plan.steps.back().name == "delayed_events"); + REQUIRE(from_v2.ok); + REQUIRE(from_v2.plan.steps.size() == 1U); + REQUIRE(from_v2.plan.steps.front().name == "delayed_events"); REQUIRE(from_empty.ok); - REQUIRE(from_empty.plan.steps.size() == 2U); + REQUIRE(from_empty.plan.steps.size() == 3U); REQUIRE(from_empty.plan.steps.front().name == "initial_schema"); REQUIRE(no_op.ok); REQUIRE(no_op.plan.steps.empty()); @@ -2502,10 +2570,10 @@ SCENARIO("Database schema inventory covers the core Matrix tables", "[database][ THEN("required Matrix storage areas have table-specific definitions") { // The pre-1.0 migration chain collapsed into v1's 64 tables; migration 002 (1.1.0) adds the two - // OAuth 2.0 tables. + // OAuth 2.0 tables and migration 003 (1.1.2) the delayed-events table. REQUIRE(tables.size() == 64U); - REQUIRE(merovingian::database::current_schema_version() == 2U); - REQUIRE(merovingian::database::current_schema_tables().size() == 66U); + REQUIRE(merovingian::database::current_schema_version() == 3U); + REQUIRE(merovingian::database::current_schema_tables().size() == 67U); REQUIRE(users_definition.has_value()); REQUIRE(current_state_definition.has_value()); REQUIRE(room_aliases_definition.has_value()); diff --git a/tests/unit/test_delayed_event_store.cpp b/tests/unit/test_delayed_event_store.cpp new file mode 100644 index 000000000..297c32595 --- /dev/null +++ b/tests/unit/test_delayed_event_store.cpp @@ -0,0 +1,904 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// The delayed-events store functions (migration 003, ADR-0824, ADR-0825): what each refuses, that memory changes only +// after a commit (and the one exception), the per-user eviction in the finalising transaction, the age prune, bound +// parameters only, and a round trip through a SQLite file. +#include "../support/TempDirectory.hpp" +#include "merovingian/database/DelayedEventOutcomeResult.hpp" +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/DelayedEventStore.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentDelayedEvent.hpp" +#include "merovingian/database/PersistentStore.hpp" +#include "merovingian/database/PersistentStoreOpenResult.hpp" +#include "merovingian/database/PreparedStatement.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +namespace db = merovingian::database; +using db::DelayedEventOutcomeResult; +using db::DelayedEventStatus; +using db::PersistentDelayedEvent; + +constexpr auto alice = std::string_view{"@alice:example.org"}; +constexpr auto bob = std::string_view{"@bob:example.org"}; +constexpr auto secret_content = std::string_view{R"({"body":"SECRET-CONTENT-MARKER"})"}; + +[[nodiscard]] auto row(std::string delay_id, std::string_view user = alice) -> PersistentDelayedEvent +{ + auto made = PersistentDelayedEvent{}; + made.delay_id = std::move(delay_id); + made.user_id = std::string{user}; + made.device_id = "DEVICE"; + made.room_id = "!room:example.org"; + made.event_type = "m.room.message"; + made.content_json = std::string{secret_content}; + made.delay_ms = 60'000U; + made.delayed_since_ms = 1'000U; + return made; +} + +[[nodiscard]] auto opened_store() -> db::PersistentStoreOpenResult +{ + auto opened = db::open_persistent_store(); + REQUIRE(opened.ok); + return opened; +} + +// Stores `count` scheduled rows for `user` named `-` and finalises each as cancelled at `100 + n`. +auto fill_finalised(db::PersistentStore& store, std::string_view prefix, std::string_view user, std::size_t count) + -> void +{ + for (auto index = std::size_t{0U}; index < count; ++index) + { + auto const id = std::string{prefix} + "-" + std::to_string(index); + REQUIRE(db::store_delayed_event(store, row(id, user))); + REQUIRE(db::cancel_delayed_event(store, id, 100U + index, 1000U)); + } +} + +struct SqliteClose final +{ + auto operator()(sqlite3* connection) const noexcept -> void + { + sqlite3_close(connection); + } +}; + +// Runs one query on its own connection to the file and returns the text of its first column of its first row. +[[nodiscard]] auto sqlite_first_text(std::filesystem::path const& path, char const* sql) -> std::string +{ + auto* raw = static_cast(nullptr); + auto const opened = sqlite3_open_v2(path.string().c_str(), &raw, SQLITE_OPEN_READONLY, nullptr); + auto const connection = std::unique_ptr{raw}; + auto text = std::string{}; + if (opened != SQLITE_OK) + { + return text; + } + auto* statement = static_cast(nullptr); + if (sqlite3_prepare_v2(connection.get(), sql, -1, &statement, nullptr) == SQLITE_OK && + sqlite3_step(statement) == SQLITE_ROW) + { + auto const* value = sqlite3_column_text(statement, 0); + if (value != nullptr) + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + text = reinterpret_cast(value); + } + } + sqlite3_finalize(statement); + return text; +} + +} // namespace + +SCENARIO("A delayed event is stored once, within its caps, and refused outside them", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("a memory store") + { + auto opened = opened_store(); + auto& store = opened.store; + + WHEN("rows at each cap and one over each cap are offered") + { + struct Case final + { + std::string label; + std::function change; + bool stored; + }; + auto const cases = std::vector{ + {"baseline", + [](PersistentDelayedEvent&) { + }, true }, + {"empty delay id", + [](PersistentDelayedEvent& r) { + r.delay_id.clear(); + }, false}, + {"empty user", + [](PersistentDelayedEvent& r) { + r.user_id.clear(); + }, false}, + {"empty room", + [](PersistentDelayedEvent& r) { + r.room_id.clear(); + }, false}, + {"empty event type", + [](PersistentDelayedEvent& r) { + r.event_type.clear(); + }, false}, + {"empty content", + [](PersistentDelayedEvent& r) { + r.content_json.clear(); + }, false}, + {"delay id at cap", + [](PersistentDelayedEvent& r) { + r.delay_id = std::string(db::max_delayed_event_id_bytes, 'd'); + }, true }, + {"delay id over cap", + [](PersistentDelayedEvent& r) { + r.delay_id = std::string(db::max_delayed_event_id_bytes + 1U, 'd'); + }, false}, + {"content at cap", + [](PersistentDelayedEvent& r) { + r.content_json = std::string(db::max_delayed_event_content_bytes, 'c'); + }, true }, + {"content over cap", + [](PersistentDelayedEvent& r) { + r.content_json = std::string(db::max_delayed_event_content_bytes + 1U, 'c'); + }, false}, + {"event type at cap", + [](PersistentDelayedEvent& r) { + r.event_type = std::string(db::max_delayed_event_field_bytes, 't'); + }, true }, + {"event type over cap", + [](PersistentDelayedEvent& r) { + r.event_type = std::string(db::max_delayed_event_field_bytes + 1U, 't'); + }, false}, + {"state key at cap", + [](PersistentDelayedEvent& r) { + r.state_key = std::string(db::max_delayed_event_field_bytes, 's'); + }, true }, + {"state key over cap", + [](PersistentDelayedEvent& r) { + r.state_key = std::string(db::max_delayed_event_field_bytes + 1U, 's'); + }, false}, + {"txn id at cap", + [](PersistentDelayedEvent& r) { + r.txn_id = std::string(db::max_delayed_event_field_bytes, 'x'); + }, true }, + {"txn id over cap", + [](PersistentDelayedEvent& r) { + r.txn_id = std::string(db::max_delayed_event_field_bytes + 1U, 'x'); + }, false}, + {"device id at cap", + [](PersistentDelayedEvent& r) { + r.device_id = std::string(db::max_delayed_event_field_bytes, 'v'); + }, true }, + {"device id over cap", + [](PersistentDelayedEvent& r) { + r.device_id = std::string(db::max_delayed_event_field_bytes + 1U, 'v'); + }, false}, + {"user id at cap", + [](PersistentDelayedEvent& r) { + r.user_id = std::string(db::max_delayed_event_field_bytes, 'u'); + }, true }, + {"user id over cap", + [](PersistentDelayedEvent& r) { + r.user_id = std::string(db::max_delayed_event_field_bytes + 1U, 'u'); + }, false}, + {"room id at cap", + [](PersistentDelayedEvent& r) { + r.room_id = std::string(db::max_delayed_event_field_bytes, 'r'); + }, true }, + {"room id over cap", + [](PersistentDelayedEvent& r) { + r.room_id = std::string(db::max_delayed_event_field_bytes + 1U, 'r'); + }, false}, + {"delay of one millisecond", + [](PersistentDelayedEvent& r) { + r.delay_ms = 1U; + }, true }, + {"delay of zero", + [](PersistentDelayedEvent& r) { + r.delay_ms = 0U; + }, false}, + {"largest delay", + [](PersistentDelayedEvent& r) { + r.delay_ms = db::max_delayed_event_delay_ms; + }, true }, + {"delay over the largest", + [](PersistentDelayedEvent& r) { + r.delay_ms = db::max_delayed_event_delay_ms + 1U; + }, false}, + {"status sent", + [](PersistentDelayedEvent& r) { + r.status = DelayedEventStatus::sent; + }, false}, + {"status cancelled", + [](PersistentDelayedEvent& r) { + r.status = DelayedEventStatus::cancelled; + }, false}, + {"status failed", + [](PersistentDelayedEvent& r) { + r.status = DelayedEventStatus::failed; + }, false}, + {"a finalised time", + [](PersistentDelayedEvent& r) { + r.finalised_at_ms = 5U; + }, false}, + {"an event id", + [](PersistentDelayedEvent& r) { + r.event_id = "$e"; + }, false}, + {"an error", + [](PersistentDelayedEvent& r) { + r.error_json = "{}"; + }, false}, + }; + auto outcomes = std::vector>{}; + auto transactions_by_refusal = std::vector{}; + for (auto const& each : cases) + { + auto candidate = row("d-case"); + each.change(candidate); + // A fresh store per case: the baseline's ID would otherwise collide with the next case. + auto fresh = opened_store(); + auto const before = db::committed_transaction_count(fresh.store); + auto const stored = db::store_delayed_event(fresh.store, std::move(candidate)); + outcomes.emplace_back(each.label, stored); + if (!each.stored) + { + transactions_by_refusal.push_back(db::committed_transaction_count(fresh.store) - before); + } + REQUIRE(fresh.store.delayed_events.size() == (stored ? 1U : 0U)); + } + + THEN("every case is stored or refused as listed, and a refusal writes nothing") + { + for (auto index = std::size_t{0U}; index < cases.size(); ++index) + { + INFO("case " << cases[index].label); + REQUIRE(outcomes[index].second == cases[index].stored); + } + REQUIRE(std::ranges::all_of(transactions_by_refusal, [](std::uint64_t count) { + return count == 0U; + })); + } + } + + WHEN("a delay ID is stored twice, and the same transaction is stored under a second delay ID") + { + auto first = row("d-1"); + first.txn_id = "txn"; + auto twin = row("d-2"); + twin.txn_id = "txn"; + auto other_device = row("d-3"); + other_device.txn_id = "txn"; + other_device.device_id = "ELSEWHERE"; + auto const stored = db::store_delayed_event(store, first); + auto const before = db::committed_transaction_count(store); + auto const repeated = db::store_delayed_event(store, row("d-1", bob)); + auto const same_transaction = db::store_delayed_event(store, twin); + auto const refused_transactions = db::committed_transaction_count(store) - before; + auto const second_device = db::store_delayed_event(store, other_device); + + THEN("the repeat and the twin are refused without a write; another device's transaction is its own") + { + REQUIRE(stored); + REQUIRE_FALSE(repeated); + REQUIRE_FALSE(same_transaction); + REQUIRE(refused_transactions == 0U); + REQUIRE(second_device); + REQUIRE(store.delayed_events.find("d-1")->user_id == alice); + REQUIRE(store.delayed_events.size() == 2U); + } + } + + WHEN("rows are stored for a user beyond any plausible per-user cap") + { + auto all = true; + for (auto index = 0; index < 150; ++index) + { + all = db::store_delayed_event(store, row("d-" + std::to_string(index))) && all; + } + + THEN("the store does not enforce the per-user cap: that is the caller's check") + { + REQUIRE(all); + REQUIRE(store.delayed_events.scheduled_count(alice) == 150U); + } + } + } +} + +SCENARIO("A delayed event is written with bound parameters only", "[database][delayed-events][delayed-event-store]") +{ + GIVEN("a memory store capturing statements") + { + auto opened = opened_store(); + auto& store = opened.store; + db::enable_statement_capture(store, 16U); + + WHEN("a state event is stored and later restarted and cancelled") + { + auto stored_row = row("d-1"); + stored_row.is_state = true; + stored_row.state_key = "@alice:example.org"; + stored_row.txn_id = "txn-9"; + stored_row.via_appservice = true; + REQUIRE(db::store_delayed_event(store, stored_row)); + REQUIRE(db::restart_delayed_event(store, "d-1", 7000U)); + REQUIRE(db::cancel_delayed_event(store, "d-1", 8000U, 10U)); + auto const& captured = store.captured_statements; + + THEN("every statement uses $n placeholders, none carries a value, and the boolean columns hold text") + { + REQUIRE(captured.size() == 3U); + for (auto const& statement : captured) + { + INFO("statement " << statement.name); + REQUIRE(statement.sql.find('$') != std::string::npos); + REQUIRE(statement.sql.find("SECRET-CONTENT-MARKER") == std::string::npos); + REQUIRE(statement.sql.find("d-1") == std::string::npos); + REQUIRE(statement.sql.find(std::string{alice}) == std::string::npos); + } + auto const& insert = captured[0]; + REQUIRE(insert.name == "insert_delayed_event"); + REQUIRE(insert.parameters.size() == 16U); + REQUIRE(insert.sql.find("$16") != std::string::npos); + REQUIRE(insert.parameters[0].value == "d-1"); + REQUIRE(insert.parameters[5].value == "true"); + REQUIRE(insert.parameters[7].value == std::string{secret_content}); + REQUIRE(insert.parameters[9].value == "true"); + REQUIRE(insert.parameters[10].value == "60000"); + REQUIRE(insert.parameters[12].value == "scheduled"); + REQUIRE(captured[1].sql == "UPDATE delayed_events SET delayed_since_ms = $1 WHERE delay_id = $2"); + REQUIRE(captured[1].parameters[0].value == "7000"); + REQUIRE(captured[2].parameters[0].value == "cancelled"); + REQUIRE(captured[2].parameters[1].value == "8000"); + } + } + } +} + +SCENARIO("A failed commit leaves the delayed-events mirror unchanged", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("a memory store holding a scheduled row and a finalised one") + { + auto opened = opened_store(); + auto& store = opened.store; + REQUIRE(db::store_delayed_event(store, row("d-live"))); + REQUIRE(db::store_delayed_event(store, row("d-done"))); + REQUIRE(db::cancel_delayed_event(store, "d-done", 500U, 100U)); + + WHEN("the commit fails for a store, a restart, a cancel, a user delete and a prune") + { + db::force_next_persist_failures(store, 1U); + auto const stored = db::store_delayed_event(store, row("d-new")); + db::force_next_persist_failures(store, 1U); + auto const restarted = db::restart_delayed_event(store, "d-live", 9000U); + db::force_next_persist_failures(store, 1U); + auto const cancelled = db::cancel_delayed_event(store, "d-live", 9000U, 100U); + db::force_next_persist_failures(store, 1U); + auto const deleted = db::delete_delayed_events_of_user(store, alice); + db::force_next_persist_failures(store, 1U); + auto const pruned = db::prune_finalised_delayed_events(store, 100'000U, 10U); + + THEN("each reports the failure and memory is exactly as it was") + { + REQUIRE_FALSE(stored); + REQUIRE_FALSE(restarted); + REQUIRE_FALSE(cancelled); + REQUIRE_FALSE(deleted); + REQUIRE_FALSE(pruned.has_value()); + REQUIRE(store.delayed_events.size() == 2U); + REQUIRE(store.delayed_events.find("d-new") == nullptr); + REQUIRE(store.delayed_events.find("d-live")->delayed_since_ms == 1000U); + REQUIRE(store.delayed_events.find("d-live")->status == DelayedEventStatus::scheduled); + REQUIRE(store.delayed_events.find("d-done") != nullptr); + REQUIRE(store.delayed_events.earliest_due() == store.delayed_events.find("d-live")); + } + } + + WHEN("each of those writes is repeated with the database working") + { + auto const restarted = db::restart_delayed_event(store, "d-live", 9000U); + auto const absent = db::restart_delayed_event(store, "d-nope", 9000U); + auto const finalised = db::restart_delayed_event(store, "d-done", 9000U); + auto const cancelled_again = db::cancel_delayed_event(store, "d-done", 9000U, 100U); + + THEN("a restart moves the start, and a row that is absent or finalised is refused") + { + REQUIRE(restarted); + REQUIRE_FALSE(absent); + REQUIRE_FALSE(finalised); + REQUIRE_FALSE(cancelled_again); + REQUIRE(store.delayed_events.find("d-live")->delayed_since_ms == 9000U); + } + } + + WHEN("the row is being sent") + { + REQUIRE(store.delayed_events.mark_being_sent("d-live")); + auto const restarted = db::restart_delayed_event(store, "d-live", 9000U); + auto const cancelled = db::cancel_delayed_event(store, "d-live", 9000U, 100U); + + THEN("neither a restart nor a cancel acts on it") + { + REQUIRE_FALSE(restarted); + REQUIRE_FALSE(cancelled); + REQUIRE(store.delayed_events.find("d-live")->status == DelayedEventStatus::scheduled); + } + } + } +} + +SCENARIO("The outcome of a send is recorded, and a failed commit still finalises memory", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("a memory store holding two scheduled rows, both marked as being sent") + { + auto opened = opened_store(); + auto& store = opened.store; + REQUIRE(db::store_delayed_event(store, row("d-ok"))); + REQUIRE(db::store_delayed_event(store, row("d-bad"))); + REQUIRE(db::store_delayed_event(store, row("d-lost"))); + REQUIRE(store.delayed_events.mark_being_sent("d-ok")); + REQUIRE(store.delayed_events.mark_being_sent("d-bad")); + REQUIRE(store.delayed_events.mark_being_sent("d-lost")); + + WHEN("a success, a failure and a success whose commit fails are recorded") + { + auto const sent = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 5000U, "$event", {}, 10U); + auto const failed = db::record_delayed_event_outcome(store, "d-bad", DelayedEventStatus::failed, 5001U, {}, + R"({"errcode":"M_FORBIDDEN","error":"no"})", 10U); + auto const before = db::committed_transaction_count(store); + db::force_next_persist_failures(store, 1U); + auto const lost = + db::record_delayed_event_outcome(store, "d-lost", DelayedEventStatus::sent, 5002U, "$lost", {}, 10U); + auto const after = db::committed_transaction_count(store); + + THEN("the first two are recorded; the third reports the failed commit but memory says sent") + { + REQUIRE(sent == DelayedEventOutcomeResult::recorded); + REQUIRE(failed == DelayedEventOutcomeResult::recorded); + REQUIRE(lost == DelayedEventOutcomeResult::commit_failed); + REQUIRE(after == before); + REQUIRE(store.delayed_events.find("d-ok")->status == DelayedEventStatus::sent); + REQUIRE(store.delayed_events.find("d-ok")->event_id == "$event"); + REQUIRE(store.delayed_events.find("d-bad")->status == DelayedEventStatus::failed); + REQUIRE(store.delayed_events.find("d-bad")->error_json == R"({"errcode":"M_FORBIDDEN","error":"no"})"); + REQUIRE(store.delayed_events.find("d-lost")->status == DelayedEventStatus::sent); + REQUIRE(store.delayed_events.find("d-lost")->finalised_at_ms == 5002U); + REQUIRE(store.delayed_events.find_sent_by_event_id("$lost") == store.delayed_events.find("d-lost")); + REQUIRE(store.delayed_events.scheduled_count(alice) == 0U); + REQUIRE(store.delayed_events.earliest_due() == nullptr); + REQUIRE_FALSE(store.delayed_events.is_being_sent("d-lost")); + } + } + + WHEN("an outcome is recorded with the wrong shape, for an absent row and for a finalised row") + { + auto const no_event = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 1U, {}, {}, 10U); + auto const no_error = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::failed, 1U, {}, {}, 10U); + auto const both = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 1U, "$e", "{}", 10U); + auto const cancelled = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::cancelled, 1U, {}, {}, 10U); + auto const scheduled = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::scheduled, 1U, "$e", {}, 10U); + auto const long_error = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::failed, 1U, {}, + std::string(db::max_delayed_event_error_bytes + 1U, 'e'), 10U); + auto const long_event = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 1U, + std::string(db::max_delayed_event_field_bytes + 1U, 'e'), {}, 10U); + auto const absent = + db::record_delayed_event_outcome(store, "d-nope", DelayedEventStatus::sent, 1U, "$e", {}, 10U); + REQUIRE(db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 1U, "$e", {}, 10U) == + DelayedEventOutcomeResult::recorded); + auto const twice = + db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 2U, "$f", {}, 10U); + auto const at_cap = + db::record_delayed_event_outcome(store, "d-bad", DelayedEventStatus::failed, 1U, {}, + std::string(db::max_delayed_event_error_bytes, 'e'), 10U); + + THEN("every one is refused and writes nothing, except the error exactly at its cap") + { + REQUIRE(no_event == DelayedEventOutcomeResult::refused); + REQUIRE(no_error == DelayedEventOutcomeResult::refused); + REQUIRE(both == DelayedEventOutcomeResult::refused); + REQUIRE(cancelled == DelayedEventOutcomeResult::refused); + REQUIRE(scheduled == DelayedEventOutcomeResult::refused); + REQUIRE(long_error == DelayedEventOutcomeResult::refused); + REQUIRE(long_event == DelayedEventOutcomeResult::refused); + REQUIRE(absent == DelayedEventOutcomeResult::refused); + REQUIRE(twice == DelayedEventOutcomeResult::refused); + REQUIRE(at_cap == DelayedEventOutcomeResult::recorded); + REQUIRE(store.delayed_events.find("d-ok")->event_id == "$e"); + } + } + } +} + +SCENARIO("Finalising keeps at most the bound of a user's finalised rows, evicting the oldest in the same transaction", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("a user holding three finalised rows and two scheduled ones, another user holding one finalised row") + { + auto opened = opened_store(); + auto& store = opened.store; + fill_finalised(store, "old", alice, 3U); + fill_finalised(store, "bob", bob, 1U); + REQUIRE(db::store_delayed_event(store, row("live-1"))); + REQUIRE(db::store_delayed_event(store, row("live-2"))); + db::enable_statement_capture(store, 16U); + + WHEN("a row is cancelled with a bound of 3") + { + auto const before = db::committed_transaction_count(store); + REQUIRE(db::cancel_delayed_event(store, "live-1", 900U, 3U)); + auto const transactions = db::committed_transaction_count(store) - before; + + THEN( + "the oldest row goes in the same transaction, the finalised row stays, and the other user is untouched") + { + REQUIRE(transactions == 1U); + REQUIRE(store.captured_statements.size() == 2U); + REQUIRE(store.captured_statements[0].name == "finalise_delayed_event"); + REQUIRE(store.captured_statements[1].name == "delete_delayed_events"); + REQUIRE(store.captured_statements[1].parameters.size() == 1U); + REQUIRE(store.captured_statements[1].parameters[0].value == "old-0"); + REQUIRE(store.delayed_events.finalised_count(alice) == 3U); + REQUIRE(store.delayed_events.find("old-0") == nullptr); + REQUIRE(store.delayed_events.find("old-1") != nullptr); + REQUIRE(store.delayed_events.find("live-1")->status == DelayedEventStatus::cancelled); + REQUIRE(store.delayed_events.finalised_count(bob) == 1U); + } + } + + WHEN("a row is recorded as sent with a bound of 1") + { + auto const before = db::committed_transaction_count(store); + auto const result = + db::record_delayed_event_outcome(store, "live-2", DelayedEventStatus::sent, 901U, "$e", {}, 1U); + auto const transactions = db::committed_transaction_count(store) - before; + + THEN("only the row just finalised remains for that user, all in one transaction") + { + REQUIRE(result == DelayedEventOutcomeResult::recorded); + REQUIRE(transactions == 1U); + REQUIRE(store.delayed_events.finalised_count(alice) == 1U); + REQUIRE(store.delayed_events.find("live-2")->status == DelayedEventStatus::sent); + REQUIRE(store.delayed_events.find("old-0") == nullptr); + REQUIRE(store.delayed_events.find("old-1") == nullptr); + REQUIRE(store.delayed_events.find("old-2") == nullptr); + } + } + + WHEN("a row is cancelled with a bound well above what the user holds") + { + auto const before = db::committed_transaction_count(store); + REQUIRE(db::cancel_delayed_event(store, "live-1", 900U, 100U)); + + THEN("nothing is evicted and the transaction holds the one update") + { + REQUIRE(db::committed_transaction_count(store) - before == 1U); + REQUIRE(store.captured_statements.size() == 1U); + REQUIRE(store.delayed_events.finalised_count(alice) == 4U); + } + } + + WHEN("the evicting commit fails") + { + db::force_next_persist_failures(store, 1U); + auto const cancelled = db::cancel_delayed_event(store, "live-1", 900U, 1U); + + THEN("neither the status nor the evictions reach memory") + { + REQUIRE_FALSE(cancelled); + REQUIRE(store.delayed_events.finalised_count(alice) == 3U); + REQUIRE(store.delayed_events.find("live-1")->status == DelayedEventStatus::scheduled); + } + } + } + + GIVEN("a user holding 300 finalised rows") + { + auto opened = opened_store(); + auto& store = opened.store; + fill_finalised(store, "many", alice, 300U); + REQUIRE(db::store_delayed_event(store, row("live"))); + db::enable_statement_capture(store, 16U); + + WHEN("one more row is cancelled with a bound of 100") + { + auto const before = db::committed_transaction_count(store); + REQUIRE(db::cancel_delayed_event(store, "live", 5000U, 100U)); + + THEN("201 rows are deleted in batches within one transaction, leaving the newest 99 and the new row") + { + REQUIRE(db::committed_transaction_count(store) - before == 1U); + REQUIRE(store.captured_statements.size() == 3U); + REQUIRE(store.delayed_events.finalised_count(alice) == 100U); + REQUIRE(store.delayed_events.find("many-200") == nullptr); + REQUIRE(store.delayed_events.find("many-201") != nullptr); + REQUIRE(store.delayed_events.find("live")->status == DelayedEventStatus::cancelled); + } + } + } +} + +SCENARIO("All of a user's delayed events are deleted in one statement", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("two users with scheduled and finalised rows") + { + auto opened = opened_store(); + auto& store = opened.store; + fill_finalised(store, "a-done", alice, 2U); + REQUIRE(db::store_delayed_event(store, row("a-live"))); + REQUIRE(db::store_delayed_event(store, row("b-live", bob))); + db::enable_statement_capture(store, 16U); + + WHEN("the first user's rows are deleted, and then those of a user with none") + { + auto const before = db::committed_transaction_count(store); + auto const deleted = db::delete_delayed_events_of_user(store, alice); + auto const transactions = db::committed_transaction_count(store) - before; + auto const none = db::delete_delayed_events_of_user(store, "@nobody:example.org"); + + THEN("one DELETE by user removes them all, the other user keeps theirs, and no write is made for none") + { + REQUIRE(deleted); + REQUIRE(transactions == 1U); + REQUIRE(store.captured_statements.size() == 1U); + REQUIRE(store.captured_statements[0].sql == "DELETE FROM delayed_events WHERE user_id = $1"); + REQUIRE(store.captured_statements[0].parameters[0].value == std::string{alice}); + REQUIRE(none); + REQUIRE(db::committed_transaction_count(store) - before == 1U); + REQUIRE(store.delayed_events.size() == 1U); + REQUIRE(store.delayed_events.find("b-live") != nullptr); + } + } + } +} + +SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up to the row limit", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("finalised rows at 100 to 109, a scheduled row and a finalised row of another user at 50") + { + auto opened = opened_store(); + auto& store = opened.store; + fill_finalised(store, "r", alice, 10U); + fill_finalised(store, "early", bob, 1U); + REQUIRE(db::cancel_delayed_event(store, "early-0", 1U, 100U) == false); + REQUIRE(db::store_delayed_event(store, row("live"))); + db::enable_statement_capture(store, 16U); + + WHEN("rows below 105 are pruned with a limit of 3") + { + auto const before = db::committed_transaction_count(store); + auto const pruned = db::prune_finalised_delayed_events(store, 105U, 3U); + + THEN("the three oldest go in one transaction and everything else stays") + { + REQUIRE(pruned == std::optional{3U}); + REQUIRE(db::committed_transaction_count(store) - before == 1U); + REQUIRE(store.delayed_events.find("r-0") == nullptr); + REQUIRE(store.delayed_events.find("r-1") == nullptr); + REQUIRE(store.delayed_events.find("r-2") != nullptr); + REQUIRE(store.delayed_events.find("live") != nullptr); + } + } + + WHEN("rows below 105 are pruned with a limit larger than the number due") + { + auto const pruned = db::prune_finalised_delayed_events(store, 105U, 500U); + auto const rest = db::prune_finalised_delayed_events(store, 105U, 500U); + + THEN("exactly the rows below the cutoff go, the cutoff row stays, and a second prune finds none") + { + REQUIRE(pruned == std::optional{6U}); + REQUIRE(rest == std::optional{0U}); + REQUIRE(store.delayed_events.find("r-4") == nullptr); + REQUIRE(store.delayed_events.find("r-5") != nullptr); + REQUIRE(store.delayed_events.find("early-0") == nullptr); + } + } + + WHEN("the limit is zero, or the cutoff is at the oldest row") + { + auto const before = db::committed_transaction_count(store); + auto const zero = db::prune_finalised_delayed_events(store, 1'000'000U, 0U); + auto const nothing = db::prune_finalised_delayed_events(store, 100U, 10U); + + THEN("nothing is deleted and nothing is written") + { + REQUIRE(zero == std::optional{0U}); + REQUIRE(nothing == std::optional{0U}); + REQUIRE(db::committed_transaction_count(store) == before); + REQUIRE(store.delayed_events.size() == 12U); + } + } + } + + GIVEN("300 finalised rows") + { + auto opened = opened_store(); + auto& store = opened.store; + fill_finalised(store, "bulk", alice, 300U); + db::enable_statement_capture(store, 16U); + + WHEN("they are all pruned") + { + auto const before = db::committed_transaction_count(store); + auto const pruned = db::prune_finalised_delayed_events(store, 1'000'000U, 500U); + + THEN("the DELETEs are batched in one transaction") + { + REQUIRE(pruned == std::optional{300U}); + REQUIRE(db::committed_transaction_count(store) - before == 1U); + REQUIRE(store.captured_statements.size() == 3U); + REQUIRE(store.delayed_events.size() == 0U); + } + } + } +} + +SCENARIO("Delayed events survive a reopen of a SQLite store with every column and index", + "[database][delayed-events][delayed-event-store][sqlite]") +{ + GIVEN("a SQLite store holding a scheduled state event, a sent row, a cancelled row and a failed row") + { + auto const path = merovingian::tests::temporary_directory() / + ("merovingian-delayed-events-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".sqlite3"); + auto first = PersistentDelayedEvent{}; + { + auto opened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(opened.ok); + first = row("d-state"); + first.is_state = true; + first.state_key = "@alice:example.org"; + first.device_id = ""; + first.via_appservice = true; + first.txn_id = ""; + first.delay_ms = db::max_delayed_event_delay_ms; + first.delayed_since_ms = 123'456'789U; + auto message = row("d-message"); + message.txn_id = "txn-1"; + message.content_json = R"({"body":"line\nbreak \"quoted\" é"})"; + REQUIRE(db::store_delayed_event(opened.store, first)); + REQUIRE(db::store_delayed_event(opened.store, message)); + REQUIRE(db::store_delayed_event(opened.store, row("d-sent"))); + REQUIRE(db::store_delayed_event(opened.store, row("d-cancelled"))); + REQUIRE(db::store_delayed_event(opened.store, row("d-failed", bob))); + REQUIRE(db::record_delayed_event_outcome(opened.store, "d-sent", DelayedEventStatus::sent, 2000U, + "$sent:example.org", {}, + 10U) == DelayedEventOutcomeResult::recorded); + REQUIRE(db::cancel_delayed_event(opened.store, "d-cancelled", 2001U, 10U)); + REQUIRE(db::record_delayed_event_outcome(opened.store, "d-failed", DelayedEventStatus::failed, 2002U, {}, + R"({"errcode":"M_FORBIDDEN","error":"nope"})", + 10U) == DelayedEventOutcomeResult::recorded); + } + + WHEN("the store is opened again") + { + auto reopened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(reopened.ok); + auto const& table = reopened.store.delayed_events; + + THEN("every row is back with every column, and every index answers") + { + REQUIRE(table.size() == 5U); + auto const* state = table.find("d-state"); + REQUIRE(state != nullptr); + REQUIRE(state->user_id == alice); + REQUIRE(state->device_id.empty()); + REQUIRE(state->room_id == "!room:example.org"); + REQUIRE(state->event_type == "m.room.message"); + REQUIRE(state->is_state); + REQUIRE(state->state_key == "@alice:example.org"); + REQUIRE(state->content_json == std::string{secret_content}); + REQUIRE(state->txn_id.empty()); + REQUIRE(state->via_appservice); + REQUIRE(state->delay_ms == db::max_delayed_event_delay_ms); + REQUIRE(state->delayed_since_ms == 123'456'789U); + REQUIRE(state->status == DelayedEventStatus::scheduled); + REQUIRE(state->finalised_at_ms == 0U); + REQUIRE(table.find("d-message")->content_json == R"({"body":"line\nbreak \"quoted\" é"})"); + REQUIRE_FALSE(table.find("d-message")->is_state); + REQUIRE_FALSE(table.find("d-message")->via_appservice); + REQUIRE(table.find("d-sent")->status == DelayedEventStatus::sent); + REQUIRE(table.find("d-sent")->event_id == "$sent:example.org"); + REQUIRE(table.find("d-sent")->finalised_at_ms == 2000U); + REQUIRE(table.find("d-cancelled")->status == DelayedEventStatus::cancelled); + REQUIRE(table.find("d-failed")->status == DelayedEventStatus::failed); + REQUIRE(table.find("d-failed")->error_json == R"({"errcode":"M_FORBIDDEN","error":"nope"})"); + REQUIRE(table.find_owned(bob, "d-failed") != nullptr); + REQUIRE(table.find_owned(alice, "d-failed") == nullptr); + REQUIRE(table.scheduled_count(alice) == 2U); + REQUIRE(table.finalised_count(alice) == 2U); + REQUIRE(table.finalised_count(bob) == 1U); + REQUIRE(table.earliest_due() == table.find("d-message")); + REQUIRE(table.earliest_due_ms_of_user(alice) == std::optional{61'000U}); + REQUIRE(table.find_by_transaction(alice, "DEVICE", "!room:example.org", "m.room.message", "txn-1") == + table.find("d-message")); + REQUIRE(table.find_sent_by_event_id("$sent:example.org") == table.find("d-sent")); + REQUIRE(table.finalised_before(2002U, 10U).size() == 2U); + } + } + + WHEN("the file holds a row whose status text is not one of the four, and the store is opened") + { + { + auto* raw = static_cast(nullptr); + REQUIRE(sqlite3_open_v2(path.string().c_str(), &raw, SQLITE_OPEN_READWRITE, nullptr) == SQLITE_OK); + auto const connection = std::unique_ptr{raw}; + REQUIRE(sqlite3_exec(connection.get(), + "UPDATE delayed_events SET status = 'done' WHERE delay_id = 'd-sent'", nullptr, + nullptr, nullptr) == SQLITE_OK); + } + auto reopened = db::open_sqlite_persistent_store(path.string()); + + THEN("that row is not loaded, never repaired, and the others are") + { + REQUIRE(reopened.ok); + REQUIRE(reopened.store.delayed_events.size() == 4U); + REQUIRE(reopened.store.delayed_events.find("d-sent") == nullptr); + REQUIRE(reopened.store.delayed_events.find("d-state") != nullptr); + REQUIRE(sqlite_first_text(path, "SELECT status FROM delayed_events WHERE delay_id = 'd-sent'") == + "done"); + } + } + + WHEN("the rows are deleted by user and by prune and the store is opened again") + { + { + auto opened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(opened.ok); + REQUIRE(db::delete_delayed_events_of_user(opened.store, bob)); + REQUIRE(db::prune_finalised_delayed_events(opened.store, 2001U, 10U) == std::optional{1U}); + } + auto reopened = db::open_sqlite_persistent_store(path.string()); + + THEN("the file agrees with memory") + { + REQUIRE(reopened.ok); + REQUIRE(reopened.store.delayed_events.size() == 3U); + REQUIRE(reopened.store.delayed_events.find("d-failed") == nullptr); + REQUIRE(reopened.store.delayed_events.find("d-sent") == nullptr); + REQUIRE(reopened.store.delayed_events.find("d-cancelled") != nullptr); + REQUIRE(sqlite_first_text(path, "SELECT count(*) FROM delayed_events") == "3"); + } + } + std::filesystem::remove(path); + } +} diff --git a/tests/unit/test_delayed_event_table.cpp b/tests/unit/test_delayed_event_table.cpp new file mode 100644 index 000000000..b413c131b --- /dev/null +++ b/tests/unit/test_delayed_event_table.cpp @@ -0,0 +1,597 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// The in-memory delayed-events table (migration 003, ADR-0824): every index, the due order, the ownership lookup, the +// being-sent mark, restart re-ordering, finalising, erasing, and that no lookup scans the rows. +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/DelayedEventStatusText.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentDelayedEvent.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +using merovingian::database::DelayedEventStatus; +using merovingian::database::DelayedEventTable; +using merovingian::database::PersistentDelayedEvent; + +constexpr auto largest_delay_ms = std::uint64_t{2'678'400'000U}; + +[[nodiscard]] auto scheduled(std::string delay_id, std::string user_id, std::uint64_t since_ms, std::uint64_t delay_ms, + std::string txn_id = {}) -> PersistentDelayedEvent +{ + auto row = PersistentDelayedEvent{}; + row.delay_id = std::move(delay_id); + row.user_id = std::move(user_id); + row.device_id = "DEVICE"; + row.room_id = "!room:example.org"; + row.event_type = "m.room.message"; + row.content_json = "{}"; + row.txn_id = std::move(txn_id); + row.delay_ms = delay_ms; + row.delayed_since_ms = since_ms; + return row; +} + +[[nodiscard]] auto due_id(DelayedEventTable const& table) -> std::string +{ + auto const* row = table.earliest_due(); + return row == nullptr ? std::string{""} : row->delay_id; +} + +[[nodiscard]] auto ids_of(std::vector const& rows) -> std::vector +{ + auto ids = std::vector{}; + for (auto const* row : rows) + { + ids.push_back(row->delay_id); + } + return ids; +} + +} // namespace + +SCENARIO("The status text round-trips and any other text is not a status", "[database][delayed-events]") +{ + GIVEN("the four statuses") + { + WHEN("each is turned into column text and parsed back") + { + THEN("the text is the one the schema documents, and unknown text is refused") + { + using merovingian::database::delayed_event_status_text; + using merovingian::database::parse_delayed_event_status; + REQUIRE(delayed_event_status_text(DelayedEventStatus::scheduled) == "scheduled"); + REQUIRE(delayed_event_status_text(DelayedEventStatus::sent) == "sent"); + REQUIRE(delayed_event_status_text(DelayedEventStatus::cancelled) == "cancelled"); + REQUIRE(delayed_event_status_text(DelayedEventStatus::failed) == "failed"); + for (auto const status : {DelayedEventStatus::scheduled, DelayedEventStatus::sent, + DelayedEventStatus::cancelled, DelayedEventStatus::failed}) + { + REQUIRE(parse_delayed_event_status(delayed_event_status_text(status)) == status); + } + REQUIRE_FALSE(parse_delayed_event_status("").has_value()); + REQUIRE_FALSE(parse_delayed_event_status("Scheduled").has_value()); + REQUIRE_FALSE(parse_delayed_event_status("scheduled ").has_value()); + REQUIRE_FALSE(parse_delayed_event_status("done").has_value()); + } + } + } +} + +SCENARIO("A delayed event is found by ID and only by its owner", "[database][delayed-events]") +{ + GIVEN("a table holding one row of each of two users") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("d-a", "@a:example.org", 0U, 1000U))); + REQUIRE(table.insert(scheduled("d-b", "@b:example.org", 0U, 1000U))); + + WHEN("the rows are looked up") + { + THEN("an unknown ID and another user's ID are the same nullptr, and the owner finds its row") + { + REQUIRE(table.size() == 2U); + REQUIRE(table.find("d-a") != nullptr); + REQUIRE(table.find_owned("@a:example.org", "d-a") == table.find("d-a")); + REQUIRE(table.find_owned("@b:example.org", "d-a") == nullptr); + REQUIRE(table.find_owned("@a:example.org", "d-missing") == nullptr); + REQUIRE(table.find_owned("@a:example.org", "") == nullptr); + REQUIRE(table.find("d-missing") == nullptr); + } + } + + WHEN("the same delay ID is inserted again, by the same or another user") + { + auto const same_user = table.insert(scheduled("d-a", "@a:example.org", 5U, 5U)); + auto const other_user = table.insert(scheduled("d-a", "@b:example.org", 5U, 5U)); + + THEN("both are refused and the first row is unchanged") + { + REQUIRE_FALSE(same_user); + REQUIRE_FALSE(other_user); + REQUIRE(table.size() == 2U); + REQUIRE(table.find("d-a")->user_id == "@a:example.org"); + REQUIRE(table.find("d-a")->delay_ms == 1000U); + } + } + } +} + +SCENARIO("Scheduled rows come due in due-time order, ties by delay ID", "[database][delayed-events]") +{ + GIVEN("rows due at 5000 (two of them), 3000 and 9000") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("d-9000", "@a:example.org", 8000U, 1000U))); + REQUIRE(table.insert(scheduled("d-5000-b", "@a:example.org", 4000U, 1000U))); + REQUIRE(table.insert(scheduled("d-3000", "@b:example.org", 0U, 3000U))); + REQUIRE(table.insert(scheduled("d-5000-a", "@b:example.org", 1000U, 4000U))); + + WHEN("the earliest is taken and finalised, again and again") + { + auto order = std::vector{}; + while (table.earliest_due() != nullptr) + { + auto const id = due_id(table); + order.push_back(id); + REQUIRE(table.finalise(id, DelayedEventStatus::cancelled, 1U, {}, {})); + } + + THEN("the order is by due time, and the two equal due times by delay ID") + { + REQUIRE(order == std::vector{"d-3000", "d-5000-a", "d-5000-b", "d-9000"}); + } + } + + WHEN("the due time of a row is asked for") + { + THEN("it is the since time plus the delay") + { + REQUIRE(merovingian::database::delayed_event_due_ms(*table.find("d-9000")) == 9000U); + REQUIRE(merovingian::database::delayed_event_due_ms(*table.find("d-5000-a")) == 5000U); + } + } + } + + GIVEN("a row with the largest allowed delay and one whose sum would overflow") + { + auto table = DelayedEventTable{}; + constexpr auto largest = std::numeric_limits::max(); + REQUIRE(table.insert(scheduled("d-largest", "@a:example.org", 1'000'000U, largest_delay_ms))); + REQUIRE(table.insert(scheduled("d-wraps", "@a:example.org", largest - 10U, 100U))); + REQUIRE(table.insert(scheduled("d-small", "@a:example.org", 1U, 1U))); + + WHEN("the due times are computed and the order is read") + { + THEN("the largest delay is exact, the overflowing sum saturates, and the saturated row is last") + { + REQUIRE(merovingian::database::delayed_event_due_ms(*table.find("d-largest")) == + 1'000'000U + largest_delay_ms); + REQUIRE(merovingian::database::delayed_event_due_ms(*table.find("d-wraps")) == largest); + REQUIRE(due_id(table) == "d-small"); + REQUIRE(table.finalise("d-small", DelayedEventStatus::cancelled, 1U, {}, {})); + REQUIRE(due_id(table) == "d-largest"); + REQUIRE(table.finalise("d-largest", DelayedEventStatus::cancelled, 1U, {}, {})); + REQUIRE(due_id(table) == "d-wraps"); + } + } + } +} + +SCENARIO("A row marked as being sent is not offered as due and is reported as being sent", "[database][delayed-events]") +{ + GIVEN("two scheduled rows, the earlier one marked as being sent") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("d-first", "@a:example.org", 0U, 1000U))); + REQUIRE(table.insert(scheduled("d-second", "@a:example.org", 0U, 2000U))); + REQUIRE(table.mark_being_sent("d-first")); + + WHEN("the due order is read") + { + THEN("the marked row is hidden from it but still counted as scheduled and still the user's earliest") + { + REQUIRE(table.is_being_sent("d-first")); + REQUIRE_FALSE(table.is_being_sent("d-second")); + REQUIRE(due_id(table) == "d-second"); + REQUIRE(table.scheduled_count("@a:example.org") == 2U); + REQUIRE(table.earliest_due_ms_of_user("@a:example.org") == std::optional{1000U}); + REQUIRE_FALSE(table.mark_being_sent("d-first")); + REQUIRE_FALSE(table.mark_being_sent("d-missing")); + } + } + + WHEN("the mark is cleared") + { + REQUIRE(table.clear_being_sent("d-first")); + auto const again = table.clear_being_sent("d-first"); + + THEN("the row is offered again, and clearing twice reports it was not marked") + { + REQUIRE_FALSE(again); + REQUIRE_FALSE(table.is_being_sent("d-first")); + REQUIRE(due_id(table) == "d-first"); + } + } + + WHEN("the marked row is restarted and then unmarked") + { + REQUIRE(table.restart("d-first", 10'000U)); + REQUIRE(table.clear_being_sent("d-first")); + + THEN("it comes back at its new place in the order") + { + REQUIRE(due_id(table) == "d-second"); + REQUIRE(table.finalise("d-second", DelayedEventStatus::cancelled, 1U, {}, {})); + REQUIRE(due_id(table) == "d-first"); + } + } + + WHEN("the marked row is finalised") + { + REQUIRE(table.finalise("d-first", DelayedEventStatus::sent, 7U, "$event", {})); + + THEN("the mark is gone with it") + { + REQUIRE_FALSE(table.is_being_sent("d-first")); + REQUIRE_FALSE(table.clear_being_sent("d-first")); + REQUIRE(due_id(table) == "d-second"); + } + } + + WHEN("a finalised row is marked") + { + REQUIRE(table.finalise("d-second", DelayedEventStatus::cancelled, 7U, {}, {})); + + THEN("the mark is refused") + { + REQUIRE_FALSE(table.mark_being_sent("d-second")); + REQUIRE_FALSE(table.is_being_sent("d-second")); + } + } + } +} + +SCENARIO("A restart moves a scheduled row in the due order and a finalised row cannot be restarted", + "[database][delayed-events]") +{ + GIVEN("three scheduled rows, one already cancelled") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("d-1", "@a:example.org", 0U, 1000U))); + REQUIRE(table.insert(scheduled("d-2", "@a:example.org", 0U, 2000U))); + REQUIRE(table.insert(scheduled("d-3", "@a:example.org", 0U, 3000U))); + REQUIRE(table.insert(scheduled("d-4", "@a:example.org", 0U, 500U))); + REQUIRE(table.finalise("d-4", DelayedEventStatus::cancelled, 9U, {}, {})); + + WHEN("the first row is restarted at 2500") + { + REQUIRE(table.restart("d-1", 2500U)); + + THEN("it is due at 3500, after the other two, and the user's earliest due follows") + { + REQUIRE(table.find("d-1")->delayed_since_ms == 2500U); + REQUIRE(due_id(table) == "d-2"); + REQUIRE(table.earliest_due_ms_of_user("@a:example.org") == std::optional{2000U}); + REQUIRE(table.finalise("d-2", DelayedEventStatus::cancelled, 1U, {}, {})); + REQUIRE(due_id(table) == "d-3"); + REQUIRE(table.finalise("d-3", DelayedEventStatus::cancelled, 1U, {}, {})); + REQUIRE(due_id(table) == "d-1"); + } + } + + WHEN("a finalised row and an unknown row are restarted") + { + auto const finalised = table.restart("d-4", 100U); + auto const unknown = table.restart("d-nope", 100U); + + THEN("both are refused and nothing moves") + { + REQUIRE_FALSE(finalised); + REQUIRE_FALSE(unknown); + REQUIRE(table.find("d-4")->delayed_since_ms == 0U); + REQUIRE(due_id(table) == "d-1"); + } + } + } +} + +SCENARIO("Finalising moves a row from the scheduled to the finalised structures", "[database][delayed-events]") +{ + GIVEN("a user with two scheduled rows and another with one") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("d-1", "@a:example.org", 0U, 1000U, "txn-1"))); + REQUIRE(table.insert(scheduled("d-2", "@a:example.org", 0U, 2000U, "txn-2"))); + REQUIRE(table.insert(scheduled("d-3", "@b:example.org", 0U, 3000U))); + + WHEN("one row is finalised as sent with an event ID") + { + REQUIRE(table.finalise("d-1", DelayedEventStatus::sent, 500U, "$event:example.org", {})); + auto const again = table.finalise("d-1", DelayedEventStatus::cancelled, 600U, {}, {}); + + THEN("the counts move, the row is findable by event ID, and a second finalise is refused") + { + REQUIRE_FALSE(again); + REQUIRE(table.scheduled_count("@a:example.org") == 1U); + REQUIRE(table.finalised_count("@a:example.org") == 1U); + REQUIRE(table.scheduled_count("@b:example.org") == 1U); + REQUIRE(table.finalised_count("@b:example.org") == 0U); + auto const* row = table.find("d-1"); + REQUIRE(row->status == DelayedEventStatus::sent); + REQUIRE(row->finalised_at_ms == 500U); + REQUIRE(row->event_id == "$event:example.org"); + REQUIRE(table.find_sent_by_event_id("$event:example.org") == row); + REQUIRE(table.find_sent_by_event_id("$other:example.org") == nullptr); + REQUIRE(due_id(table) == "d-2"); + REQUIRE(table.earliest_due_ms_of_user("@a:example.org") == std::optional{2000U}); + } + } + + WHEN("rows are finalised as cancelled and as failed, and scheduled is passed as the new status") + { + auto const to_scheduled = table.finalise("d-1", DelayedEventStatus::scheduled, 1U, {}, {}); + REQUIRE(table.finalise("d-1", DelayedEventStatus::failed, 40U, {}, R"({"errcode":"M_FORBIDDEN"})")); + REQUIRE(table.finalise("d-2", DelayedEventStatus::cancelled, 30U, {}, {})); + auto const unknown = table.finalise("d-nope", DelayedEventStatus::cancelled, 30U, {}, {}); + + THEN("a finalise back to scheduled and one of an unknown row are refused, and the error is kept") + { + REQUIRE_FALSE(to_scheduled); + REQUIRE_FALSE(unknown); + REQUIRE(table.find("d-1")->error_json == R"({"errcode":"M_FORBIDDEN"})"); + REQUIRE(table.find("d-1")->event_id.empty()); + REQUIRE(table.scheduled_count("@a:example.org") == 0U); + REQUIRE_FALSE(table.earliest_due_ms_of_user("@a:example.org").has_value()); + REQUIRE(table.find_sent_by_event_id("") == nullptr); + REQUIRE(ids_of(table.finalised_of_user("@a:example.org", 10U)) == + std::vector{"d-2", "d-1"}); + } + } + } +} + +SCENARIO("A user's finalised rows and the whole table's finalised rows are read oldest first", + "[database][delayed-events]") +{ + GIVEN("finalised rows of two users at several times, two of them at the same time") + { + auto table = DelayedEventTable{}; + for (auto const& [id, user, when] : std::vector>{ + {"d-a", "@a:example.org", 300U}, + {"d-b", "@a:example.org", 100U}, + {"d-c", "@b:example.org", 200U}, + {"d-d", "@a:example.org", 100U}, + {"d-e", "@b:example.org", 400U} + }) + { + REQUIRE(table.insert(scheduled(id, user, 0U, 1000U))); + REQUIRE(table.finalise(id, DelayedEventStatus::cancelled, when, {}, {})); + } + REQUIRE(table.insert(scheduled("d-live", "@a:example.org", 0U, 1000U))); + + WHEN("one user's rows and the table's rows are read") + { + THEN("each list is by finalised time then delay ID, bounded by the limit and the cutoff") + { + REQUIRE(ids_of(table.finalised_of_user("@a:example.org", 10U)) == + std::vector{"d-b", "d-d", "d-a"}); + REQUIRE(ids_of(table.finalised_of_user("@a:example.org", 2U)) == + std::vector{"d-b", "d-d"}); + REQUIRE(table.finalised_of_user("@a:example.org", 0U).empty()); + REQUIRE(table.finalised_of_user("@nobody:example.org", 5U).empty()); + REQUIRE(table.finalised_count("@a:example.org") == 3U); + REQUIRE(ids_of(table.finalised_before(1000U, 10U)) == + std::vector{"d-b", "d-d", "d-c", "d-a", "d-e"}); + REQUIRE(ids_of(table.finalised_before(1000U, 3U)) == std::vector{"d-b", "d-d", "d-c"}); + REQUIRE(ids_of(table.finalised_before(300U, 10U)) == std::vector{"d-b", "d-d", "d-c"}); + REQUIRE(ids_of(table.finalised_before(301U, 10U)).size() == 4U); + REQUIRE(table.finalised_before(100U, 10U).empty()); + REQUIRE(table.finalised_before(1000U, 0U).empty()); + } + } + } +} + +SCENARIO("A transaction ID is looked up by user, device, room, event type and ID, and an empty one is not indexed", + "[database][delayed-events]") +{ + GIVEN("a row with a transaction ID and two rows with none") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("d-txn", "@a:example.org", 0U, 1000U, "txn-1"))); + REQUIRE(table.insert(scheduled("d-none-1", "@a:example.org", 0U, 1000U))); + REQUIRE(table.insert(scheduled("d-none-2", "@a:example.org", 0U, 1000U))); + + WHEN("each part of the key is changed in turn") + { + auto const row = table.find("d-txn"); + auto const find = [&table](std::string_view user, std::string_view device, std::string_view room, + std::string_view type, std::string_view txn) { + return table.find_by_transaction(user, device, room, type, txn); + }; + + THEN("only the exact tuple finds the row, and an empty transaction ID finds nothing") + { + REQUIRE(find("@a:example.org", "DEVICE", "!room:example.org", "m.room.message", "txn-1") == row); + REQUIRE(find("@b:example.org", "DEVICE", "!room:example.org", "m.room.message", "txn-1") == nullptr); + REQUIRE(find("@a:example.org", "OTHER", "!room:example.org", "m.room.message", "txn-1") == nullptr); + REQUIRE(find("@a:example.org", "DEVICE", "!other:example.org", "m.room.message", "txn-1") == nullptr); + REQUIRE(find("@a:example.org", "DEVICE", "!room:example.org", "m.room.other", "txn-1") == nullptr); + REQUIRE(find("@a:example.org", "DEVICE", "!room:example.org", "m.room.message", "txn-2") == nullptr); + REQUIRE(find("@a:example.org", "DEVICE", "!room:example.org", "m.room.message", "") == nullptr); + } + } + + WHEN("fields that would join into the same text are split differently") + { + auto shifted = scheduled("d-shift", "@a:example.org", 0U, 1000U, "txn-1"); + shifted.device_id = "DEVICE!room:example.org"; + shifted.room_id = ""; + auto const accepted = table.insert(shifted); + + THEN("the keys differ, so the row is accepted and the first row is still the one found") + { + REQUIRE(accepted); + REQUIRE(table.find_by_transaction("@a:example.org", "DEVICE", "!room:example.org", "m.room.message", + "txn-1") == table.find("d-txn")); + REQUIRE(table.find_by_transaction("@a:example.org", "DEVICE!room:example.org", "", "m.room.message", + "txn-1") == table.find("d-shift")); + } + } + + WHEN("a second row with the same transaction tuple is inserted") + { + auto const accepted = table.insert(scheduled("d-twin", "@a:example.org", 0U, 1000U, "txn-1")); + REQUIRE(table.finalise("d-txn", DelayedEventStatus::cancelled, 5U, {}, {})); + + THEN("it is refused, and the finalised row still answers the lookup (any status)") + { + REQUIRE_FALSE(accepted); + REQUIRE(table.find("d-twin") == nullptr); + REQUIRE(table.find_by_transaction("@a:example.org", "DEVICE", "!room:example.org", "m.room.message", + "txn-1") == table.find("d-txn")); + } + } + } +} + +SCENARIO("Erasing a row, or all of a user's rows, removes it from every index", "[database][delayed-events]") +{ + GIVEN("two users with scheduled, sent and cancelled rows") + { + auto table = DelayedEventTable{}; + REQUIRE(table.insert(scheduled("a-1", "@a:example.org", 0U, 1000U, "txn-a"))); + REQUIRE(table.insert(scheduled("a-2", "@a:example.org", 0U, 2000U))); + REQUIRE(table.insert(scheduled("a-3", "@a:example.org", 0U, 3000U))); + REQUIRE(table.insert(scheduled("b-1", "@b:example.org", 0U, 4000U))); + REQUIRE(table.insert(scheduled("b-2", "@b:example.org", 0U, 5000U))); + REQUIRE(table.finalise("a-2", DelayedEventStatus::sent, 10U, "$sent-a", {})); + REQUIRE(table.finalise("a-3", DelayedEventStatus::cancelled, 11U, {}, {})); + REQUIRE(table.finalise("b-2", DelayedEventStatus::sent, 12U, "$sent-b", {})); + REQUIRE(table.mark_being_sent("b-1")); + + WHEN("one row is erased") + { + auto const erased = table.erase("a-2"); + auto const again = table.erase("a-2"); + + THEN("it leaves the table, its event index and its user's finalised list; a second erase is false") + { + REQUIRE(erased); + REQUIRE_FALSE(again); + REQUIRE(table.size() == 4U); + REQUIRE(table.find("a-2") == nullptr); + REQUIRE(table.find_sent_by_event_id("$sent-a") == nullptr); + REQUIRE(table.finalised_count("@a:example.org") == 1U); + REQUIRE(table.find_sent_by_event_id("$sent-b") != nullptr); + } + } + + WHEN("all of the first user's rows are erased") + { + auto const erased = table.erase_user("@a:example.org"); + auto const none = table.erase_user("@a:example.org"); + + THEN("every row of that user is gone from every index and the other user's rows are untouched") + { + REQUIRE(erased == 3U); + REQUIRE(none == 0U); + REQUIRE(table.size() == 2U); + REQUIRE(table.scheduled_count("@a:example.org") == 0U); + REQUIRE(table.finalised_count("@a:example.org") == 0U); + REQUIRE(table.find_by_transaction("@a:example.org", "DEVICE", "!room:example.org", "m.room.message", + "txn-a") == nullptr); + REQUIRE(table.find_sent_by_event_id("$sent-a") == nullptr); + REQUIRE(ids_of(table.finalised_before(1000U, 10U)) == std::vector{"b-2"}); + REQUIRE(table.scheduled_count("@b:example.org") == 1U); + REQUIRE(table.find_owned("@b:example.org", "b-1") != nullptr); + } + } + + WHEN("the row marked as being sent is erased") + { + REQUIRE(table.erase("b-1")); + + THEN("the mark goes with it") + { + REQUIRE_FALSE(table.is_being_sent("b-1")); + REQUIRE(due_id(table) == "a-1"); + } + } + + WHEN("the table is cleared") + { + table.clear(); + + THEN("nothing is left in any index") + { + REQUIRE(table.size() == 0U); + REQUIRE(table.earliest_due() == nullptr); + REQUIRE(table.scheduled_count("@b:example.org") == 0U); + REQUIRE_FALSE(table.is_being_sent("b-1")); + REQUIRE(table.finalised_before(1000U, 10U).empty()); + REQUIRE(table.find_sent_by_event_id("$sent-b") == nullptr); + } + } + } +} + +SCENARIO("No lookup of the delayed-events table examines more rows than it returns", "[database][delayed-events]") +{ + GIVEN("a table of 2000 rows across 20 users, half of them with a transaction ID") + { + auto table = DelayedEventTable{}; + for (auto index = std::size_t{0U}; index < 2000U; ++index) + { + auto const user = "@user" + std::to_string(index % 20U) + ":example.org"; + auto const txn = index % 2U == 0U ? "txn-" + std::to_string(index) : std::string{}; + REQUIRE(table.insert(scheduled("d-" + std::to_string(index), user, index, 100'000U, txn))); + } + REQUIRE(table.finalise("d-7", DelayedEventStatus::sent, 5U, "$seven", {})); + auto const before = table.rows_examined(); + + WHEN("each single-key question is asked once") + { + auto const* by_id = table.find("d-1500"); + auto const* owned = table.find_owned("@user0:example.org", "d-1500"); + auto const* due = table.earliest_due(); + auto const* by_txn = table.find_by_transaction("@user0:example.org", "DEVICE", "!room:example.org", + "m.room.message", "txn-1500"); + auto const* by_event = table.find_sent_by_event_id("$seven"); + auto const count = table.scheduled_count("@user3:example.org"); + auto const earliest = table.earliest_due_ms_of_user("@user3:example.org"); + auto const finalised = table.finalised_of_user("@user7:example.org", 5U); + auto const old = table.finalised_before(1000U, 5U); + auto const examined = table.rows_examined() - before; + + THEN("the answers are right and the rows examined stay at the few rows the answers hold") + { + REQUIRE(by_id != nullptr); + REQUIRE(owned == by_id); + REQUIRE(due != nullptr); + REQUIRE(due->delay_id == "d-0"); + REQUIRE(by_txn == by_id); + REQUIRE(by_event == table.find("d-7")); + REQUIRE(count == 100U); + REQUIRE(earliest.has_value()); + REQUIRE(finalised.size() == 1U); + REQUIRE(old.size() == 1U); + REQUIRE(examined <= 8U); + } + } + } +} diff --git a/tests/unit/test_worker_table_load_profile.cpp b/tests/unit/test_worker_table_load_profile.cpp index 8846104eb..df4fdfd89 100644 --- a/tests/unit/test_worker_table_load_profile.cpp +++ b/tests/unit/test_worker_table_load_profile.cpp @@ -202,6 +202,9 @@ namespace // token checks run in main; the worker serves no client API. "oauth_clients", "oauth_sessions", + // ADR-0824: delayed events (migration 003). Main only: the federation worker sends no events and holds no + // signing key, so it never reads a scheduled event. + "delayed_events", // Vestigial: declared in src/database/Schema.cpp's DDL but never // populated or queried by any runtime code path, main's or the // worker's (confirmed by grep finding no reference outside @@ -274,6 +277,7 @@ SCENARIO("The federation_worker load profile excludes every table outside the al REQUIRE_FALSE(merovingian::database::table_load_profile_includes("account_threepids", profile)); REQUIRE_FALSE(merovingian::database::table_load_profile_includes("oauth_clients", profile)); REQUIRE_FALSE(merovingian::database::table_load_profile_includes("oauth_sessions", profile)); + REQUIRE_FALSE(merovingian::database::table_load_profile_includes("delayed_events", profile)); } AND_THEN("an arbitrary non-allowlisted table is excluded, proving this is a fail-closed allowlist " "rather than a denylist") From 31ac2194871886543430dc24ff6ca7e60373a74a Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:12:43 +0000 Subject: [PATCH 04/29] Document migration 003 and the delayed-events table Co-Authored-By: Claude Fable 5.2 --- docs/database-persistence.md | 57 ++++++++++++++++++++++++++++-------- migrations/AGENTS.md | 11 ++++++- src/database/AGENTS.md | 20 +++++++++++++ 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/docs/database-persistence.md b/docs/database-persistence.md index f404d2cf0..3dba52374 100644 --- a/docs/database-persistence.md +++ b/docs/database-persistence.md @@ -25,8 +25,8 @@ value. Test fixtures that do not exercise persistence select it through migration statements (`INSERT`/`UPDATE`/`DELETE`) are accepted as no-op schema-state transitions, enabling backfill and corrective migrations should a later migration need one. -- The schema has two steps. `schema::current_schema_version()` is `2`, and - `migrations/` holds `001_initial_schema.sql` and `002_oauth_tables.sql`. +- The schema has three steps. `schema::current_schema_version()` is `3`, and + `migrations/` holds `001_initial_schema.sql`, `002_oauth_tables.sql` and `003_delayed_events.sql`. - Version `1` (`initial_schema`) is the frozen v1 schema. Every pre-1.0 migration (versions 2 to 26 as of 0.12.46) was collapsed into it in 0.12.47 ([ADR-0224](adr/0224-pre-1-0-migrations-collapse-into-one-frozen-v1-schema.md)), @@ -42,13 +42,13 @@ value. Test fixtures that do not exercise persistence select it through `stream_ordering` is deliberately unindexed) Nothing is seeded or backfilled: the old state-transition backfill and state-group seeding only ever mattered for upgrading existing databases and no longer exist. The compiled catalog in - `src/database/Migration.cpp` has two upgrade steps (version `1`, - `initial_schema`; version `2`, `oauth_tables`) and two downgrade steps, newest - first (`drop_oauth_tables` to version `1`, then `drop_initial_schema` to + `src/database/Migration.cpp` has three upgrade steps (version `1`, + `initial_schema`; version `2`, `oauth_tables`; version `3`, `delayed_events`) and three downgrade steps, newest + first (`drop_delayed_events` to version `2`, `drop_oauth_tables` to version `1`, then `drop_initial_schema` to version `0`, which drops every v1 table). - `migrations/001_initial_schema.sql` is the frozen record of v1 and - `002_oauth_tables.sql` of version 2; a unit test requires each file to match - the build's step statement for statement. + `migrations/001_initial_schema.sql` is the frozen record of v1, + `002_oauth_tables.sql` of version 2 and `003_delayed_events.sql` of version 3; a unit test requires each file to + match the build's step statement for statement. - Version `2` (`oauth_tables`, 1.1.0) adds the OAuth 2.0 tables ([ADR-0776](adr/0776-an-oauth-session-is-a-row-keyed-by-user-and-device-and-the-token-tables-do-not-change.md)), and changes no existing table: @@ -76,6 +76,37 @@ value. Test fixtures that do not exercise persistence select it through backends, with sessions by (user, device), a per-user index, a session count per client and the sessionless clients ordered by last use, which the retention sweep reads (30 days, never a client with a session). +- Version `3` (`delayed_events`, 1.1.2) adds one table for MSC4140 delayed events + ([ADR-0824](adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md), + [ADR-0825](adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md)), + and changes no existing table: + - `delayed_events` (`delay_id TEXT PRIMARY KEY`, then `user_id`, `device_id`, `room_id`, `event_type`, `is_state`, + `state_key`, `content_json`, `txn_id`, `via_appservice`, `delay_ms`, `delayed_since_ms`, `status`, + `finalised_at_ms`, `event_id`, `error_json`, all `TEXT NOT NULL`: the integers are decimal text, the two booleans + are `true` or `false` like `users.deactivated`, `status` is `scheduled`, `sent`, `cancelled` or `failed`, and the + columns that do not apply are the empty string, or `0` for `finalised_at_ms`), with `idx_delayed_events_user_id` + on `user_id`, which serves the delete at deactivation. No foreign key. `device_id` is the scheduling request's + device and is not deleted with it: a delayed event belongs to the user. + - Caps, enforced by `store_delayed_event` and the finalising functions (`DelayedEventStore.hpp`): `delay_id` 64 + bytes, `content_json` 65,536 bytes, `error_json` 1,024 bytes, 255 bytes for `user_id`, `device_id`, `room_id`, + `event_type`, `state_key`, `txn_id` and `event_id`, and `delay_ms` from 1 to 2,678,400,000 (31 days). A row over a + cap is refused, never truncated. The store does not enforce the per-user cap on scheduled rows: the caller does, + because it must compute `Retry-After` from the user's rows. + - In memory: `PersistentStore::delayed_events` (`DelayedEventTable`), hydrated on both backends, with no lock of + its own (`runtime.mutex` serialises it). It indexes the rows by delay ID, by user (the scheduled rows by due + time, the finalised rows by finalised time), every scheduled row not being sent by due time (`delayed_since_ms + + delay_ms`, then delay ID, so the earliest due is O(1)), by (user, device, room, event type, transaction ID) for + idempotency, by event ID for sent rows, and every finalised row by finalised time for the age prune. A memory-only + "being sent" mark hides a row from the due order while a send is in progress. No reader scans the table. A row + whose `status` text is not one of the four is skipped when hydrating, never repaired. + - Rows leave in four ways, all in one transaction: the user's cancel and the outcome of a send each evict that + user's oldest finalised rows beyond `server.delayed_events.max_finalised_per_user`; the age prune deletes + finalised rows older than 7 days (oldest first, at most 500 a transaction); and account deactivation deletes all + of a user's rows (`DELETE ... WHERE user_id = $1`). Logout and device deletion delete nothing. + - The one place memory changes without a commit: `record_delayed_event_outcome`. The event was already sent, so + memory is finalised even when the finalising commit fails (the function says so), and the database row still + reads `scheduled`, so the next start-up sends it again (at least once, ADR-0825). Every other function changes + memory only after the commit. - The schema registry is split by version ([ADR-0778](adr/0778-the-schema-is-built-from-per-version-tables-and-each-migration-step-from-what-its-version-adds.md)): `initial_schema_definitions()` is what v1 created, and the SQLite and @@ -915,8 +946,9 @@ value. Test fixtures that do not exercise persistence select it through (`[fixture]`, one per backend) open it, run `migrate_*_schema_to`, start the runtime on it and read back the users, access tokens, rooms, events, media, account data, pusher and device keys the manifest lists, then send a new event signed with the old release's key. Opening the 1.0.3 database applies migration 002 (the OAuth 2.0 tables), so - the scenarios first assert the v1 to v2 step ran: the schema version is `current_schema_version()`, the ledger gained the - `oauth_tables` row, and `oauth_clients` and `oauth_sessions` exist and take a row. They then assert everything the manifest lists is still served. + the scenarios first assert the v1 to v3 steps ran: the schema version is `current_schema_version()`, the ledger gained the + `oauth_tables` and `delayed_events` rows, and `oauth_clients`, `oauth_sessions` and `delayed_events` exist and take a row (the fixture + held none of them before). They then assert everything the manifest lists is still served. The PostgreSQL scenario needs `MEROVINGIAN_TEST_POSTGRESQL_URI` and reports itself skipped without it. - Both stores read the migration ledger in numeric order (`ORDER BY CAST(version AS INTEGER)`): `schema_migrations.version` is `TEXT`, and @@ -1217,7 +1249,7 @@ migration's table cannot go unclassified: `event_state_groups`, `forward_extremities` (the last four added in 0.12.13 by ADR-0064 phase A — `state_groups` was vestigial before this and is now moved out of the never-reads list below). -- **Worker never reads** (52 tables): `users`, `devices`, `access_tokens`, +- **Worker never reads** (53 tables): `users`, `devices`, `access_tokens`, `refresh_tokens`, `federation_destinations`, `federation_transactions`, `invites`, `state_transitions`, `sync_stream_watermark`, `event_stream_watermark`, `stream_token_epoch`, `device_keys`, `one_time_keys`, `fallback_keys`, @@ -1229,7 +1261,8 @@ migration's table cannot go unclassified: `client_txn_ids`, `pushers`, `notifications`, `openid_tokens`, `login_tokens`, `appservice_txn_cursor`, `appservice_outbox`, `deferred_pdus`, `receipts`, `appservice_directory`, `media_pending_uploads`, `oauth_clients` and `oauth_sessions` (migration - 002; every OAuth 2.0 endpoint runs in main), plus six tables `Schema.cpp` + 002; every OAuth 2.0 endpoint runs in main), `delayed_events` (migration 003; the federation worker sends no events + and holds no signing key, so it never reads a scheduled event), plus six tables `Schema.cpp` declares but that no runtime code path (main's or the worker's) ever populates or queries — `event_json`, `key_backups`, `push_rules`, `rate_limits`, `room_versions`, `state_group_edges` — vestigial from diff --git a/migrations/AGENTS.md b/migrations/AGENTS.md index 62f6931ca..50c7383b2 100644 --- a/migrations/AGENTS.md +++ b/migrations/AGENTS.md @@ -45,6 +45,15 @@ with `schema_step_creating` and its downgrade with `schema_step_dropping`, a fil - Never rename a step or reuse a version number: the ledger check identifies a step by both. +## Migration 003: delayed events (1.1.2, ADR-0824, ADR-0825) + +`003_delayed_events.sql` creates `delayed_events` (sixteen `TEXT` columns, `delay_id` the primary key) and +`idx_delayed_events_user_id` on `(user_id)`. It follows migration 002's pattern exactly: its own arrays +`delayed_event_tables` and `delayed_event_indexes` in `Schema.cpp`, a case in both `*_added_by` functions, the step +`delayed_events_migration()` built with `schema_step_creating` and its downgrade `downgrade_delayed_events_migration()` +(`drop_delayed_events`, to version 2) at the front of the downgrade catalog, and the file equal to the step. The +table is classified in the worker's never-reads set (the worker sends no events), not in the allowlist. + ## File naming ``` @@ -53,7 +62,7 @@ NNN_snake_case_description.sql `NNN` is a zero-padded three-digit integer: `001`, `002`, ..., `010`, `011`, ... The next migration number is always `max(existing) + 1`. -Current highest: `002`. +Current highest: `003`. ## File format diff --git a/src/database/AGENTS.md b/src/database/AGENTS.md index dbfc49cf4..ae194169c 100644 --- a/src/database/AGENTS.md +++ b/src/database/AGENTS.md @@ -15,6 +15,9 @@ Manages schema migrations and provides a dual-backend (SQLite / PostgreSQL) pers | `PreparedStatement.cpp` | Prepared statement wrapper; parameters are always bound, never interpolated | | `Schema.cpp` | The schema by version: the frozen v1 arrays `core_tables` and `core_indexes` (ADR-0224: a schema change is a new numbered migration, never an edit to v1), the v2 arrays `oauth_tables` and `oauth_indexes`, `schema_definitions_added_by`/`schema_indexes_added_by`, the initial (v1) and current lists built from them, the `CREATE TABLE`/`CREATE INDEX` SQL, and `current_schema_version()` (ADR-0778) | | `OAuthTables.cpp` | `OAuthTables`, the in-memory copy of `oauth_clients` and `oauth_sessions`: clients by ID, sessions by (user, device), a per-user index, a session count per client and the sessionless clients ordered by last use (ADR-0776, ADR-0777) | +| `DelayedEventTable.cpp` | `DelayedEventTable`, the in-memory copy of `delayed_events`: by delay ID, per user (scheduled by due time, finalised by finalised time), the due order that skips a being-sent row, by (user, device, room, type, transaction ID), by event ID, and the finalised rows by finalised time (ADR-0824). Also `delayed_event_due_ms` (a saturating sum) | +| `DelayedEventStore.cpp` | The delayed-event store functions and their caps (`max_delayed_event_id_bytes` 64, `max_delayed_event_content_bytes` 65,536, `max_delayed_event_error_bytes` 1,024, `max_delayed_event_field_bytes` 255, `max_delayed_event_delay_ms` 31 days): store, restart, cancel, record the outcome of a send, delete a user's rows, prune by age. Finalising evicts the user's oldest finalised rows in the same transaction (ADR-0824, ADR-0825) | +| `DelayedEventStatusText.cpp` | The `status` column's text for `DelayedEventStatus` and its strict parse (an unknown text makes hydration skip the row) | | `OAuthStore.cpp` | The OAuth 2.0 store functions and their caps (`max_oauth_client_metadata_bytes` 16 KiB, `max_oauth_scope_bytes` 2048, `max_oauth_client_id_bytes` 128): insert-or-keep clients, the once-a-day touch, delete a set of clients, sessionless clients oldest first, and the session row's prepare/remember pair | | `EventTable.cpp` | `EventTable`, the in-memory mirror of the `events` table with its indexes by event ID, stream order, room and stream order, room and time, and relation target (ADR-0215, ADR-0300, ADR-0430, ADR-0435) | | `RedactionView.cpp` | Client-facing redaction presentation: adds `redacts` to `m.room.redaction` events and `redacted_because` under `unsigned` of redacted events (CSAZ-11) | @@ -215,6 +218,23 @@ single-table revokes. The grant's `oauth.session.granted` audit row is in the same transaction (ADR-0629's pattern: `audit_insert_statement`, then `remember_audit_event` after the commit). Neither table is read by the federation worker. +## Delayed events (migration 003, ADR-0824) + +`PersistentStore::delayed_events` is a `DelayedEventTable`: write it only through `DelayedEventStore.hpp` (hydration +uses `insert`). It owns the rows, shows them read-only and keeps every index right, so never loop it: ask for a row +by `find`/`find_owned` (the one place the ownership rule lives, so another user's row and a missing row are the same +answer), the next send by `earliest_due`, a user's due time by `earliest_due_ms_of_user`, idempotency by +`find_by_transaction`, a sent event by `find_sent_by_event_id`, and old rows by `finalised_before`. Tests prove cost with +`rows_examined`. The store functions change memory only after `commit_persistent_transaction` returns true, with ONE +exception: `record_delayed_event_outcome` finalises memory even when its commit fails (the event was already sent, and a +row left scheduled would be sent again on every run; the database row still reads `scheduled`, so the next start-up sends +it once more, ADR-0825). A new function that finalises a row keeps that bounded eviction in the same transaction as the +status update. The "being sent" mark is memory only and `restart` and `cancel` refuse a marked row. **Every path that +deletes a user's account removes the user's rows** with `delete_delayed_events_of_user`; logout and device deletion do +not (a call's delayed leave must outlive its device). `store_delayed_event` does not enforce the per-user cap on +scheduled rows: the caller does, because it computes `Retry-After`. Never log a row's `content_json`. The federation +worker never reads the table. + ## Backend selection The backend is chosen at startup from config (`database.backend = sqlite | postgresql`). From ffa671fb050935cbffe9c533a00743de2d4be9ee Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:23:14 +0000 Subject: [PATCH 05/29] Add the server.delayed_events.* configuration and snapshot it into the client API limits Co-Authored-By: Claude Fable 5.2 --- include/merovingian/config/Config.hpp | 5 + .../config/DelayedEventsConfig.hpp | 24 ++ include/merovingian/config/ServerConfig.hpp | 3 + .../homeserver/ClientApiLimits.hpp | 9 + src/config/AGENTS.md | 2 +- src/config/Config.cpp | 11 + src/config/ConfigParser.cpp | 29 ++ src/config/ReloadPlan.cpp | 18 ++ src/config/ReloadPolicy.cpp | 7 + src/homeserver/ClientServerRuntime.cpp | 4 + .../test_configurable_client_limits_flow.cpp | 26 ++ tests/meson.build | 1 + tests/unit/test_config_delayed_events.cpp | 257 ++++++++++++++++++ 13 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 include/merovingian/config/DelayedEventsConfig.hpp create mode 100644 tests/unit/test_config_delayed_events.cpp diff --git a/include/merovingian/config/Config.hpp b/include/merovingian/config/Config.hpp index 640dcf230..8e64a21fb 100644 --- a/include/merovingian/config/Config.hpp +++ b/include/merovingian/config/Config.hpp @@ -32,6 +32,11 @@ namespace merovingian::config inline constexpr std::uint32_t min_audit_log_retention_days = 30U; inline constexpr std::uint32_t max_audit_log_retention_days = 3650U; +// server.delayed_events.* ranges (ADR-0826). +inline constexpr std::uint64_t min_delayed_event_delay_ms = 1'000U; +inline constexpr std::uint64_t max_delayed_event_delay_ms = 2'678'400'000U; // 31 days +inline constexpr std::uint32_t max_delayed_events_per_user = 1'000U; + class Config final { public: diff --git a/include/merovingian/config/DelayedEventsConfig.hpp b/include/merovingian/config/DelayedEventsConfig.hpp new file mode 100644 index 000000000..8f1027fca --- /dev/null +++ b/include/merovingian/config/DelayedEventsConfig.hpp @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include + +namespace merovingian::config +{ + +// Delayed events (MSC4140; ADR-0826): the operator's switch and limits. All four keys are snapshotted at start-up, so +// a change needs a restart. validate() refuses a value outside the range named on each member. +struct DelayedEventsConfig final +{ + // Off: no new event can be scheduled. Events already scheduled are still sent and can still be managed. + bool enabled{true}; + // The longest delay accepted, in milliseconds. 1000..2678400000 (31 days). Element Call needs 3600000 or more. + std::uint64_t max_delay_ms{86'400'000U}; + // Scheduled events one user may hold. 1..1000. + std::uint32_t max_scheduled_per_user{100U}; + // Finalised (sent, cancelled or failed) events kept per user. 1..1000. + std::uint32_t max_finalised_per_user{100U}; +}; + +} // namespace merovingian::config diff --git a/include/merovingian/config/ServerConfig.hpp b/include/merovingian/config/ServerConfig.hpp index 84dc1455a..872ea83d7 100644 --- a/include/merovingian/config/ServerConfig.hpp +++ b/include/merovingian/config/ServerConfig.hpp @@ -4,6 +4,7 @@ #include "merovingian/config/ClientApiConfig.hpp" #include "merovingian/config/CorsConfig.hpp" +#include "merovingian/config/DelayedEventsConfig.hpp" #include "merovingian/config/EmailConfig.hpp" #include "merovingian/config/HttpTransportConfig.hpp" #include "merovingian/config/IdentityServerConfig.hpp" @@ -40,6 +41,8 @@ struct ServerConfig final TurnServerConfig turn{}; // MatrixRTC transport discovery (MSC4143). Empty by default: no transport is advertised (see RtcConfig). RtcConfig rtc{}; + // Delayed events (MSC4140): on by default, with a maximum delay and per-user caps (see DelayedEventsConfig). + DelayedEventsConfig delayed_events{}; // The built-in OAuth 2.0 API. Off by default (see OAuthConfig). OAuthConfig oauth{}; // Identity Service API client config for 3PID invites/bind/unbind/ diff --git a/include/merovingian/homeserver/ClientApiLimits.hpp b/include/merovingian/homeserver/ClientApiLimits.hpp index 6abdbcee7..f68ed9769 100644 --- a/include/merovingian/homeserver/ClientApiLimits.hpp +++ b/include/merovingian/homeserver/ClientApiLimits.hpp @@ -83,6 +83,15 @@ struct ClientApiLimits final // at most 16 KiB stay small in memory. At the cap a new registration first evicts idle clients, least recently // used first; tests lower it. std::size_t max_oauth_clients{10000U}; + // ADR-0826 (copied from server.delayed_events.*): whether new delayed events may be scheduled; scheduled ones are + // still sent when this is false. + bool delayed_events_enabled{true}; + // The longest delay, in milliseconds, a delayed event may be scheduled with. + std::uint64_t max_delayed_event_delay_ms{86'400'000U}; + // The scheduled delayed events one user may hold. + std::uint32_t max_scheduled_delayed_events_per_user{100U}; + // The finalised delayed events kept per user. + std::uint32_t max_finalised_delayed_events_per_user{100U}; sync::SlidingSyncLimits sliding_sync{500U, 1024U, 1024U}; std::uint32_t sliding_sync_connections_per_device{16U}; }; diff --git a/src/config/AGENTS.md b/src/config/AGENTS.md index 566a37554..6b02517b7 100644 --- a/src/config/AGENTS.md +++ b/src/config/AGENTS.md @@ -11,7 +11,7 @@ Parses and validates the server configuration and plans (but does not yet apply) | `RuntimeConfigSnapshot.cpp` | `RuntimeConfigSnapshot`: holds a `Config` and can swap it for a reload-only candidate. Only the unit tests construct it today; the server does not | | `RtcConfig.hpp`; TURN and RTC rules in `Config.cpp` | The `server.turn.uris` list rules (`turn:`/`turns:` entries, 512 bytes, 16 entries, no repeats, never both with `server.turn.server`; ADR-0821) and the `server.rtc.livekit.service_url` rule (an `oauth::ParsedUri` with scheme `https` and a host; ADR-0822). `RtcConfig.hpp` is that section's struct. | | `ReloadPlan.cpp` | Diffs old vs new config and classifies each change; used by `--plan-config-reload`. No live reload path (SIGHUP/admin socket) exists yet | -| `ReloadPolicy.cpp` | `reload_policy_for_key()`: which keys need a restart. Keys are reloadable by default. Restart is required for `server.name`, the `database.*` URI and role keys, listener TLS certificate/key and `reverse_proxy` keys, `security.registration.token_file`, `database.pool_size`, `security.media.thumbnail_cache_size`, `security.media.thumbnail_max_concurrent_per_user`, `security.secrets.master_key_file`, `security.federation.key_resolution_*`, `security.federation.join_response_max_size`, `client_rate_limits.*`, `log_modules.*`, `federation.worker.*`, `server.cors.*`, `server.http.*`, `server.identity_server.*`, `server.push.*`, `server.oauth.*`, `appservice.*` and `presence.*`. The rate-limit engine and per-module log map are built once at start-up, so they are not hot-reloadable. | +| `ReloadPolicy.cpp` | `reload_policy_for_key()`: which keys need a restart. Keys are reloadable by default. Restart is required for `server.name`, the `database.*` URI and role keys, listener TLS certificate/key and `reverse_proxy` keys, `security.registration.token_file`, `database.pool_size`, `security.media.thumbnail_cache_size`, `security.media.thumbnail_max_concurrent_per_user`, `security.secrets.master_key_file`, `security.federation.key_resolution_*`, `security.federation.join_response_max_size`, `client_rate_limits.*`, `log_modules.*`, `federation.worker.*`, `server.cors.*`, `server.http.*`, `server.identity_server.*`, `server.push.*`, `server.oauth.*`, `server.delayed_events.*`, `appservice.*` and `presence.*`. The rate-limit engine and per-module log map are built once at start-up, so they are not hot-reloadable. | ## Rules diff --git a/src/config/Config.cpp b/src/config/Config.cpp index c7c8b1dc0..d6a10eb63 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -1036,6 +1036,17 @@ auto validate(Config const& config) -> std::vector {std::string{key}, "must be a positive byte size no greater than " + std::to_string(maximum)}); } }; + validate_count("server.delayed_events.max_scheduled_per_user", + config.server().delayed_events.max_scheduled_per_user, max_delayed_events_per_user); + validate_count("server.delayed_events.max_finalised_per_user", + config.server().delayed_events.max_finalised_per_user, max_delayed_events_per_user); + if (config.server().delayed_events.max_delay_ms < min_delayed_event_delay_ms || + config.server().delayed_events.max_delay_ms > max_delayed_event_delay_ms) + { + findings.push_back({"server.delayed_events.max_delay_ms", + "must be between " + std::to_string(min_delayed_event_delay_ms) + " and " + + std::to_string(max_delayed_event_delay_ms) + " milliseconds"}); + } validate_count("server.client_api.max_sync_rooms", config.server().client_api.max_sync_rooms, 10000U); validate_count("server.client_api.max_sync_events_per_room", config.server().client_api.max_sync_events_per_room, 1000U); diff --git a/src/config/ConfigParser.cpp b/src/config/ConfigParser.cpp index 0aace1949..d23e64a07 100644 --- a/src/config/ConfigParser.cpp +++ b/src/config/ConfigParser.cpp @@ -993,6 +993,35 @@ namespace { server.rtc.livekit_service_url = std::string{value}; } + // Delayed events (ADR-0826). validate() judges the ranges. + else if (key == "server.delayed_events.enabled") + { + if (!parse_bool_value(value, server.delayed_events.enabled)) + { + add_parse_finding(findings, std::string{key}, "expected boolean value"); + } + } + else if (key == "server.delayed_events.max_delay_ms") + { + if (!parse_u64_value(value, server.delayed_events.max_delay_ms)) + { + add_parse_finding(findings, std::string{key}, "expected unsigned integer millisecond value"); + } + } + else if (key == "server.delayed_events.max_scheduled_per_user") + { + if (!parse_u32_value(value, server.delayed_events.max_scheduled_per_user)) + { + add_parse_finding(findings, std::string{key}, "expected unsigned integer count"); + } + } + else if (key == "server.delayed_events.max_finalised_per_user") + { + if (!parse_u32_value(value, server.delayed_events.max_finalised_per_user)) + { + add_parse_finding(findings, std::string{key}, "expected unsigned integer count"); + } + } else if (key == "server.turn.username") { server.turn.username = std::string{value}; diff --git a/src/config/ReloadPlan.cpp b/src/config/ReloadPlan.cpp index b15152cb4..9ebcad7cd 100644 --- a/src/config/ReloadPlan.cpp +++ b/src/config/ReloadPlan.cpp @@ -587,6 +587,24 @@ auto build_reload_plan(Config const& current, Config const& next) -> ReloadPlan add_change(plan, "server.rtc.livekit.service_url"); } + // Delayed events: every field is diffed, and every change is restart_required (see ReloadPolicy.cpp). + if (current.server().delayed_events.enabled != next.server().delayed_events.enabled) + { + add_change(plan, "server.delayed_events.enabled"); + } + if (current.server().delayed_events.max_delay_ms != next.server().delayed_events.max_delay_ms) + { + add_change(plan, "server.delayed_events.max_delay_ms"); + } + if (current.server().delayed_events.max_scheduled_per_user != next.server().delayed_events.max_scheduled_per_user) + { + add_change(plan, "server.delayed_events.max_scheduled_per_user"); + } + if (current.server().delayed_events.max_finalised_per_user != next.server().delayed_events.max_finalised_per_user) + { + add_change(plan, "server.delayed_events.max_finalised_per_user"); + } + // The OAuth 2.0 API: every field is diffed, and every change is restart_required (see ReloadPolicy.cpp). if (current.server().oauth.enabled != next.server().oauth.enabled) { diff --git a/src/config/ReloadPolicy.cpp b/src/config/ReloadPolicy.cpp index 5d63c05ae..6e159cac1 100644 --- a/src/config/ReloadPolicy.cpp +++ b/src/config/ReloadPolicy.cpp @@ -118,6 +118,13 @@ auto reload_policy_for_key(std::string_view key) noexcept -> ReloadPolicy return ReloadPolicy::restart_required; } + // Delayed events: the switch and the limits are copied into the client-server runtime's limits at start-up + // (ADR-0826); SIGHUP does not push new values. + if (starts_with(key, "server.delayed_events.")) + { + return ReloadPolicy::restart_required; + } + // Presence: main and the federation worker copy `presence.enabled` at start-up (the worker // reads its own config file once), and SIGHUP does not push a new value to either. if (starts_with(key, "presence.")) diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 9345930f6..3cb3fecd6 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -15001,6 +15001,10 @@ auto start_client_server(config::Config const& config, ClientServerStartOptions client_api.sliding_sync_max_room_subscriptions, client_api.sliding_sync_max_required_state_entries}; rt.limits.sliding_sync_connections_per_device = client_api.sliding_sync_connections_per_device; + rt.limits.delayed_events_enabled = config.server().delayed_events.enabled; + rt.limits.max_delayed_event_delay_ms = config.server().delayed_events.max_delay_ms; + rt.limits.max_scheduled_delayed_events_per_user = config.server().delayed_events.max_scheduled_per_user; + rt.limits.max_finalised_delayed_events_per_user = config.server().delayed_events.max_finalised_per_user; // Snapshot the CORS policy at startup. CORS is HTTP-behaviour // configuration (per docs/user-manual.md) and so requires a restart to // take effect, matching every other HTTP-behaviour key. diff --git a/tests/integration/test_configurable_client_limits_flow.cpp b/tests/integration/test_configurable_client_limits_flow.cpp index d3e87a9eb..609b3f5e2 100644 --- a/tests/integration/test_configurable_client_limits_flow.cpp +++ b/tests/integration/test_configurable_client_limits_flow.cpp @@ -98,3 +98,29 @@ SCENARIO("client API runtime limits are copied from the server configuration", " } } } + +SCENARIO("delayed-event runtime limits are copied from server.delayed_events", "[homeserver][limits][delayed-events]") +{ + GIVEN("an operator configuration with non-default delayed-event settings") + { + auto config = make_client_api_config(); + config.server().delayed_events.enabled = false; + config.server().delayed_events.max_delay_ms = 3'600'000U; + config.server().delayed_events.max_scheduled_per_user = 7U; + config.server().delayed_events.max_finalised_per_user = 9U; + + WHEN("the client server runtime starts") + { + auto const started = merovingian::homeserver::start_client_server(config); + + THEN("the runtime limits hold the configured values") + { + REQUIRE(started.started); + REQUIRE_FALSE(started.runtime.limits.delayed_events_enabled); + REQUIRE(started.runtime.limits.max_delayed_event_delay_ms == 3'600'000U); + REQUIRE(started.runtime.limits.max_scheduled_delayed_events_per_user == 7U); + REQUIRE(started.runtime.limits.max_finalised_delayed_events_per_user == 9U); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index 4994e74b4..319f2e17a 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -337,6 +337,7 @@ shell = find_program('sh') 'unit/test_config_identity_server.cpp', 'unit/test_config_turn_uris.cpp', 'unit/test_config_rtc.cpp', + 'unit/test_config_delayed_events.cpp', 'unit/test_identity_bind_domains.cpp', 'unit/test_config_parser.cpp', 'unit/test_config_reload_plan.cpp', diff --git a/tests/unit/test_config_delayed_events.cpp b/tests/unit/test_config_delayed_events.cpp new file mode 100644 index 000000000..331603b5b --- /dev/null +++ b/tests/unit/test_config_delayed_events.cpp @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// server.delayed_events.*: the operator's switch and limits for delayed events (MSC4140; ADR-0826). The routes are not +// in Matrix v1.19, so these are project-level checks of parsing, validation and reload classification. + +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ConfigParseResult.hpp" +#include "merovingian/config/ConfigParser.hpp" +#include "merovingian/config/ConfigValidationFinding.hpp" +#include "merovingian/config/DelayedEventsConfig.hpp" +#include "merovingian/config/ReloadPlan.hpp" +#include "merovingian/config/ReloadPolicy.hpp" +#include "merovingian/config/ServerConfig.hpp" + +#include + +#include +#include +#include +#include + +namespace +{ + +[[nodiscard]] auto has_finding(std::vector const& findings, + std::string_view field) -> bool +{ + for (auto const& finding : findings) + { + if (finding.field == field) + { + return true; + } + } + return false; +} + +[[nodiscard]] auto parse_key(std::string_view key, std::string_view value) -> merovingian::config::ConfigParseResult +{ + return merovingian::config::parse_key_value_config(std::string{key} + "=" + std::string{value} + "\n"); +} + +[[nodiscard]] auto reported_in_plan(merovingian::config::ReloadPlan const& plan, std::string_view key) -> bool +{ + for (auto const& change : plan.changes()) + { + if (change.key == key) + { + return true; + } + } + return false; +} + +constexpr auto enabled_key = std::string_view{"server.delayed_events.enabled"}; +constexpr auto delay_key = std::string_view{"server.delayed_events.max_delay_ms"}; +constexpr auto scheduled_key = std::string_view{"server.delayed_events.max_scheduled_per_user"}; +constexpr auto finalised_key = std::string_view{"server.delayed_events.max_finalised_per_user"}; + +constexpr auto all_non_default = std::string_view{"server.delayed_events.enabled=false\n" + "server.delayed_events.max_delay_ms=3600000\n" + "server.delayed_events.max_scheduled_per_user=7\n" + "server.delayed_events.max_finalised_per_user=9\n"}; + +} // namespace + +SCENARIO("server.delayed_events.* default to on with a one-day delay and 100 events per user", + "[config][delayed-events]") +{ + GIVEN("a configuration that does not mention the section") + { + auto const parsed = merovingian::config::parse_key_value_config(""); + + THEN("delayed events are enabled with the documented limits and nothing is reported") + { + auto const& delayed = parsed.config.server().delayed_events; + CHECK(delayed.enabled); + CHECK(delayed.max_delay_ms == std::uint64_t{86'400'000U}); + CHECK(delayed.max_scheduled_per_user == 100U); + CHECK(delayed.max_finalised_per_user == 100U); + CHECK(parsed.findings.empty()); + CHECK(merovingian::config::is_valid(parsed.config)); + } + } +} + +SCENARIO("each server.delayed_events.* key is parsed", "[config][delayed-events]") +{ + GIVEN("a configuration that sets every key to a non-default value") + { + auto const parsed = merovingian::config::parse_key_value_config(all_non_default); + + THEN("the values are held and the configuration is valid") + { + auto const& delayed = parsed.config.server().delayed_events; + CHECK_FALSE(delayed.enabled); + CHECK(delayed.max_delay_ms == std::uint64_t{3'600'000U}); + CHECK(delayed.max_scheduled_per_user == 7U); + CHECK(delayed.max_finalised_per_user == 9U); + CHECK(parsed.findings.empty()); + CHECK(merovingian::config::is_valid(parsed.config)); + } + } + + GIVEN("enabled written as true") + { + auto const parsed = parse_key(enabled_key, "true"); + + THEN("it is on") + { + CHECK(parsed.config.server().delayed_events.enabled); + CHECK(parsed.findings.empty()); + } + } +} + +SCENARIO("server.delayed_events.enabled refuses anything that is not a boolean", "[config][delayed-events]") +{ + GIVEN("values that are not true or false") + { + THEN("each is refused with a finding naming the key") + { + for (auto const* bad : {"", "yes", "1", "0", "TRUE", "True", "on", "2"}) + { + INFO(bad); + auto const parsed = parse_key(enabled_key, bad); + REQUIRE(has_finding(parsed.findings, enabled_key)); + } + } + } +} + +SCENARIO("server.delayed_events.max_delay_ms accepts 1000 to 2678400000 milliseconds", "[config][delayed-events]") +{ + GIVEN("values at and inside the boundaries") + { + THEN("each is accepted and held exactly") + { + for (auto const value : + {std::uint64_t{1'000U}, std::uint64_t{1'001U}, std::uint64_t{3'600'000U}, std::uint64_t{86'400'000U}, + std::uint64_t{2'678'399'999U}, std::uint64_t{2'678'400'000U}}) + { + INFO(value); + auto const parsed = parse_key(delay_key, std::to_string(value)); + REQUIRE_FALSE(has_finding(parsed.findings, delay_key)); + REQUIRE(parsed.config.server().delayed_events.max_delay_ms == value); + REQUIRE(merovingian::config::is_valid(parsed.config)); + } + } + } + + GIVEN("values outside the range, or not an unsigned integer") + { + THEN("each is refused with a finding naming the key") + { + for (auto const* bad : {"0", "1", "999", "2678400001", "4294967296", "99999999999999999999", "-1", + "-86400000", "", "abc", "1.5", "1e6", "+1000", "0x3e8", "1h"}) + { + INFO(bad); + auto const parsed = parse_key(delay_key, bad); + REQUIRE(has_finding(parsed.findings, delay_key)); + } + } + } +} + +SCENARIO("the per-user delayed event counts accept 1 to 1000", "[config][delayed-events]") +{ + GIVEN("values at and inside the boundaries") + { + THEN("each is accepted and held exactly for both counts") + { + for (auto const value : {1U, 2U, 100U, 999U, 1000U}) + { + INFO(value); + auto const scheduled = parse_key(scheduled_key, std::to_string(value)); + REQUIRE_FALSE(has_finding(scheduled.findings, scheduled_key)); + REQUIRE(scheduled.config.server().delayed_events.max_scheduled_per_user == value); + REQUIRE(merovingian::config::is_valid(scheduled.config)); + + auto const finalised = parse_key(finalised_key, std::to_string(value)); + REQUIRE_FALSE(has_finding(finalised.findings, finalised_key)); + REQUIRE(finalised.config.server().delayed_events.max_finalised_per_user == value); + REQUIRE(merovingian::config::is_valid(finalised.config)); + } + } + } + + GIVEN("values outside the range, or not an unsigned integer") + { + THEN("each is refused with a finding naming its own key") + { + for (auto const* bad : {"0", "1001", "1000000", "4294967296", "-1", "-100", "", "abc", "1.5"}) + { + INFO(bad); + auto const scheduled = parse_key(scheduled_key, bad); + REQUIRE(has_finding(scheduled.findings, scheduled_key)); + REQUIRE_FALSE(has_finding(scheduled.findings, finalised_key)); + + auto const finalised = parse_key(finalised_key, bad); + REQUIRE(has_finding(finalised.findings, finalised_key)); + REQUIRE_FALSE(has_finding(finalised.findings, scheduled_key)); + } + } + } +} + +SCENARIO("server.delayed_events.* changes are diffed by the reload plan and need a restart", "[config][delayed-events]") +{ + GIVEN("two configurations that differ in every delayed-events key") + { + auto const before = merovingian::config::parse_key_value_config(""); + auto const after = merovingian::config::parse_key_value_config(all_non_default); + + WHEN("a reload is planned") + { + auto const plan = merovingian::config::build_reload_plan(before.config, after.config); + + THEN("each key is named and the plan needs a restart") + { + CHECK(reported_in_plan(plan, enabled_key)); + CHECK(reported_in_plan(plan, delay_key)); + CHECK(reported_in_plan(plan, scheduled_key)); + CHECK(reported_in_plan(plan, finalised_key)); + CHECK(plan.has_restart_required_changes()); + } + + THEN("the policy classifies each key restart-required") + { + for (auto const key : {enabled_key, delay_key, scheduled_key, finalised_key}) + { + INFO(key); + CHECK(merovingian::config::reload_policy_for_key(key) == + merovingian::config::ReloadPolicy::restart_required); + } + } + } + } + + GIVEN("two configurations that differ in one key only") + { + auto const before = merovingian::config::parse_key_value_config(""); + + THEN("the plan names exactly that key") + { + for (auto const key : {enabled_key, delay_key, scheduled_key, finalised_key}) + { + INFO(key); + auto const after = parse_key(key, key == enabled_key ? "false" : "5000"); + auto const plan = merovingian::config::build_reload_plan(before.config, after.config); + REQUIRE(plan.changes().size() == 1U); + REQUIRE(reported_in_plan(plan, key)); + } + } + } +} From bd65c0be003f5e1f9a3e3aabce429bc00a280a4f Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:23:14 +0000 Subject: [PATCH 06/29] Document the server.delayed_events.* keys in the user manual and example configs Co-Authored-By: Claude Fable 5.2 --- config/merovingian.conf.bsd.example | 11 +++++++++++ config/merovingian.conf.example | 11 +++++++++++ docs/user-manual.md | 13 +++++++++++++ 3 files changed, 35 insertions(+) diff --git a/config/merovingian.conf.bsd.example b/config/merovingian.conf.bsd.example index c526135db..c98f366e2 100644 --- a/config/merovingian.conf.bsd.example +++ b/config/merovingian.conf.bsd.example @@ -155,6 +155,17 @@ server.http.ipv6_client_prefix_length=64 # or holding ? is refused. https://host, https://host:8443 and https://host/jwt are fine. # server.rtc.livekit.service_url=https://livekit-jwt.example.org/lk-jwt-service +# Delayed events (MSC4140, optional; restart required) +# On by default: MatrixRTC clients (Element Call) schedule a "leave" that the server sends if the client vanishes. +# false stops new scheduling; events already scheduled are still sent. +# server.delayed_events.enabled=true +# The longest delay accepted, in milliseconds: 1000..2678400000 (31 days). Element Call needs 3600000 or more. +# server.delayed_events.max_delay_ms=86400000 +# Scheduled events one user may hold: 1..1000. +# server.delayed_events.max_scheduled_per_user=100 +# Finalised (sent, cancelled or failed) events kept per user: 1..1000. +# server.delayed_events.max_finalised_per_user=100 + # Listener exposure # Keep TLS disabled only on loopback binds behind a declared reverse proxy. # If binding to a public interface, set tls=true, reverse_proxy=false, and diff --git a/config/merovingian.conf.example b/config/merovingian.conf.example index 596edd463..f8016723d 100644 --- a/config/merovingian.conf.example +++ b/config/merovingian.conf.example @@ -128,6 +128,17 @@ server.http.ipv6_client_prefix_length=64 # or holding ? is refused. https://host, https://host:8443 and https://host/jwt are fine. # server.rtc.livekit.service_url=https://livekit-jwt.example.org/lk-jwt-service +# Delayed events (MSC4140, optional; restart required) +# On by default: MatrixRTC clients (Element Call) schedule a "leave" that the server sends if the client vanishes. +# false stops new scheduling; events already scheduled are still sent. +# server.delayed_events.enabled=true +# The longest delay accepted, in milliseconds: 1000..2678400000 (31 days). Element Call needs 3600000 or more. +# server.delayed_events.max_delay_ms=86400000 +# Scheduled events one user may hold: 1..1000. +# server.delayed_events.max_scheduled_per_user=100 +# Finalised (sent, cancelled or failed) events kept per user: 1..1000. +# server.delayed_events.max_finalised_per_user=100 + # Listener exposure # Keep TLS disabled only on loopback binds behind a declared reverse proxy. # If binding to a public interface, set tls=true, reverse_proxy=false, and diff --git a/docs/user-manual.md b/docs/user-manual.md index 05fca380c..fd94524f7 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -844,6 +844,18 @@ entry, which clients built on the Rust SDK fall back to. `/_matrix/client/versio and `org.matrix.msc4143.stable` (ADR-0822). The URL is shown to anyone who can read the well-known document, so give it only an address you are happy to publish. +##### Delayed events — `server.delayed_events.*` + +All four keys require a restart (they are copied into the runtime at start-up). The keys below are checked at +start-up; anything outside the range, or not a boolean or unsigned integer, refuses the configuration (ADR-0826). + +| Key | Default | Range | Meaning | +|---|---|---|---| +| `server.delayed_events.enabled` | `true` | `true` or `false` | `false` stops new events being scheduled; events already scheduled are still sent and can still be managed. | +| `server.delayed_events.max_delay_ms` | `86400000` (24 hours) | `1000` to `2678400000` (31 days) | The longest delay accepted, in milliseconds. Element Call needs `3600000` (one hour) or more. | +| `server.delayed_events.max_scheduled_per_user` | `100` | `1` to `1000` | Scheduled events one user may hold. | +| `server.delayed_events.max_finalised_per_user` | `100` | `1` to `1000` | Finalised events kept per user (and for at most 7 days). | + Not implemented yet: - **Delayed events (MSC4140).** A participant whose client vanishes (a crash or lost network) is not removed from the @@ -1973,6 +1985,7 @@ take effect. | `database.pool_size` | Restart required | | `server.turn.*` | Reloadable | | `server.rtc.*` | Reloadable | +| `server.delayed_events.*` | Restart required | | `security.trust_safety.*` | Reloadable | | `security.access_token_lifetime_ms` / `security.refresh_token_lifetime_ms` / `security.audit_log_retention_days` | Reloadable | | `security.login_throttle.*` | Reloadable | From 5f5c83544ca5554920c337c423e899f1dd67e314 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:46:23 +0000 Subject: [PATCH 07/29] Refuse a NUL byte in any text field of a delayed-event row Co-Authored-By: Claude Fable 5.2 --- src/database/DelayedEventStore.cpp | 14 +++++++++-- tests/unit/test_delayed_event_store.cpp | 32 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/database/DelayedEventStore.cpp b/src/database/DelayedEventStore.cpp index 672f04e13..971a69c65 100644 --- a/src/database/DelayedEventStore.cpp +++ b/src/database/DelayedEventStore.cpp @@ -45,10 +45,20 @@ namespace return value.size() <= cap; } + // PostgreSQL TEXT cannot hold a NUL byte and the table's transaction key joins its parts with one, so a field + // holding one is refused, never stored. + [[nodiscard]] auto has_nul(std::string_view value) noexcept -> bool + { + return value.find('\0') != std::string_view::npos; + } + [[nodiscard]] auto row_is_storable(PersistentDelayedEvent const& row) noexcept -> bool { - return !row.delay_id.empty() && !row.user_id.empty() && !row.room_id.empty() && !row.event_type.empty() && - !row.content_json.empty() && within(row.delay_id, max_delayed_event_id_bytes) && + return !has_nul(row.delay_id) && !has_nul(row.user_id) && !has_nul(row.device_id) && !has_nul(row.room_id) && + !has_nul(row.event_type) && !has_nul(row.state_key) && !has_nul(row.txn_id) && + !has_nul(row.content_json) && !row.delay_id.empty() && !row.user_id.empty() && !row.room_id.empty() && + !row.event_type.empty() && !row.content_json.empty() && + within(row.delay_id, max_delayed_event_id_bytes) && within(row.content_json, max_delayed_event_content_bytes) && within(row.user_id, max_delayed_event_field_bytes) && within(row.device_id, max_delayed_event_field_bytes) && diff --git a/tests/unit/test_delayed_event_store.cpp b/tests/unit/test_delayed_event_store.cpp index 297c32595..d5dca9efd 100644 --- a/tests/unit/test_delayed_event_store.cpp +++ b/tests/unit/test_delayed_event_store.cpp @@ -195,6 +195,38 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t [](PersistentDelayedEvent& r) { r.device_id = std::string(db::max_delayed_event_field_bytes, 'v'); }, true }, + {"delay id with a NUL byte", + [](PersistentDelayedEvent& r) { + r.delay_id += std::string{'\0'} + "x"; + }, false}, + {"user id with a NUL byte", + [](PersistentDelayedEvent& r) { + r.user_id += std::string{'\0'} + "x"; + }, false}, + {"device id with a NUL byte", + [](PersistentDelayedEvent& r) { + r.device_id += std::string{'\0'} + "x"; + }, false}, + {"room id with a NUL byte", + [](PersistentDelayedEvent& r) { + r.room_id += std::string{'\0'} + "x"; + }, false}, + {"event type with a NUL byte", + [](PersistentDelayedEvent& r) { + r.event_type += std::string{'\0'} + "x"; + }, false}, + {"state key with a NUL byte", + [](PersistentDelayedEvent& r) { + r.state_key += std::string{'\0'} + "x"; + }, false}, + {"txn id with a NUL byte", + [](PersistentDelayedEvent& r) { + r.txn_id += std::string{'\0'} + "x"; + }, false}, + {"content json with a NUL byte", + [](PersistentDelayedEvent& r) { + r.content_json += std::string{'\0'} + "x"; + }, false}, {"device id over cap", [](PersistentDelayedEvent& r) { r.device_id = std::string(db::max_delayed_event_field_bytes + 1U, 'v'); From 2b34b20cef33068cec80e967703981dcce2d7e3d Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:46:24 +0000 Subject: [PATCH 08/29] Add the delayed-events service functions and the shared send-path helpers they call Co-Authored-By: Claude Fable 5.2 --- .../ClientServerRuntimeInternal.hpp | 44 ++ .../homeserver/DelayedEventAction.hpp | 16 + .../DelayedEventScheduleRequest.hpp | 32 + .../merovingian/homeserver/DelayedEvents.hpp | 74 +++ .../homeserver/HomeserverRuntime.hpp | 9 + .../merovingian/homeserver/RoomService.hpp | 21 + src/homeserver/AGENTS.md | 36 + src/homeserver/ClientServerRuntime.cpp | 26 +- src/homeserver/DelayedEvents.cpp | 623 ++++++++++++++++++ src/homeserver/RoomService.cpp | 63 +- src/homeserver/meson.build | 1 + 11 files changed, 934 insertions(+), 11 deletions(-) create mode 100644 include/merovingian/homeserver/ClientServerRuntimeInternal.hpp create mode 100644 include/merovingian/homeserver/DelayedEventAction.hpp create mode 100644 include/merovingian/homeserver/DelayedEventScheduleRequest.hpp create mode 100644 include/merovingian/homeserver/DelayedEvents.hpp create mode 100644 src/homeserver/DelayedEvents.cpp diff --git a/include/merovingian/homeserver/ClientServerRuntimeInternal.hpp b/include/merovingian/homeserver/ClientServerRuntimeInternal.hpp new file mode 100644 index 000000000..c095df58b --- /dev/null +++ b/include/merovingian/homeserver/ClientServerRuntimeInternal.hpp @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/homeserver/OperationResult.hpp" + +#include +#include +#include +#include + +namespace merovingian::homeserver +{ + +// Module-internal: what `ClientServerRuntime.cpp` keeps in its anonymous namespace and `DelayedEvents.cpp` (delayed +// events, MSC4140, ADR-0825) needs as well, so that a delayed event is judged and sent as the ordinary `/send` and +// `/state` routes would judge and send it, never by a copy. Each function is a call-through to the one the routes +// use, defined in `ClientServerRuntime.cpp`. Not public API. + +// The answers the `/send` route gives for an event type or a transaction ID over 255 bytes +// (`client_txn_part_is_too_long`): 413 M_TOO_LARGE for the type, 400 M_INVALID_PARAM for the transaction ID. The route +// and the delayed events' schedule-time check both read their reasons here. +inline constexpr auto event_type_too_long_reason = std::string_view{"the event type is at most 255 bytes"}; +inline constexpr auto txn_id_too_long_reason = std::string_view{"the transaction ID is at most 255 bytes"}; + +// True when `part` is over the 255 bytes an event type or a transaction ID may have where it is part of an idempotency +// key. +[[nodiscard]] auto event_txn_part_is_too_long(std::string_view part) noexcept -> bool; + +// The rate-limit route template of `target` (its path with the identifiers replaced, `.../send/{eventType}/{txnId}`), +// the bucket name every request for the route shares; `/_matrix/client/{unmatched}` for a path the server does not +// serve. The query string is ignored. +[[nodiscard]] auto rate_limit_route_template(std::string_view target) -> std::string; + +// The event body (`{"type", "content", "state_key"?}`) the `/send` and `/state` routes build from a request body, or +// nullopt when `content` is not a JSON object. +[[nodiscard]] auto route_event_body(std::string_view event_type, std::string_view content, + std::optional state_key = std::nullopt) -> std::optional; + +// The Matrix errcode the dispatcher gives a refusal of `status` that names none (403 M_FORBIDDEN, 404 M_NOT_FOUND, 401 +// M_UNKNOWN_TOKEN, else M_UNKNOWN). +[[nodiscard]] auto errcode_for_refusal_status(std::uint16_t status) -> std::string_view; + +} // namespace merovingian::homeserver diff --git a/include/merovingian/homeserver/DelayedEventAction.hpp b/include/merovingian/homeserver/DelayedEventAction.hpp new file mode 100644 index 000000000..64d4cc892 --- /dev/null +++ b/include/merovingian/homeserver/DelayedEventAction.hpp @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +namespace merovingian::homeserver +{ + +// What an owner asks of a delayed event they scheduled (MSC4140): send it now, cancel it, or restart its timer. +enum class DelayedEventAction +{ + send, + cancel, + restart, +}; + +} // namespace merovingian::homeserver diff --git a/include/merovingian/homeserver/DelayedEventScheduleRequest.hpp b/include/merovingian/homeserver/DelayedEventScheduleRequest.hpp new file mode 100644 index 000000000..45f7aaf20 --- /dev/null +++ b/include/merovingian/homeserver/DelayedEventScheduleRequest.hpp @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include +#include +#include + +namespace merovingian::homeserver +{ + +// A request to schedule a delayed event (MSC4140), after the route has parsed it. `content_json` is the serialised +// JSON object the event will carry. `state_key` is set for a state event (an empty value is the empty state key). +// `txn_id` is empty for the legacy state form, which has none. `device_id` is the scheduling request's device, empty +// when its token has none. `via_appservice` is set when an application service asserted the sender (the send passes +// it on). `stable_errcode` selects the errcode of a too-large delay: `M_DELAY_TOO_LARGE` when set, +// `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE` otherwise. +struct DelayedEventScheduleRequest final +{ + std::string user_id{}; + std::string device_id{}; + std::string room_id{}; + std::string event_type{}; + std::optional state_key{}; + std::string content_json{}; + std::string txn_id{}; + std::uint64_t delay_ms{0U}; + bool via_appservice{false}; + bool stable_errcode{false}; +}; + +} // namespace merovingian::homeserver diff --git a/include/merovingian/homeserver/DelayedEvents.hpp b/include/merovingian/homeserver/DelayedEvents.hpp new file mode 100644 index 000000000..4e1cf2dc1 --- /dev/null +++ b/include/merovingian/homeserver/DelayedEvents.hpp @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/homeserver/DelayedEventAction.hpp" +#include "merovingian/homeserver/DelayedEventScheduleRequest.hpp" +#include "merovingian/homeserver/HomeserverRuntime.hpp" +#include "merovingian/homeserver/OperationResult.hpp" +#include "merovingian/homeserver/StopFlag.hpp" + +#include +#include +#include + +namespace merovingian::homeserver +{ + +struct ClientServerRuntime; + +// Delayed events (MSC4140, ADR-0823 to ADR-0826): scheduling, managing, looking up and sending an event a user asked +// to have sent later. Every function takes `ClientServerRuntime&` because the send-time rate limit lives there. Each +// REQUIRES the caller to hold `runtime.homeserver.mutex` (a request handler does), except `send_due_delayed_events`, +// which takes it itself. None makes a network call. Nothing here logs `content_json` or a state key. + +// Wall-clock milliseconds since the Unix epoch, or `test_delayed_event_clock_ms` when a test set it. Every "now" of +// the feature comes from here or from the job's `now_ms` parameter. Needs no lock. +[[nodiscard]] auto delayed_event_now_ms(HomeserverRuntime const& runtime) -> std::uint64_t; + +// Calls `HomeserverRuntime::delayed_events_wake` when one is installed, so the "delayed-events" job re-reads its next +// deadline. Caller holds runtime.mutex; the hook never blocks. +auto wake_delayed_events(HomeserverRuntime& runtime) -> void; + +// Schedules a delayed event, in this order: the feature must be enabled (403 M_FORBIDDEN); the shape (413 M_TOO_LARGE +// for an event type over 255 bytes, 400 M_INVALID_PARAM for a transaction ID over 255 bytes, the ordinary state path's +// answer for a state key holding NUL or over 255 bytes, 400 M_INVALID_PARAM for a NUL in the type or transaction ID or +// a zero delay, 400 M_BAD_JSON for content that is not a JSON object, 413 M_TOO_LARGE for content over 65,536 bytes); +// the delay at most `max_delayed_event_delay_ms` (400, `M_DELAY_TOO_LARGE` or `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE`); +// an existing event of the same (user, device, room, type, transaction ID), which is answered `ok` with its ID and +// stores nothing; the room known and the sender joined (the answers `send_event` gives), and for a state event the +// sender's power (403, "event authorization or signing failed"); then the per-user cap (429 M_LIMIT_EXCEEDED with +// `retry_after_ms`, the time to the user's earliest due event rounded up to a whole second, at least 1000). On success +// stores the row, wakes the job, and answers `ok` with `value` the new delay ID (at most 64 bytes, URL-safe, at least +// 128 random bits). A store failure is 500. +[[nodiscard]] auto schedule_delayed_event(ClientServerRuntime& runtime, DelayedEventScheduleRequest const& request) + -> OperationResult; + +// Sends, cancels or restarts the delayed event `delay_id` of `user_id`. An ID that is unknown, another user's, empty or +// over 64 bytes is 404 M_NOT_FOUND "Delayed event not found.", identically. Outcomes (all `ok`, empty `value`, on 200): +// scheduled: restart (the timer starts again now), cancel, or send (sends now; on success the row is sent). Sent: send +// is 200, restart and cancel 409. Cancelled, or failed: cancel is 200, restart and send 409. Being sent right now: 409 +// for all three. A 409 is errcode M_UNKNOWN with reason "The delayed event has already been sent." / "...already been +// cancelled." / "The delayed event is being sent.". A send that fails answers with the send's own OperationResult +// (status, errcode, reason, retry_after_ms), unchanged, and leaves the row scheduled with its timer unchanged: an +// account refusal is 401 M_USER_LOCKED, 403 M_USER_SUSPENDED or 403 M_USER_DEACTIVATED, a rate-limit denial 429 +// M_LIMIT_EXCEEDED with `retry_after_ms`, and the rest are what `send_event` would answer. A store failure is 500. +[[nodiscard]] auto manage_delayed_event(ClientServerRuntime& runtime, std::string_view user_id, + std::string_view delay_id, DelayedEventAction action) -> OperationResult; + +// The lookup of one delayed event: the same 404 as `manage_delayed_event` for an unknown, foreign, empty or over-long +// ID; on a hit `ok` with `value` the JSON body (`delay_id`, `room_id`, `type`, `state_key` only for a state event, +// `delay_ms`, `delayed_since_ts`, `content`, and `finalised` {`finalised_ts`, `event_id` when sent, `error` when +// failed} only once finalised). +[[nodiscard]] auto delayed_event_lookup(ClientServerRuntime& runtime, std::string_view user_id, + std::string_view delay_id) -> OperationResult; + +// The "delayed-events" job's body (main.cpp; tests call it directly). Takes runtime.homeserver.mutex itself, one hold +// per delayed event: sends each scheduled event due at `now_ms` or earlier, earliest first, at most 100 a run and none +// once `stop` is requested, never before its due time; then deletes at most 500 finalised rows older than 7 days +// before `now_ms`. A failed send is recorded and never retried. Returns how long to wait before the next run: 0 ms +// when due events remain, else the time to the next deadline, at most 60 s (60 s when nothing is scheduled). +[[nodiscard]] auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, StopFlag const& stop) + -> std::chrono::milliseconds; + +} // namespace merovingian::homeserver diff --git a/include/merovingian/homeserver/HomeserverRuntime.hpp b/include/merovingian/homeserver/HomeserverRuntime.hpp index e83a5367a..1e20bb612 100644 --- a/include/merovingian/homeserver/HomeserverRuntime.hpp +++ b/include/merovingian/homeserver/HomeserverRuntime.hpp @@ -236,6 +236,9 @@ struct HomeserverRuntime final // clock that stands still, as two requests inside one millisecond see it, // or one stepped back. Always empty in production. std::optional test_frozen_event_clock_ms{}; + // Test-only: when set, `delayed_event_now_ms` returns this (Unix ms) instead of the system clock, so a test moves + // the delayed events' clock (schedule, restart, due time) without sleeping. Always empty in production. + std::optional test_delayed_event_clock_ms{}; // Test-only: called by POST /rooms/{roomId}/upgrade once the replacement room exists and before the // tombstone is sent, with runtime.mutex released, so a test can change the old room as a concurrent // request would (C21). Always empty in production. @@ -283,6 +286,12 @@ struct HomeserverRuntime final // on another thread (PeriodicJob::wake does neither). Cleared under runtime.mutex at shutdown // before the thread is stopped. std::function typing_expiry_wake{}; + // Wakes the "delayed-events" job (ADR-0825) so a deadline earlier than the job's current sleep is not missed. Set + // by the process entry point once that thread exists; null in tests, which call send_due_delayed_events + // directly. Called through wake_delayed_events with runtime.mutex held, so it must not block or wait for + // runtime.mutex on another thread (PeriodicJob::wake does neither). Cleared under runtime.mutex at shutdown + // before the thread is stopped. + std::function delayed_events_wake{}; // Users whose profile change has rooms left to reach, with the index in database.rooms to // resume from (F-11, ADR-0353). No token and no event body: the job sends as the user and reads // the profile when it sends. In memory only: a restart leaves those rooms for the next profile diff --git a/include/merovingian/homeserver/RoomService.hpp b/include/merovingian/homeserver/RoomService.hpp index e1051a10d..a730d1c51 100644 --- a/include/merovingian/homeserver/RoomService.hpp +++ b/include/merovingian/homeserver/RoomService.hpp @@ -359,6 +359,27 @@ auto claim_membership_event_template(canonicaljson::Object& event, std::string_v [[nodiscard]] auto send_event(HomeserverRuntime& runtime, std::string_view access_token, std::string_view room_id, std::string_view event_json, std::optional origin_server_ts = std::nullopt) -> OperationResult; +// The five functions below are module-internal: `DelayedEvents.cpp` (delayed events, MSC4140, ADR-0825) is their only +// caller. The caller holds runtime.mutex. +// +// `send_event`'s core with the sender already known: `event_json` is a `{"type", "content", "state_key"?}` body, and +// the event is stamped with the event clock's reading (never a caller's `origin_server_ts`). `via_appservice` is set +// when an application service asserted the sender (ADR-0628). +[[nodiscard]] auto send_event_as_resolved_user(HomeserverRuntime& runtime, std::string const& user_id, + std::string_view room_id, std::string_view event_json, + bool via_appservice) -> OperationResult; +// What `send_event_as_resolved_user` answers, with the same logging, when the room is not held (404 shape, reason +// "unknown room") or `user_id` is not joined to it (403, "not joined"); nullopt when neither. The same function +// `send_event_as_resolved_user` itself asks. +[[nodiscard]] auto room_send_refusal(HomeserverRuntime& runtime, std::string const& user_id, std::string_view room_id) + -> std::optional; +// 403 "event authorization or signing failed": what an event the room's auth rules refuse is answered, shared with +// the delayed events' schedule-time power check. +[[nodiscard]] auto event_authorization_refusal() -> OperationResult; +// 413 M_TOO_LARGE for an event over the Matrix size limits (CSAZ-9), and 400 M_INVALID_PARAM for a state key holding a +// NUL byte: the answers `send_event_as_resolved_user` gives, for a request checked before the send. +[[nodiscard]] auto event_too_large_refusal() -> OperationResult; +[[nodiscard]] auto invalid_state_key_refusal() -> OperationResult; // Spec (Client-Server API, "Events on Change of Profile Information"): when a user changes their // display name or avatar URL, "an m.room.member event (with a join membership) is sent to every // room the user is a member of, to update the displayname and avatar_url". Sends that event, diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index ca39f3565..7d9d2c42b 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -27,6 +27,8 @@ This is the largest and most complex module — read this file carefully before | `AppserviceDelivery.cpp` | Application Service API outbound queue (ADR-0163): `enqueue_appservice_delivery` (request path, no network) and `drain_appservice_queues` (the one drainer thread, run by a `PeriodicJob` from `main.cpp`) | | `ClientTxnPrune.cpp` | `prune_client_transactions`: the sweep the `client-txn-prune` `PeriodicJob` runs every hour, deleting to-device `client_txn_ids` records older than 7 days (F-54, ADR-0381) | | `RetentionSweep.cpp` | `run_retention_sweep`: the sweep the `retention-sweep` `PeriodicJob` runs every hour. Deletes sessions and tokens that can no longer authenticate and `audit_log` rows past `security.audit_log_retention_days`, OAuth 2.0 session rows whose device holds no live access or refresh token (judged again in the hold that deletes them, ADR-0808), and OAuth 2.0 clients with no session unused for 30 days (`oauth_client_retention`, least recently used first; never a client with a session, nor one `oauth_client_has_pending_authorisation` names: a live pending authorisation or an unredeemed code, answered in O(1) from the stores; ADR-0777, ADR-0782), in transactions of at most 500 rows (`retention_batch_rows`), at most 20 batches per kind per sweep. `runtime.mutex` is taken for one batch at a time, never across the whole sweep (ADR-0488, ADR-0489) | +| `DelayedEvents.cpp` | Delayed events (MSC4140, ADR-0823 to ADR-0826): `schedule_delayed_event`, `manage_delayed_event` (send, cancel, restart), `delayed_event_lookup`, the clock `delayed_event_now_ms` (and its seam `test_delayed_event_clock_ms`), `wake_delayed_events`, and `send_due_delayed_events`, the body of the `delayed-events` `PeriodicJob` in `main.cpp`. The caller holds `runtime.mutex` for all but the job body, which takes it for one send at a time | +| `ClientServerRuntimeInternal.hpp` | Call-throughs from `ClientServerRuntime.cpp`'s anonymous namespace that `DelayedEvents.cpp` shares with the `/send` and `/state` routes: `rate_limit_route_template` (`normalized_target`), `route_event_body` (`event_body_from_content`), `event_txn_part_is_too_long`, `errcode_for_refusal_status`, and the two reasons the `/send` route gives for a too-long type or transaction ID. Not public API | | `OAuthClientRegistration.cpp` | `POST /_merovingian/oauth2/register` (`register_oauth_client`), the client ID (`oauth_client_id_of`, ADR-0779), the cap and its eviction (ADR-0780), and `find_registered_oauth_client` | | `OAuthJsonResponse.cpp` | `oauth_json_response` / `oauth_error_response` / `oauth_empty_response`: every OAuth JSON (or empty) answer, with `no-store` (ADR-0781) | | `OAuthTokenEndpoint.cpp` | `POST /_merovingian/oauth2/token` (`handle_oauth_token`): the authorization code grant, the refresh grant (ADR-0791 to ADR-0793) and the device code grant (ADR-0804) | @@ -1200,6 +1202,40 @@ event type, before anything is created. Membership alone is not enough. Alias `P room's forward extremities (`last_known_event_id` in `ClientServerRuntime.cpp` is the model). A redacted event is served from the table's redacted JSON; do not add a second copy. +## Delayed events (ADR-0823 to ADR-0826) + +`DelayedEvents.cpp` schedules, manages, looks up and sends an event a user asked to have sent later; the routes (the +next layer) only parse a request and call it. Invariants a change must keep: + +- **One sender at a time, marked.** A send takes `runtime.mutex` for one event, marks the row as being sent + (`DelayedEventTable::mark_being_sent`, memory only) before `send_event_as_resolved_user`, and finalising clears the + mark. A marked row is offered by nobody: restart, cancel and send answer 409 "being sent". The job sends ONE event per + hold and at most 100 a run; it checks its `StopFlag` between events. A record the store refuses leaves the row + marked, so it is never sent again by that process; never clear the mark to retry. +- **Never early.** The job sends only a row whose `delayed_event_due_ms` is at or before the `now_ms` it was given, + re-read under the lock it sends under. "Now" is `delayed_event_now_ms` (production and handlers) or the job's + parameter (tests); never read the clock anywhere else, and never sleep in a test. +- **Authorised when sent.** The send is the ordinary `send_event_as_user`, so the room's auth rules judge the event + against the state NOW, after the account check (unknown or deactivated 403 `M_USER_DEACTIVATED`, locked 401 + `M_USER_LOCKED`, suspended 403 `M_USER_SUSPENDED`) and the send-time rate limit (`RateLimitEngine::check` with an empty + address bucket, and the user bucket and policy target an ordinary PUT of that event would have). Any failure of + the job's send is recorded `failed` with the standard error object and is never retried. A failed `send` action + returns the send's own result and leaves the row scheduled, timer unchanged. Never add a copy of a send-path check; + call the shared function. +- **No `origin_server_ts` and no `client_txn_ids` row** for a delayed event: it is stamped when it is sent and carries + no `unsigned.transaction_id`. `unsigned.delay_id` (and the unstable name) is added by `ViewerEventUnsigned` for the + scheduling user only, by one `find_sent_by_event_id`. +- **Every path that deactivates an account deletes its rows** (`delete_delayed_events_of_user`, in + `deactivate_user_account`, a failure logged and not fatal). Logout and device deletion do not. A new path that closes + an account does it too. +- **Lookup is by `find_owned` only**, and an unknown, another user's, empty or over-long ID is the one 404 `M_NOT_FOUND` + "Delayed event not found.", byte for byte. +- **Logging** is under `delayed_events` (`delayed_event.scheduled`, `.restarted`, `.cancelled`, `.sent`, `.failed`, + `.finalise_commit_failed`) with `delay_id`, `user_id`, `room_id`, `event_type`, and `event_id`, `status`, `errcode` + where they apply. NEVER log `content_json` or a state key (a call-membership state key holds a device ID). +- `send_event_as_user`'s room and membership refusal is `room_send_refusal`, and its power refusal + `event_authorization_refusal`, both shared with the schedule-time check; change them, not a copy. + ## Typing expiry (R13, ADR-0199) Sending a message does not end the sender's typing (ADR-0298): typing ends only on the client's diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 3cb3fecd6..7a0697976 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -86,6 +86,7 @@ #include "merovingian/homeserver/ClientDeviceSessions.hpp" #include "merovingian/homeserver/ClientOutboundProxy.hpp" #include "merovingian/homeserver/ClientServerClock.hpp" +#include "merovingian/homeserver/ClientServerRuntimeInternal.hpp" #include "merovingian/homeserver/ClientServerStartOptions.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" #include "merovingian/homeserver/CreateRoomOptions.hpp" @@ -14759,6 +14760,27 @@ namespace } // namespace +auto event_txn_part_is_too_long(std::string_view part) noexcept -> bool +{ + return client_txn_part_is_too_long(part); +} + +auto rate_limit_route_template(std::string_view target) -> std::string +{ + return normalized_target(target); +} + +auto route_event_body(std::string_view event_type, std::string_view content, std::optional state_key) + -> std::optional +{ + return event_body_from_content(event_type, content, std::move(state_key)); +} + +auto errcode_for_refusal_status(std::uint16_t status) -> std::string_view +{ + return error_code_for_status(status); +} + auto remember_signed_in_device(ClientServerRuntime& runtime, std::string_view user_id, std::string_view device_id, std::string_view display_name, SignedInDevice kind) -> void { @@ -20618,11 +20640,11 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp // A room event's type is under the size limits, answered as they all are, 413 M_TOO_LARGE. if (client_txn_part_is_too_long(path->event_type)) { - return dispatch_err(req, rt, 413U, "M_TOO_LARGE", "the event type is at most 255 bytes"); + return dispatch_err(req, rt, 413U, "M_TOO_LARGE", event_type_too_long_reason); } if (client_txn_part_is_too_long(path->txn_id)) { - return dispatch_err(req, rt, 400U, "M_INVALID_PARAM", "the transaction ID is at most 255 bytes"); + return dispatch_err(req, rt, 400U, "M_INVALID_PARAM", txn_id_too_long_reason); } // CS API §10.5.1: idempotent send — replay the original response to the same device (F-52, // ADR-0380): a transaction ID is scoped to a device, so another device's is another request. diff --git a/src/homeserver/DelayedEvents.cpp b/src/homeserver/DelayedEvents.cpp new file mode 100644 index 000000000..e4010703c --- /dev/null +++ b/src/homeserver/DelayedEvents.cpp @@ -0,0 +1,623 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "merovingian/homeserver/DelayedEvents.hpp" + +#include "merovingian/canonicaljson/CanonicalJsonError.hpp" +#include "merovingian/canonicaljson/ParseError.hpp" +#include "merovingian/canonicaljson/Parser.hpp" +#include "merovingian/canonicaljson/Serializer.hpp" +#include "merovingian/canonicaljson/Value.hpp" +#include "merovingian/core/QueryParams.hpp" +#include "merovingian/crypto/Encoding.hpp" +#include "merovingian/crypto/Random.hpp" +#include "merovingian/database/DelayedEventOutcomeResult.hpp" +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/DelayedEventStore.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentDelayedEvent.hpp" +#include "merovingian/database/PersistentStore.hpp" +#include "merovingian/events/Event.hpp" +#include "merovingian/events/Limits.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerRuntimeInternal.hpp" +#include "merovingian/homeserver/LocalDatabase.hpp" +#include "merovingian/homeserver/LocalServices.hpp" +#include "merovingian/homeserver/LocalUser.hpp" +#include "merovingian/homeserver/RoomPower.hpp" +#include "merovingian/homeserver/RoomService.hpp" +#include "merovingian/homeserver/RuntimeMutex.hpp" +#include "merovingian/http/RateLimitDecision.hpp" +#include "merovingian/observability/BoundedText.hpp" +#include "merovingian/observability/LogEventSeverity.hpp" +#include "merovingian/observability/Observability.hpp" +#include "merovingian/observability/StructuredLogField.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace merovingian::homeserver +{ +namespace +{ + + constexpr auto log_module = std::string_view{"delayed_events"}; + + // At most this many events are sent in one run of the job, so other threads get the runtime lock between runs. + constexpr auto max_sends_per_run = std::size_t{100U}; + // The age prune deletes at most this many finalised rows per run, in one transaction (ADR-0824). + constexpr auto prune_batch_rows = std::size_t{500U}; + // A finalised row is kept at most this long: 7 days. + constexpr auto finalised_retention_ms = std::uint64_t{7U} * 24U * 60U * 60U * 1000U; + // The job never sleeps longer than this, and sleeps this long with nothing scheduled. + constexpr auto max_job_wait_ms = std::uint64_t{60'000U}; + // The most bytes of an error's reason the stored error object keeps. + constexpr auto max_error_reason_bytes = std::size_t{255U}; + // The bytes of random behind a delay ID: 128 bits. + constexpr auto delay_id_random_bytes = std::size_t{16U}; + // The `retry_after_ms` of a refused schedule is at least one second. + constexpr auto min_retry_after_ms = std::uint64_t{1000U}; + + auto log_event(std::string_view event, database::PersistentDelayedEvent const& row, + std::vector extra = {}, + observability::LogEventSeverity severity = observability::LogEventSeverity::info) -> void + { + // Never `content_json` and never the state key: a call-membership state key holds a device ID. + auto fields = std::vector{ + {"delay_id", row.delay_id, false}, + {"user_id", row.user_id, false}, + {"room_id", row.room_id, false}, + {"event_type", row.event_type, false}, + }; + for (auto& field : extra) + { + fields.push_back(std::move(field)); + } + observability::log_diagnostic(log_module, event, fields, severity); + } + + [[nodiscard]] auto refusal(std::uint16_t status, std::string errcode, std::string reason) -> OperationResult + { + auto result = make_operation_result(false, {}, std::move(reason), status); + result.errcode = std::move(errcode); + return result; + } + + // The one answer for a delay ID that is unknown, another user's, empty or over-long. + [[nodiscard]] auto not_found_result() -> OperationResult + { + return refusal(404U, "M_NOT_FOUND", "Delayed event not found."); + } + + [[nodiscard]] auto conflict_result(std::string reason) -> OperationResult + { + return refusal(409U, "M_UNKNOWN", std::move(reason)); + } + + [[nodiscard]] auto ok_result(std::string value = {}) -> OperationResult + { + return make_operation_result(true, std::move(value)); + } + + [[nodiscard]] auto contains_nul(std::string_view value) noexcept -> bool + { + return value.find('\0') != std::string_view::npos; + } + + // The bound the finalised rows of one user are kept to, from the configured limit (1 to 1000). + [[nodiscard]] auto finalised_bound(ClientServerRuntime const& runtime) noexcept -> std::size_t + { + return std::max(runtime.limits.max_finalised_delayed_events_per_user, 1U); + } + + // A fresh delay ID: 128 random bits in unpadded URL-safe base64 (22 characters). Nullopt if the random source + // failed. + [[nodiscard]] auto make_delay_id() -> std::optional + { + auto const random = crypto::secure_random_bytes(delay_id_random_bytes); + if (!random.has_value()) + { + return std::nullopt; + } + return crypto::base64_urlsafe_encode( + std::string_view{reinterpret_cast(random->data()), random->size()}); + } + + // The `retry_after_ms` of a refused schedule: the time from `now_ms` to `due_ms`, rounded up to a whole second, + // at least one second (Delayed events, "MUST"). + [[nodiscard]] auto retry_after_for(std::uint64_t now_ms, std::uint64_t due_ms) noexcept -> std::uint32_t + { + auto const wait_ms = due_ms > now_ms ? due_ms - now_ms : std::uint64_t{0U}; + auto const rounded_ms = ((wait_ms + 999U) / 1000U) * 1000U; + return static_cast(std::min(std::max(rounded_ms, min_retry_after_ms), + std::numeric_limits::max())); + } + + [[nodiscard]] auto json_number(std::uint64_t value) -> canonicaljson::Value + { + return canonicaljson::Value{static_cast( + std::min(value, static_cast(std::numeric_limits::max())))}; + } + + [[nodiscard]] auto json_text(std::string_view value) -> canonicaljson::Value + { + return canonicaljson::Value{std::string{value}}; + } + + // The standard error object `{"errcode", "error"}` that a failed send leaves in its row. The reason is cut to 255 + // bytes on a character boundary; the whole object stays within `max_delayed_event_error_bytes`. + [[nodiscard]] auto error_object_json(OperationResult const& failure) -> std::string + { + auto const errcode = + failure.errcode.empty() ? std::string{errcode_for_refusal_status(failure.status)} : failure.errcode; + auto build = [&errcode](std::string_view reason) { + auto members = canonicaljson::Object{}; + members.push_back(canonicaljson::make_member("errcode", json_text(errcode))); + members.push_back(canonicaljson::make_member("error", json_text(reason))); + return canonicaljson::serialize_canonical(canonicaljson::Value{std::move(members)}).output; + }; + auto json = build(observability::bounded_utf8(failure.reason, max_error_reason_bytes)); + if (json.size() > database::max_delayed_event_error_bytes) + { + json = build({}); + } + return json; + } + + // The account check the request gate makes, applied when a delayed event is sent: a user who is unknown or + // deactivated, locked or suspended sends nothing (the answers and messages are the gate's). Nullopt when the + // account may send. Caller holds runtime.mutex. + [[nodiscard]] auto account_refusal(HomeserverRuntime const& runtime, std::string_view user_id) + -> std::optional + { + auto const user = std::ranges::find_if(runtime.database.users, [user_id](LocalUser const& candidate) { + return candidate.user_id == user_id; + }); + if (user == runtime.database.users.end() || user->deactivated) + { + return refusal(403U, "M_USER_DEACTIVATED", "This account has been deactivated."); + } + if (user->locked) + { + return refusal(401U, "M_USER_LOCKED", "This account has been locked"); + } + if (user->suspended) + { + return refusal(403U, "M_USER_SUSPENDED", "You cannot perform this action while suspended."); + } + return std::nullopt; + } + + // The path the ordinary `PUT` route of `row`'s event has: `.../send/{eventType}/{txnId}` for a message and + // `.../state/{eventType}/{stateKey}` for a state event (no key segment for the empty state key). + [[nodiscard]] auto ordinary_route_path(database::PersistentDelayedEvent const& row) -> std::string + { + auto path = std::string{"/_matrix/client/v3/rooms/"}; + path += core::percent_encode_path_component(row.room_id); + if (row.is_state) + { + path += "/state/"; + path += core::percent_encode_path_component(row.event_type); + if (!row.state_key.empty()) + { + path += '/'; + path += core::percent_encode_path_component(row.state_key); + } + return path; + } + path += "/send/"; + path += core::percent_encode_path_component(row.event_type); + path += '/'; + path += core::percent_encode_path_component(row.txn_id.empty() ? std::string_view{"0"} : row.txn_id); + return path; + } + + // The per-user rate limit an ordinary request for `row`'s event would meet, checked at send time (Delayed events: + // "MUST be subject to rate limits"). The per-address bucket is skipped (an empty first argument); the user's + // bucket and the policy are the ones `allow()` of the dispatcher builds for an authenticated PUT of that event. + // A null engine allows. Caller holds runtime.mutex. + [[nodiscard]] auto send_rate_limit_refusal(ClientServerRuntime& runtime, + database::PersistentDelayedEvent const& row) + -> std::optional + { + if (runtime.rate_limit_engine == nullptr) + { + return std::nullopt; + } + auto const path = ordinary_route_path(row); + auto const route = rate_limit_route_template(path); + auto user_key = row.user_id; + user_key.push_back('|'); + user_key.append(route); + auto const decision = runtime.rate_limit_engine->check({}, route, user_key, path); + if (decision.allowed) + { + return std::nullopt; + } + auto result = refusal(429U, "M_LIMIT_EXCEEDED", "Too many requests. Please try again later."); + result.retry_after_ms = decision.retry_after_ms; + return result; + } + + // One send of the scheduled row `row` (design 3.2 steps 2 to 4): the account check, the send-time rate limit, then + // the ordinary send path, which authorises the event against the room's state now. On success `value` is the + // event ID. Nothing is recorded here. Caller holds runtime.mutex and has marked the row as being sent. + [[nodiscard]] auto attempt_send(ClientServerRuntime& runtime, database::PersistentDelayedEvent const& row) + -> OperationResult + { + if (auto account = account_refusal(runtime.homeserver, row.user_id); account.has_value()) + { + return std::move(*account); + } + if (auto limited = send_rate_limit_refusal(runtime, row); limited.has_value()) + { + return std::move(*limited); + } + auto body = row.is_state ? route_event_body(row.event_type, row.content_json, row.state_key) + : route_event_body(row.event_type, row.content_json); + if (!body.has_value()) + { + return refusal(400U, "M_BAD_JSON", "event content must be a JSON object"); + } + return send_event_as_resolved_user(runtime.homeserver, row.user_id, row.room_id, *body, row.via_appservice); + } + + // Records the outcome of the send `result` of `row` as sent or failed, at `now_ms`, and logs it. A commit failure + // is logged at error level and the caller carries on (ADR-0825: memory is finalised, the next start sends the + // event again). Caller holds runtime.mutex. + auto record_send_outcome(ClientServerRuntime& runtime, database::PersistentDelayedEvent const& row, + OperationResult const& result, std::uint64_t now_ms) -> void + { + auto& store = runtime.homeserver.database.persistent_store; + auto const outcome = + result.ok ? database::record_delayed_event_outcome(store, row.delay_id, database::DelayedEventStatus::sent, + now_ms, result.value, {}, finalised_bound(runtime)) + : database::record_delayed_event_outcome(store, row.delay_id, + database::DelayedEventStatus::failed, now_ms, {}, + error_object_json(result), finalised_bound(runtime)); + if (result.ok) + { + log_event("delayed_event.sent", row, + { + {"event_id", result.value, false} + }); + } + else + { + log_event( + "delayed_event.failed", row, + { + {"status", std::to_string(result.status), false}, + {"errcode", + result.errcode.empty() ? std::string{errcode_for_refusal_status(result.status)} : result.errcode, + false }, + }, + observability::LogEventSeverity::warning); + } + if (outcome != database::DelayedEventOutcomeResult::recorded) + { + log_event( + "delayed_event.finalise_commit_failed", row, + { + {"outcome", outcome == database::DelayedEventOutcomeResult::refused ? "refused" : "commit_failed", + false} + }, + observability::LogEventSeverity::error); + } + } + + // Sends the earliest-due row if it is due at `now_ms`, under its own hold of the runtime lock, and records the + // outcome (any failure is `failed`, never retried). False when nothing was due. + [[nodiscard]] auto send_next_due(ClientServerRuntime& runtime, std::uint64_t now_ms) -> bool + { + auto const guard = RuntimeLock{runtime.homeserver.mutex}; + auto& table = runtime.homeserver.database.persistent_store.delayed_events; + auto const* next = table.earliest_due(); + if (next == nullptr || database::delayed_event_due_ms(*next) > now_ms) + { + return false; + } + // A copy: the send and the outcome change the table, which invalidates the pointer. + auto const row = *next; + if (!table.mark_being_sent(row.delay_id)) + { + return false; + } + auto const result = attempt_send(runtime, row); + // Finalising clears the mark. A record that was refused leaves the row marked, so it is never sent again by + // this process (and is logged), never in a loop. + record_send_outcome(runtime, row, result, now_ms); + return true; + } + +} // namespace + +auto delayed_event_now_ms(HomeserverRuntime const& runtime) -> std::uint64_t +{ + if (runtime.test_delayed_event_clock_ms.has_value()) + { + return *runtime.test_delayed_event_clock_ms; + } + auto const now = + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); + return now < 0 ? std::uint64_t{0U} : static_cast(now); +} + +auto wake_delayed_events(HomeserverRuntime& runtime) -> void +{ + if (runtime.delayed_events_wake) + { + runtime.delayed_events_wake(); + } +} + +auto schedule_delayed_event(ClientServerRuntime& runtime, DelayedEventScheduleRequest const& request) -> OperationResult +{ + // 1. The feature switch. + if (!runtime.limits.delayed_events_enabled) + { + return refusal(403U, "M_FORBIDDEN", "Delayed events are not enabled on this homeserver."); + } + // 2. The shape: the answers the ordinary `/send` and `/state` routes give. + if (event_txn_part_is_too_long(request.event_type)) + { + return refusal(413U, "M_TOO_LARGE", std::string{event_type_too_long_reason}); + } + if (event_txn_part_is_too_long(request.txn_id)) + { + return refusal(400U, "M_INVALID_PARAM", std::string{txn_id_too_long_reason}); + } + if (contains_nul(request.event_type)) + { + return refusal(400U, "M_INVALID_PARAM", "the event type must not contain NUL"); + } + if (contains_nul(request.txn_id)) + { + return refusal(400U, "M_INVALID_PARAM", "the transaction ID must not contain NUL"); + } + if (request.state_key.has_value()) + { + if (events::state_key_contains_nul(*request.state_key)) + { + return invalid_state_key_refusal(); + } + if (request.state_key->size() > events::max_state_key_length_bytes) + { + return event_too_large_refusal(); + } + } + if (request.content_json.size() > database::max_delayed_event_content_bytes) + { + return event_too_large_refusal(); + } + { + auto const parsed = canonicaljson::parse_lossless(request.content_json); + if (parsed.error != canonicaljson::ParseError::none || + std::get_if(&parsed.value.storage()) == nullptr) + { + return refusal(400U, "M_BAD_JSON", "event content must be a JSON object"); + } + } + if (request.delay_ms == 0U) + { + return refusal(400U, "M_INVALID_PARAM", "delay_ms must be at least 1"); + } + // 3. The longest delay. + if (request.delay_ms > runtime.limits.max_delayed_event_delay_ms) + { + return refusal(400U, request.stable_errcode ? "M_DELAY_TOO_LARGE" : "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE", + "The requested delay exceeds the allowed maximum."); + } + auto& store = runtime.homeserver.database.persistent_store; + // 4. Idempotency: the same (user, device, room, type, transaction ID) is the same delayed event. + if (!request.txn_id.empty()) + { + if (auto const* existing = store.delayed_events.find_by_transaction( + request.user_id, request.device_id, request.room_id, request.event_type, request.txn_id); + existing != nullptr) + { + return ok_result(existing->delay_id); + } + } + // 5. Could it be sent now: the room and the sender's membership, then for a state event the sender's power. + if (auto unsendable = room_send_refusal(runtime.homeserver, request.user_id, request.room_id); + unsendable.has_value()) + { + return std::move(*unsendable); + } + if (request.state_key.has_value() && + !may_send_state_event(store, request.room_id, request.user_id, request.event_type)) + { + return event_authorization_refusal(); + } + // 6. The per-user cap. + auto const now_ms = delayed_event_now_ms(runtime.homeserver); + if (store.delayed_events.scheduled_count(request.user_id) >= runtime.limits.max_scheduled_delayed_events_per_user) + { + auto result = refusal(429U, "M_LIMIT_EXCEEDED", "The maximum number of delayed events has been reached."); + result.retry_after_ms = + retry_after_for(now_ms, store.delayed_events.earliest_due_ms_of_user(request.user_id).value_or(now_ms)); + return result; + } + + auto delay_id = make_delay_id(); + if (!delay_id.has_value()) + { + return refusal(500U, "M_UNKNOWN", "could not generate a delay ID"); + } + auto row = database::PersistentDelayedEvent{}; + row.delay_id = *delay_id; + row.user_id = request.user_id; + row.device_id = request.device_id; + row.room_id = request.room_id; + row.event_type = request.event_type; + row.is_state = request.state_key.has_value(); + row.state_key = request.state_key.value_or(std::string{}); + row.content_json = request.content_json; + row.txn_id = request.txn_id; + row.via_appservice = request.via_appservice; + row.delay_ms = request.delay_ms; + row.delayed_since_ms = now_ms; + if (!database::store_delayed_event(store, row)) + { + return refusal(500U, "M_UNKNOWN", "the delayed event could not be stored"); + } + log_event("delayed_event.scheduled", row, + { + {"delay_ms", std::to_string(row.delay_ms), false} + }); + wake_delayed_events(runtime.homeserver); + return ok_result(*delay_id); +} + +auto manage_delayed_event(ClientServerRuntime& runtime, std::string_view user_id, std::string_view delay_id, + DelayedEventAction action) -> OperationResult +{ + auto& store = runtime.homeserver.database.persistent_store; + auto const* found = store.delayed_events.find_owned(user_id, delay_id); + if (found == nullptr) + { + return not_found_result(); + } + // A copy: every change below invalidates the pointer. + auto const row = *found; + auto const now_ms = delayed_event_now_ms(runtime.homeserver); + if (row.status == database::DelayedEventStatus::scheduled) + { + if (store.delayed_events.is_being_sent(row.delay_id)) + { + return conflict_result("The delayed event is being sent."); + } + switch (action) + { + case DelayedEventAction::restart: + if (!database::restart_delayed_event(store, row.delay_id, now_ms)) + { + return refusal(500U, "M_UNKNOWN", "the delayed event could not be restarted"); + } + log_event("delayed_event.restarted", row); + return ok_result(); + case DelayedEventAction::cancel: + if (!database::cancel_delayed_event(store, row.delay_id, now_ms, finalised_bound(runtime))) + { + return refusal(500U, "M_UNKNOWN", "the delayed event could not be cancelled"); + } + log_event("delayed_event.cancelled", row); + return ok_result(); + case DelayedEventAction::send: { + if (!store.delayed_events.mark_being_sent(row.delay_id)) + { + return conflict_result("The delayed event is being sent."); + } + auto result = attempt_send(runtime, row); + if (!result.ok) + { + // The row stays scheduled with its timer unchanged; the caller answers as the send did. + std::ignore = store.delayed_events.clear_being_sent(row.delay_id); + return result; + } + record_send_outcome(runtime, row, result, now_ms); + return ok_result(); + } + } + return refusal(500U, "M_UNKNOWN", "unknown action"); + } + if (row.status == database::DelayedEventStatus::sent) + { + return action == DelayedEventAction::send ? ok_result() + : conflict_result("The delayed event has already been sent."); + } + // Cancelled by its owner, or cancelled because sending it failed. + return action == DelayedEventAction::cancel ? ok_result() + : conflict_result("The delayed event has already been cancelled."); +} + +auto delayed_event_lookup(ClientServerRuntime& runtime, std::string_view user_id, std::string_view delay_id) + -> OperationResult +{ + auto const* row = runtime.homeserver.database.persistent_store.delayed_events.find_owned(user_id, delay_id); + if (row == nullptr) + { + return not_found_result(); + } + auto parsed = canonicaljson::parse_lossless(row->content_json); + if (parsed.error != canonicaljson::ParseError::none) + { + return refusal(500U, "M_UNKNOWN", "the delayed event's content could not be read"); + } + auto members = canonicaljson::Object{}; + members.push_back(canonicaljson::make_member("delay_id", json_text(row->delay_id))); + members.push_back(canonicaljson::make_member("room_id", json_text(row->room_id))); + members.push_back(canonicaljson::make_member("type", json_text(row->event_type))); + if (row->is_state) + { + members.push_back(canonicaljson::make_member("state_key", json_text(row->state_key))); + } + members.push_back(canonicaljson::make_member("delay_ms", json_number(row->delay_ms))); + members.push_back(canonicaljson::make_member("delayed_since_ts", json_number(row->delayed_since_ms))); + members.push_back(canonicaljson::make_member("content", std::move(parsed.value))); + if (row->status != database::DelayedEventStatus::scheduled) + { + auto finalised = canonicaljson::Object{}; + finalised.push_back(canonicaljson::make_member("finalised_ts", json_number(row->finalised_at_ms))); + if (row->status == database::DelayedEventStatus::sent) + { + finalised.push_back(canonicaljson::make_member("event_id", json_text(row->event_id))); + } + if (row->status == database::DelayedEventStatus::failed) + { + auto error = canonicaljson::parse_lossless(row->error_json); + if (error.error == canonicaljson::ParseError::none) + { + finalised.push_back(canonicaljson::make_member("error", std::move(error.value))); + } + } + members.push_back(canonicaljson::make_member("finalised", canonicaljson::Value{std::move(finalised)})); + } + auto const serialised = canonicaljson::serialize_canonical(canonicaljson::Value{std::move(members)}); + if (serialised.error != canonicaljson::CanonicalJsonError::none) + { + return refusal(500U, "M_UNKNOWN", "the delayed event could not be serialised"); + } + return ok_result(serialised.output); +} + +auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, StopFlag const& stop) + -> std::chrono::milliseconds +{ + // One send per hold of the runtime lock, the stop flag read between items, at most `max_sends_per_run` a run. + auto sent = std::size_t{0U}; + while (sent < max_sends_per_run && !stop.stop_requested() && send_next_due(runtime, now_ms)) + { + ++sent; + } + auto const guard = RuntimeLock{runtime.homeserver.mutex}; + auto& store = runtime.homeserver.database.persistent_store; + if (!stop.stop_requested()) + { + // One batch of the age prune, in this hold. A failed commit leaves the rows for the next run. + auto const cutoff_ms = now_ms > finalised_retention_ms ? now_ms - finalised_retention_ms : std::uint64_t{0U}; + std::ignore = database::prune_finalised_delayed_events(store, cutoff_ms, prune_batch_rows); + } + auto const* next = store.delayed_events.earliest_due(); + if (next == nullptr) + { + return std::chrono::milliseconds{max_job_wait_ms}; + } + auto const due_ms = database::delayed_event_due_ms(*next); + auto const wait_ms = due_ms > now_ms ? due_ms - now_ms : std::uint64_t{0U}; + return std::chrono::milliseconds{std::min(wait_ms, max_job_wait_ms)}; +} + +} // namespace merovingian::homeserver diff --git a/src/homeserver/RoomService.cpp b/src/homeserver/RoomService.cpp index e61c9cb38..6557a1bd0 100644 --- a/src/homeserver/RoomService.cpp +++ b/src/homeserver/RoomService.cpp @@ -9629,15 +9629,13 @@ namespace return state_content_refusal_result(*refusal); } - // send_event once the sender is known. The profile propagation job (ADR-0353) calls it directly: it holds no token. - // `via_appservice` is set when an application service asserted the sender: the server-administrator override of - // a redaction is then not applied (ADR-0628). - [[nodiscard]] auto send_event_as_user(HomeserverRuntime& runtime, std::string const& user_id, - std::string_view room_id, std::string_view event_json, - std::optional origin_server_ts, bool via_appservice = false) - -> OperationResult + // The refusal send_event_as_user gives when `room_id` is not a room this server holds or `user_id` is not joined to + // it (logged as room.event.rejected), or nullopt when the room can be sent to. Shared with the delayed events' + // schedule-time check (`room_send_refusal`), so the two answer alike. Caller holds runtime.mutex. + [[nodiscard]] auto unsendable_room_refusal(HomeserverRuntime& runtime, std::string const& user_id, + std::string_view room_id) -> std::optional { - auto* room = find_room(runtime.database, room_id); + auto const* room = find_room(runtime.database, room_id); if (room == nullptr) { log_diagnostic("room.event.rejected", { @@ -9656,6 +9654,26 @@ namespace }); return make_operation_result(false, {}, "not joined", 403U); } + return std::nullopt; + } + + // send_event once the sender is known. The profile propagation job (ADR-0353) calls it directly: it holds no token. + // `via_appservice` is set when an application service asserted the sender: the server-administrator override of + // a redaction is then not applied (ADR-0628). + [[nodiscard]] auto send_event_as_user(HomeserverRuntime& runtime, std::string const& user_id, + std::string_view room_id, std::string_view event_json, + std::optional origin_server_ts, bool via_appservice = false) + -> OperationResult + { + if (auto refusal = unsendable_room_refusal(runtime, user_id, room_id); refusal.has_value()) + { + return std::move(*refusal); + } + auto* room = find_room(runtime.database, room_id); + if (room == nullptr) + { + return make_operation_result(false, {}, "unknown room"); + } if (event_json.empty()) { log_diagnostic("room.event.rejected", { @@ -9710,7 +9728,7 @@ namespace {"status", "403", false}, {"reason", "event authorization or signing failed", false} }); - return make_operation_result(false, {}, "event authorization or signing failed", 403U); + return event_authorization_refusal(); } // CSAZ-11. Spec (Client-Server API, PUT /redact): a user may redact their own events, another // user's only with the `redact` level, and a server administrator any event sent by a user on @@ -9839,6 +9857,33 @@ namespace } // NOLINTEND(bugprone-easily-swappable-parameters) +auto send_event_as_resolved_user(HomeserverRuntime& runtime, std::string const& user_id, std::string_view room_id, + std::string_view event_json, bool via_appservice) -> OperationResult +{ + return send_event_as_user(runtime, user_id, room_id, event_json, std::nullopt, via_appservice); +} + +auto room_send_refusal(HomeserverRuntime& runtime, std::string const& user_id, std::string_view room_id) + -> std::optional +{ + return unsendable_room_refusal(runtime, user_id, room_id); +} + +auto event_authorization_refusal() -> OperationResult +{ + return make_operation_result(false, {}, "event authorization or signing failed", 403U); +} + +auto event_too_large_refusal() -> OperationResult +{ + return event_too_large_result(); +} + +auto invalid_state_key_refusal() -> OperationResult +{ + return invalid_state_key_result(); +} + namespace { diff --git a/src/homeserver/meson.build b/src/homeserver/meson.build index a6e9a4d2b..46bddf1fd 100644 --- a/src/homeserver/meson.build +++ b/src/homeserver/meson.build @@ -20,6 +20,7 @@ homeserver_sources = files( 'DefaultPushRuleset.cpp', 'DeferredPduRetry.cpp', 'DeferredPduRetryJob.cpp', + 'DelayedEvents.cpp', 'EmailPush.cpp', 'EmailPushJob.cpp', 'PasswordReset.cpp', From be9f60a25a77e0e70d4fa9fb93942ae37a6565d5 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:46:24 +0000 Subject: [PATCH 09/29] Run the delayed-events job, delete a deactivated account's rows and show unsigned.delay_id, with their flow tests Co-Authored-By: Claude Fable 5.2 --- src/homeserver/AuthService.cpp | 12 + src/main.cpp | 26 + src/sync/ViewerEventUnsigned.cpp | 13 + .../integration/test_delayed_events_flow.cpp | 1291 +++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1343 insertions(+) create mode 100644 tests/integration/test_delayed_events_flow.cpp diff --git a/src/homeserver/AuthService.cpp b/src/homeserver/AuthService.cpp index a880e062e..412d003fc 100644 --- a/src/homeserver/AuthService.cpp +++ b/src/homeserver/AuthService.cpp @@ -26,6 +26,7 @@ #include "merovingian/crypto/TokenHmacKey.hpp" #include "merovingian/crypto/TokenHmacKeys.hpp" #include "merovingian/crypto/TokenKey.hpp" +#include "merovingian/database/DelayedEventStore.hpp" #include "merovingian/database/OAuthStore.hpp" #include "merovingian/federation/KeySignatures.hpp" #include "merovingian/homeserver/AccountSummary.hpp" @@ -2585,6 +2586,17 @@ auto deactivate_user_account(HomeserverRuntime& runtime, std::string_view user_i }, observability::LogEventSeverity::warning); } + // Every delayed event of the account, scheduled and finalised, goes too (MSC4140, ADR-0824): a deactivated + // account sends nothing. A failure is logged and deactivation goes on, because the send-time account check of + // the delayed-events job refuses a deactivated sender (ADR-0825). + if (!database::delete_delayed_events_of_user(runtime.database.persistent_store, user_id)) + { + log_diagnostic("account.deactivated.delayed_events_removal_failed", + { + {"user_id", std::string{user_id}, false} + }, + observability::LogEventSeverity::warning); + } // The password hash is replaced with a value no password can produce, so a // credential leak predating deactivation cannot be replayed even if a future // change were to soften the login gate. diff --git a/src/main.cpp b/src/main.cpp index 2606ec74b..84f471f3e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,6 +18,7 @@ #include "merovingian/homeserver/ClientServerRuntime.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" #include "merovingian/homeserver/DeferredPduRetryJob.hpp" +#include "merovingian/homeserver/DelayedEvents.hpp" #include "merovingian/homeserver/EmailPushJob.hpp" #include "merovingian/homeserver/FederationProxy.hpp" #include "merovingian/homeserver/HttpConnectionDispatcher.hpp" @@ -931,6 +932,24 @@ struct ListenerBinding final sync_pool.request_stop(); return std::nullopt; } + // Sends the delayed events (MSC4140, ADR-0825) that have fallen due, one send per hold of the runtime lock, and + // ages out finalised ones. It sleeps until the earliest deadline and is woken when an earlier one is scheduled. + // It only ever touches `runtime`, which outlives this function. Declared after the pools so it is stopped and + // joined before them. + auto delayed_events_job = merovingian::homeserver::PeriodicJob{ + "delayed-events", [&runtime](merovingian::homeserver::StopFlag const& stop) { + return merovingian::homeserver::send_due_delayed_events( + runtime, merovingian::homeserver::delayed_event_now_ms(runtime.homeserver), stop); + }}; + if (!delayed_events_job.start()) + { + LOG_CRITICAL("Delayed events thread failed to start; refusing to serve"); + dispatcher.request_stop(); + pool.request_stop(); + media_fetch_pool.request_stop(); + sync_pool.request_stop(); + return std::nullopt; + } // Sends a profile change to the rooms its request left (F-11, ADR-0353), taking the runtime lock // per room. It only ever touches `runtime`, which outlives this function. Declared after the // pools so it is stopped and joined before them. @@ -1037,6 +1056,9 @@ struct ListenerBinding final runtime.homeserver.profile_propagation_wake = [&profile_propagation_job]() { profile_propagation_job.wake(); }; + runtime.homeserver.delayed_events_wake = [&delayed_events_job]() { + delayed_events_job.wake(); + }; } auto threads = std::vector{}; threads.reserve(bindings.size()); @@ -1109,8 +1131,12 @@ struct ListenerBinding final auto const guard = merovingian::homeserver::RuntimeLock{runtime.homeserver.mutex}; runtime.homeserver.typing_expiry_wake = nullptr; runtime.homeserver.profile_propagation_wake = nullptr; + runtime.homeserver.delayed_events_wake = nullptr; } typing_expiry_job.stop(); + // The wake hook is cleared above; a send in progress ends first (it makes no network call). Events still + // scheduled stay in the database and are sent by the next start (ADR-0825). + delayed_events_job.stop(); // Rooms still queued are left for the user's next profile change (ADR-0353). One room's send ends // first; it makes no network call. profile_propagation_job.stop(); diff --git a/src/sync/ViewerEventUnsigned.cpp b/src/sync/ViewerEventUnsigned.cpp index 184b406af..ad210f2bf 100644 --- a/src/sync/ViewerEventUnsigned.cpp +++ b/src/sync/ViewerEventUnsigned.cpp @@ -5,6 +5,9 @@ #include "merovingian/canonicaljson/ParseError.hpp" #include "merovingian/canonicaljson/Parser.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentDelayedEvent.hpp" +#include "merovingian/database/PersistentStore.hpp" #include #include @@ -107,6 +110,16 @@ auto ViewerEventUnsigned::apply(database::PersistentEvent const& event, canonica { added.push_back(canonicaljson::make_member("transaction_id", canonicaljson::Value{std::move(*transaction)})); } + // MSC4140 (ADR-0824): the user who scheduled the delayed event this event became is told its delay ID, under the + // stable name and the unstable one, for as long as the finalised record is kept. Nobody else is. One hash lookup + // by event ID; no scan. + if (auto const* delayed = store_.delayed_events.find_sent_by_event_id(event.event_id); + delayed != nullptr && delayed->user_id == user_id_) + { + added.push_back(canonicaljson::make_member("delay_id", canonicaljson::Value{std::string{delayed->delay_id}})); + added.push_back(canonicaljson::make_member("org.matrix.msc4140.delay_id", + canonicaljson::Value{std::string{delayed->delay_id}})); + } if (auto membership = visibility_.membership_at(event); membership.has_value()) { added.push_back(canonicaljson::make_member("membership", canonicaljson::Value{std::move(*membership)})); diff --git a/tests/integration/test_delayed_events_flow.cpp b/tests/integration/test_delayed_events_flow.cpp new file mode 100644 index 000000000..eba8be54e --- /dev/null +++ b/tests/integration/test_delayed_events_flow.cpp @@ -0,0 +1,1291 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// Delayed events (MSC4140, ADR-0823 to ADR-0826) below the routes: the service functions that schedule, manage and look +// up a delayed event, the job body that sends it, and what they leave in `/sync`. A real runtime and a real store +// (in memory, or a SQLite file for the restart scenarios). No scenario sleeps, starts the job's thread or depends on +// how long anything took: time is `HomeserverRuntime::test_delayed_event_clock_ms` and the job's `now_ms` parameter. +// The feature is not in the Matrix v1.19 specification, so there is no conformance scenario. + +#include "../support/InMemoryDatabaseConfig.hpp" +#include "../support/JsonTestSupport.hpp" +#include "../support/MasterKey.hpp" +#include "../support/RegistrationToken.hpp" +#include "../support/TempDirectory.hpp" +#include "merovingian/canonicaljson/Serializer.hpp" +#include "merovingian/canonicaljson/Value.hpp" +#include "merovingian/config/ClientRateLimitsConfig.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/config/DatabaseBackend.hpp" +#include "merovingian/config/DatabaseConfig.hpp" +#include "merovingian/config/ListenersConfig.hpp" +#include "merovingian/config/LogModulesConfig.hpp" +#include "merovingian/config/SecurityConfig.hpp" +#include "merovingian/config/ServerConfig.hpp" +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentStore.hpp" +#include "merovingian/homeserver/AuthService.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerStartResult.hpp" +#include "merovingian/homeserver/DeactivationAudit.hpp" +#include "merovingian/homeserver/DelayedEventAction.hpp" +#include "merovingian/homeserver/DelayedEventScheduleRequest.hpp" +#include "merovingian/homeserver/DelayedEvents.hpp" +#include "merovingian/homeserver/LocalHttpResponse.hpp" +#include "merovingian/homeserver/OperationResult.hpp" +#include "merovingian/homeserver/RuntimeMutex.hpp" +#include "merovingian/homeserver/StopFlag.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +namespace hs = merovingian::homeserver; +namespace cj = merovingian::canonicaljson; +namespace db = merovingian::database; +using namespace merovingian::tests; + +constexpr auto t0 = std::uint64_t{1'700'000'000'000U}; +constexpr auto password = std::string_view{"CorrectHorse7!"}; +constexpr auto alice = std::string_view{"@alice:example.org"}; +constexpr auto bob = std::string_view{"@bob:example.org"}; + +[[nodiscard]] auto base_config(merovingian::config::DatabaseConfig database, + merovingian::config::ClientRateLimitsConfig limits = {}) -> merovingian::config::Config +{ + auto security = merovingian::config::SecurityConfig{}; + security.secrets.master_key_file = shared_master_key_file(); + enable_token_registration(security); + return { + merovingian::config::ServerConfig{}, + merovingian::config::ListenersConfig{}, + std::move(database), + std::move(security), + std::move(limits), + merovingian::config::LogModulesConfig{}, + }; +} + +[[nodiscard]] auto memory_config(merovingian::config::ClientRateLimitsConfig limits = {}) -> merovingian::config::Config +{ + return base_config(in_memory_database_config(), std::move(limits)); +} + +[[nodiscard]] auto sqlite_config(std::filesystem::path const& path) -> merovingian::config::Config +{ + auto database = merovingian::config::DatabaseConfig{}; + database.backend = merovingian::config::DatabaseBackend::sqlite; + database.sqlite_path = path.string(); + return base_config(std::move(database)); +} + +[[nodiscard]] auto request(hs::ClientServerRuntime& runtime, std::string method, std::string target, + std::string const& token = {}, std::string body = {}) -> hs::LocalHttpResponse +{ + return hs::handle_client_server_request(runtime, {std::move(method), std::move(target), token, std::move(body)}) + .response; +} + +// Registers `name` and signs it in on device `device`; the access token. +[[nodiscard]] auto sign_in(hs::ClientServerRuntime& runtime, std::string const& name, std::string const& device) + -> std::string +{ + REQUIRE(request(runtime, "POST", "/_matrix/client/v3/register", {}, registration_json(name, password)).status == + 200U); + auto const login = + request(runtime, "POST", "/_matrix/client/v3/login", {}, + R"({"type":"m.login.password","identifier":{"type":"m.id.user","user":"@)" + name + + R"(:example.org"},"password":")" + std::string{password} + R"(","device_id":")" + device + R"("})"); + REQUIRE(login.status == 200U); + auto const body = parse_object(login.body); + REQUIRE(string_member(body, "access_token") != nullptr); + return *string_member(body, "access_token"); +} + +// A running homeserver with alice and bob in one public room. Alice created it. +class Fixture final +{ +public: + explicit Fixture(merovingian::config::Config const& config) + : m_started{hs::start_client_server(config)} + { + REQUIRE(m_started.started); + alice_token = sign_in(runtime(), "alice", "D1"); + bob_token = sign_in(runtime(), "bob", "D2"); + auto const created = + request(runtime(), "POST", "/_matrix/client/v3/createRoom", alice_token, R"({"preset":"public_chat"})"); + REQUIRE(created.status == 200U); + room_id = *string_member(parse_object(created.body), "room_id"); + REQUIRE(request(runtime(), "POST", "/_matrix/client/v3/join/" + room_id, bob_token, "{}").status == 200U); + runtime().homeserver.test_delayed_event_clock_ms = t0; + runtime().homeserver.test_frozen_event_clock_ms = static_cast(t0); + } + + [[nodiscard]] auto runtime() noexcept -> hs::ClientServerRuntime& + { + return m_started.runtime; + } + + [[nodiscard]] auto table() noexcept -> db::DelayedEventTable& + { + return m_started.runtime.homeserver.database.persistent_store.delayed_events; + } + + [[nodiscard]] auto event_count() noexcept -> std::size_t + { + return m_started.runtime.homeserver.database.persistent_store.events.size(); + } + + std::string alice_token{}; + std::string bob_token{}; + std::string room_id{}; + +private: + hs::ClientServerStartResult m_started; +}; + +[[nodiscard]] auto message_request(Fixture const& fixture, std::string_view user = alice, std::string txn = "txn-1", + std::uint64_t delay_ms = 60'000U) -> hs::DelayedEventScheduleRequest +{ + auto made = hs::DelayedEventScheduleRequest{}; + made.user_id = std::string{user}; + made.device_id = "D1"; + made.room_id = fixture.room_id; + made.event_type = "m.test.delayed"; + made.content_json = R"({"body":"later"})"; + made.txn_id = std::move(txn); + made.delay_ms = delay_ms; + return made; +} + +[[nodiscard]] auto state_request(Fixture const& fixture, std::string_view user = alice, + std::uint64_t delay_ms = 60'000U) -> hs::DelayedEventScheduleRequest +{ + auto made = message_request(fixture, user, {}, delay_ms); + made.event_type = "m.test.state"; + made.state_key = std::string{}; + made.content_json = R"({"v":1})"; + return made; +} + +// Every service function below needs the runtime lock; a request handler holds it, so a test takes it here. +[[nodiscard]] auto schedule(Fixture& fixture, hs::DelayedEventScheduleRequest const& made) -> hs::OperationResult +{ + auto const guard = hs::RuntimeLock{fixture.runtime().homeserver.mutex}; + return hs::schedule_delayed_event(fixture.runtime(), made); +} + +[[nodiscard]] auto manage(Fixture& fixture, std::string_view user, std::string_view delay_id, + hs::DelayedEventAction action) -> hs::OperationResult +{ + auto const guard = hs::RuntimeLock{fixture.runtime().homeserver.mutex}; + return hs::manage_delayed_event(fixture.runtime(), user, delay_id, action); +} + +[[nodiscard]] auto lookup(Fixture& fixture, std::string_view user, std::string_view delay_id) -> hs::OperationResult +{ + auto const guard = hs::RuntimeLock{fixture.runtime().homeserver.mutex}; + return hs::delayed_event_lookup(fixture.runtime(), user, delay_id); +} + +[[nodiscard]] auto run_job(Fixture& fixture, std::uint64_t now_ms, hs::StopFlag const& stop = {}) + -> std::chrono::milliseconds +{ + return hs::send_due_delayed_events(fixture.runtime(), now_ms, stop); +} + +[[nodiscard]] auto scheduled_id(Fixture& fixture, hs::DelayedEventScheduleRequest const& made) -> std::string +{ + auto const result = schedule(fixture, made); + REQUIRE(result.ok); + REQUIRE_FALSE(result.value.empty()); + return result.value; +} + +[[nodiscard]] auto status_of(Fixture& fixture, std::string_view user, std::string_view delay_id) + -> db::DelayedEventStatus +{ + auto const* row = fixture.table().find_owned(user, delay_id); + REQUIRE(row != nullptr); + return row->status; +} + +// The delayed event as `GET` shows it, parsed. +[[nodiscard]] auto looked_up(Fixture& fixture, std::string_view user, std::string_view delay_id) -> cj::Object +{ + auto const result = lookup(fixture, user, delay_id); + REQUIRE(result.ok); + return parse_object(result.value); +} + +struct Synced final +{ + std::string next_batch{}; + std::optional event{}; +}; + +// One `/sync` for `token`; `event` is the first timeline event of `type` in `fixture`'s room. +[[nodiscard]] auto sync_for(Fixture& fixture, std::string const& token, std::string const& since, std::string_view type) + -> Synced +{ + auto const response = request( + fixture.runtime(), "GET", + since.empty() ? std::string{"/_matrix/client/v3/sync"} : "/_matrix/client/v3/sync?since=" + since, token); + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + auto synced = Synced{}; + synced.next_batch = *string_member(body, "next_batch"); + auto const* rooms = object_member_as_object(body, "rooms"); + auto const* joined = rooms == nullptr ? nullptr : object_member_as_object(*rooms, "join"); + auto const* room = joined == nullptr ? nullptr : object_member_as_object(*joined, fixture.room_id); + auto const* timeline = room == nullptr ? nullptr : object_member_as_object(*room, "timeline"); + auto const* events = timeline == nullptr ? nullptr : object_member_as_array(*timeline, "events"); + if (events == nullptr) + { + return synced; + } + for (auto const& entry : *events) + { + auto const* object = std::get_if(&entry.storage()); + if (object != nullptr && string_member(*object, "type") != nullptr && *string_member(*object, "type") == type) + { + synced.event = *object; + return synced; + } + } + return synced; +} + +// Sets `user`'s power in the room to `level` (alice sends the new power levels). +auto set_power(Fixture& fixture, std::string_view user, std::int64_t level) -> void +{ + auto const current = + request(fixture.runtime(), "GET", "/_matrix/client/v3/rooms/" + fixture.room_id + "/state/m.room.power_levels", + fixture.alice_token); + REQUIRE(current.status == 200U); + auto const levels = parse_object(current.body); + auto changed = cj::Object{}; + auto users = cj::Object{}; + for (auto const& member : levels) + { + if (member.key != "users") + { + changed.push_back(member); + continue; + } + if (auto const* held = std::get_if(&member.value->storage()); held != nullptr) + { + for (auto const& entry : *held) + { + if (entry.key != user) + { + users.push_back(entry); + } + } + } + } + users.push_back(cj::make_member(std::string{user}, cj::Value{level})); + changed.push_back(cj::make_member("users", cj::Value{std::move(users)})); + auto const body = cj::serialize_canonical(cj::Value{std::move(changed)}).output; + REQUIRE(request(fixture.runtime(), "PUT", + "/_matrix/client/v3/rooms/" + fixture.room_id + "/state/m.room.power_levels", fixture.alice_token, + body) + .status == 200U); +} + +[[nodiscard]] auto nul_text(std::string_view before, std::string_view after) -> std::string +{ + return std::string{before} + std::string(1U, '\0') + std::string{after}; +} + +[[nodiscard]] auto unique_sqlite_path() -> std::filesystem::path +{ + auto const now = std::chrono::steady_clock::now().time_since_epoch().count(); + return temporary_directory() / ("merovingian-delayed-events-" + std::to_string(now) + ".sqlite3"); +} + +} // namespace + +SCENARIO("A scheduled delayed event gets an opaque ID and shows every field on lookup", + "[homeserver][delayed-events][schedule][lookup]") +{ + GIVEN("alice in a room with bob") + { + auto fixture = Fixture{memory_config()}; + + WHEN("alice schedules a message event and a state event") + { + auto const message_id = scheduled_id(fixture, message_request(fixture)); + auto const state_id = scheduled_id(fixture, state_request(fixture)); + + THEN("each ID is short, URL-safe and unguessable, and the lookup shows the stored fields") + { + CHECK(message_id != state_id); + CHECK(message_id.size() >= 22U); + CHECK(message_id.size() <= 64U); + CHECK(message_id.find_first_not_of( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_") == std::string::npos); + + auto const message = looked_up(fixture, alice, message_id); + CHECK(*string_member(message, "delay_id") == message_id); + CHECK(*string_member(message, "room_id") == fixture.room_id); + CHECK(*string_member(message, "type") == "m.test.delayed"); + CHECK(string_member(message, "state_key") == nullptr); + CHECK(*int_member(message, "delay_ms") == 60'000); + CHECK(*int_member(message, "delayed_since_ts") == static_cast(t0)); + CHECK(*string_member(*object_member_as_object(message, "content"), "body") == "later"); + CHECK(object_member_as_object(message, "finalised") == nullptr); + + auto const state = looked_up(fixture, alice, state_id); + REQUIRE(string_member(state, "state_key") != nullptr); + CHECK(string_member(state, "state_key")->empty()); + CHECK(*string_member(state, "type") == "m.test.state"); + } + } + } +} + +SCENARIO("Another user's delay ID and an unknown one are the same answer for lookup and every action", + "[homeserver][delayed-events][manage][lookup][privacy]") +{ + GIVEN("alice's delayed event and bob asking about it") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture)); + + WHEN("bob looks it up and acts on it, and asks about IDs that do not exist") + { + auto const unknown = std::string{"no-such-delay-id"}; + auto const too_long = std::string(65U, 'x'); + + THEN("every answer is the one 404 M_NOT_FOUND, byte for byte") + { + auto const reference = lookup(fixture, bob, unknown); + CHECK_FALSE(reference.ok); + CHECK(reference.status == 404U); + CHECK(reference.errcode == "M_NOT_FOUND"); + CHECK(reference.reason == "Delayed event not found."); + auto const same = [&](hs::OperationResult const& other) { + return !other.ok && other.status == reference.status && other.errcode == reference.errcode && + other.reason == reference.reason && other.value == reference.value && + other.retry_after_ms == reference.retry_after_ms; + }; + CHECK(same(lookup(fixture, bob, id))); + CHECK(same(lookup(fixture, bob, {}))); + CHECK(same(lookup(fixture, bob, too_long))); + for (auto const action : + {hs::DelayedEventAction::send, hs::DelayedEventAction::cancel, hs::DelayedEventAction::restart}) + { + CHECK(same(manage(fixture, bob, id, action))); + CHECK(same(manage(fixture, bob, unknown, action))); + CHECK(same(manage(fixture, bob, {}, action))); + CHECK(same(manage(fixture, bob, too_long, action))); + } + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + } +} + +SCENARIO("A delayed event is never sent early, and is then sent once with its delay ID shown only to its sender", + "[homeserver][delayed-events][job][sync]") +{ + GIVEN("a message event due 60 s after it was scheduled and both users synced") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture)); + auto const alice_since = sync_for(fixture, fixture.alice_token, {}, "m.test.delayed").next_batch; + auto const bob_since = sync_for(fixture, fixture.bob_token, {}, "m.test.delayed").next_batch; + auto const before = fixture.event_count(); + auto const due = t0 + 60'000U; + + WHEN("the job runs one millisecond before the due time") + { + auto const wait = run_job(fixture, due - 1U); + + THEN("nothing is sent and the job asks to run again in one millisecond") + { + CHECK(fixture.event_count() == before); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + CHECK(wait == std::chrono::milliseconds{1}); + } + } + + WHEN("the job runs at the due time, then again") + { + auto const first_wait = run_job(fixture, due); + auto const second_wait = run_job(fixture, due + 5U); + + THEN("exactly one event is sent, and the room's members receive it") + { + CHECK(fixture.event_count() == before + 1U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(first_wait == std::chrono::milliseconds{60'000}); + CHECK(second_wait == std::chrono::milliseconds{60'000}); + + auto const finalised = *object_member_as_object(looked_up(fixture, alice, id), "finalised"); + auto const event_id = *string_member(finalised, "event_id"); + CHECK(*int_member(finalised, "finalised_ts") == static_cast(due)); + + auto const for_alice = sync_for(fixture, fixture.alice_token, alice_since, "m.test.delayed"); + REQUIRE(for_alice.event.has_value()); + CHECK(*string_member(*for_alice.event, "event_id") == event_id); + CHECK(*string_member(*for_alice.event, "sender") == alice); + auto const* alice_unsigned = object_member_as_object(*for_alice.event, "unsigned"); + REQUIRE(alice_unsigned != nullptr); + REQUIRE(string_member(*alice_unsigned, "delay_id") != nullptr); + CHECK(*string_member(*alice_unsigned, "delay_id") == id); + REQUIRE(string_member(*alice_unsigned, "org.matrix.msc4140.delay_id") != nullptr); + CHECK(*string_member(*alice_unsigned, "org.matrix.msc4140.delay_id") == id); + CHECK(string_member(*alice_unsigned, "transaction_id") == nullptr); + + auto const for_bob = sync_for(fixture, fixture.bob_token, bob_since, "m.test.delayed"); + REQUIRE(for_bob.event.has_value()); + CHECK(*string_member(*for_bob.event, "event_id") == event_id); + auto const* bob_unsigned = object_member_as_object(*for_bob.event, "unsigned"); + CHECK((bob_unsigned == nullptr || string_member(*bob_unsigned, "delay_id") == nullptr)); + CHECK((bob_unsigned == nullptr || + string_member(*bob_unsigned, "org.matrix.msc4140.delay_id") == nullptr)); + CHECK((bob_unsigned == nullptr || string_member(*bob_unsigned, "transaction_id") == nullptr)); + } + } + } +} + +SCENARIO("A delayed state event is sent as a state event and stamped with the time it is sent", + "[homeserver][delayed-events][job][timestamp]") +{ + GIVEN("a state event scheduled at t0 and an event clock that has since moved on") + { + auto fixture = Fixture{memory_config()}; + auto const state_id = scheduled_id(fixture, state_request(fixture, alice, 30'000U)); + auto const since = sync_for(fixture, fixture.bob_token, {}, "m.test.state").next_batch; + auto const sent_at = t0 + 123'456U; + fixture.runtime().homeserver.test_frozen_event_clock_ms = static_cast(sent_at); + + WHEN("the job runs after the due time") + { + std::ignore = run_job(fixture, t0 + 30'000U); + + THEN("the event is a state event whose origin_server_ts is the send time, not the schedule time") + { + auto const seen = sync_for(fixture, fixture.bob_token, since, "m.test.state"); + REQUIRE(seen.event.has_value()); + REQUIRE(string_member(*seen.event, "state_key") != nullptr); + CHECK(string_member(*seen.event, "state_key")->empty()); + CHECK(*int_member(*seen.event, "origin_server_ts") == static_cast(sent_at)); + CHECK(status_of(fixture, alice, state_id) == db::DelayedEventStatus::sent); + } + } + } +} + +SCENARIO("Restarting a delayed event moves its deadline, and every cell of the outcome table is answered", + "[homeserver][delayed-events][manage]") +{ + GIVEN("a scheduled message event") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture)); + auto const before = fixture.event_count(); + + WHEN("it is restarted 20 s later") + { + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 20'000U; + auto const restarted = manage(fixture, alice, id, hs::DelayedEventAction::restart); + + THEN("the old due time sends nothing and the new one sends") + { + CHECK(restarted.ok); + CHECK(restarted.value.empty()); + CHECK(*int_member(looked_up(fixture, alice, id), "delayed_since_ts") == + static_cast(t0 + 20'000U)); + std::ignore = run_job(fixture, t0 + 60'000U); + CHECK(fixture.event_count() == before); + std::ignore = run_job(fixture, t0 + 80'000U - 1U); + CHECK(fixture.event_count() == before); + std::ignore = run_job(fixture, t0 + 80'000U); + CHECK(fixture.event_count() == before + 1U); + } + } + + WHEN("it is cancelled") + { + auto const cancelled = manage(fixture, alice, id, hs::DelayedEventAction::cancel); + + THEN("it is finalised as cancelled with neither an event nor an error, and is never sent") + { + CHECK(cancelled.ok); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::cancelled); + auto const finalised = *object_member_as_object(looked_up(fixture, alice, id), "finalised"); + CHECK(string_member(finalised, "event_id") == nullptr); + CHECK(object_member_as_object(finalised, "error") == nullptr); + std::ignore = run_job(fixture, t0 + 10'000'000U); + CHECK(fixture.event_count() == before); + } + } + } + + GIVEN("one delayed event in each state of its life") + { + auto fixture = Fixture{memory_config()}; + auto const scheduled = scheduled_id(fixture, message_request(fixture, alice, "a", 10'000U)); + auto const sent = scheduled_id(fixture, message_request(fixture, alice, "b", 1'000U)); + auto const cancelled = scheduled_id(fixture, message_request(fixture, alice, "c")); + auto const failed = scheduled_id(fixture, message_request(fixture, bob, "d", 2'000U)); + auto const being_sent = scheduled_id(fixture, message_request(fixture, alice, "e")); + REQUIRE(manage(fixture, alice, cancelled, hs::DelayedEventAction::cancel).ok); + REQUIRE(request(fixture.runtime(), "POST", "/_matrix/client/v3/rooms/" + fixture.room_id + "/leave", + fixture.bob_token, "{}") + .status == 200U); + // `sent` is due first (1 s), then `failed` (bob has left, 2 s); `scheduled` (10 s) is not yet due. + std::ignore = run_job(fixture, t0 + 5'000U); + REQUIRE(status_of(fixture, alice, sent) == db::DelayedEventStatus::sent); + REQUIRE(status_of(fixture, bob, failed) == db::DelayedEventStatus::failed); + REQUIRE(fixture.table().mark_being_sent(being_sent)); + + using Action = hs::DelayedEventAction; + struct Expected final + { + std::string_view name; + std::string const* id; + std::string_view user; + Action action; + bool ok; + std::uint16_t status; + std::string_view reason; + }; + auto const sent_reason = std::string_view{"The delayed event has already been sent."}; + auto const cancelled_reason = std::string_view{"The delayed event has already been cancelled."}; + auto const being_reason = std::string_view{"The delayed event is being sent."}; + + THEN("the outcome table of the design holds cell by cell") + { + auto const cells = std::vector{ + {"sent restart", &sent, alice, Action::restart, false, 409U, sent_reason }, + {"sent cancel", &sent, alice, Action::cancel, false, 409U, sent_reason }, + {"sent send", &sent, alice, Action::send, true, 200U, {} }, + {"cancelled restart", &cancelled, alice, Action::restart, false, 409U, cancelled_reason}, + {"cancelled cancel", &cancelled, alice, Action::cancel, true, 200U, {} }, + {"cancelled send", &cancelled, alice, Action::send, false, 409U, cancelled_reason}, + {"failed restart", &failed, bob, Action::restart, false, 409U, cancelled_reason}, + {"failed cancel", &failed, bob, Action::cancel, true, 200U, {} }, + {"failed send", &failed, bob, Action::send, false, 409U, cancelled_reason}, + {"being sent restart", &being_sent, alice, Action::restart, false, 409U, being_reason }, + {"being sent cancel", &being_sent, alice, Action::cancel, false, 409U, being_reason }, + {"being sent send", &being_sent, alice, Action::send, false, 409U, being_reason }, + }; + for (auto const& cell : cells) + { + INFO(cell.name); + auto const result = manage(fixture, cell.user, *cell.id, cell.action); + CHECK(result.ok == cell.ok); + CHECK(result.status == cell.status); + if (!cell.ok) + { + CHECK(result.errcode == "M_UNKNOWN"); + CHECK(result.reason == cell.reason); + } + } + CHECK(status_of(fixture, alice, scheduled) == db::DelayedEventStatus::scheduled); + CHECK(status_of(fixture, alice, being_sent) == db::DelayedEventStatus::scheduled); + } + } +} + +SCENARIO("The send action sends now, and a refused send leaves the event scheduled with its timer unchanged", + "[homeserver][delayed-events][manage][send]") +{ + GIVEN("a scheduled message event") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture)); + auto const since = sync_for(fixture, fixture.bob_token, {}, "m.test.delayed").next_batch; + + WHEN("it is sent long before its time") + { + auto const result = manage(fixture, alice, id, hs::DelayedEventAction::send); + + THEN("the event reaches the room and the row is sent") + { + CHECK(result.ok); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(sync_for(fixture, fixture.bob_token, since, "m.test.delayed").event.has_value()); + } + } + + WHEN("the sender has left the room and then asks to send") + { + REQUIRE(request(fixture.runtime(), "POST", "/_matrix/client/v3/rooms/" + fixture.room_id + "/leave", + fixture.alice_token, "{}") + .status == 200U); + auto const before = fixture.event_count(); + auto const result = manage(fixture, alice, id, hs::DelayedEventAction::send); + + THEN("the answer is the send's own, the row is still scheduled and its timer is unchanged") + { + CHECK_FALSE(result.ok); + CHECK(result.status == 403U); + CHECK(result.reason == "not joined"); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + CHECK_FALSE(fixture.table().is_being_sent(id)); + CHECK(fixture.table().find(id)->delayed_since_ms == t0); + CHECK(fixture.event_count() == before); + } + } + } +} + +SCENARIO("A delayed event is authorised when it is sent, not when it was scheduled", + "[homeserver][delayed-events][job][authorisation]") +{ + GIVEN("bob holds the power to send a state event and schedules one and a message") + { + auto fixture = Fixture{memory_config()}; + set_power(fixture, bob, 50); + auto const state_id = scheduled_id(fixture, state_request(fixture, bob, 10'000U)); + auto const message_id = scheduled_id(fixture, message_request(fixture, bob, "m", 10'000U)); + + WHEN("his power is lowered and the due time passes") + { + set_power(fixture, bob, 0); + auto const before = fixture.event_count(); + std::ignore = run_job(fixture, t0 + 10'000U); + + THEN("the state event is not sent and fails with the room's refusal, and is not tried again") + { + CHECK(status_of(fixture, bob, state_id) == db::DelayedEventStatus::failed); + auto const error = *object_member_as_object( + *object_member_as_object(looked_up(fixture, bob, state_id), "finalised"), "error"); + CHECK(*string_member(error, "errcode") == "M_FORBIDDEN"); + CHECK(string_member(error, "error") != nullptr); + // The message needs no power, so exactly it was sent. + CHECK(status_of(fixture, bob, message_id) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 1U); + std::ignore = run_job(fixture, t0 + 20'000U); + CHECK(fixture.event_count() == before + 1U); + } + } + } + + GIVEN("bob schedules a message and then leaves the room") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, bob, "m", 10'000U)); + REQUIRE(request(fixture.runtime(), "POST", "/_matrix/client/v3/rooms/" + fixture.room_id + "/leave", + fixture.bob_token, "{}") + .status == 200U); + auto const before = fixture.event_count(); + + WHEN("the due time passes and the job runs twice") + { + std::ignore = run_job(fixture, t0 + 10'000U); + auto const after_first = fixture.event_count(); + std::ignore = run_job(fixture, t0 + 20'000U); + + THEN("nothing is sent, the row is failed with the error in the lookup, and no second attempt is made") + { + CHECK(after_first == before); + CHECK(fixture.event_count() == before); + CHECK(status_of(fixture, bob, id) == db::DelayedEventStatus::failed); + auto const error = *object_member_as_object( + *object_member_as_object(looked_up(fixture, bob, id), "finalised"), "error"); + CHECK(*string_member(error, "errcode") == "M_FORBIDDEN"); + CHECK(*string_member(error, "error") == "not joined"); + } + } + } +} + +SCENARIO("A suspended, locked or deactivated sender sends nothing when the event falls due", + "[homeserver][delayed-events][job][account]") +{ + GIVEN("bob's delayed event and an account in one of the three states") + { + auto const mode = GENERATE(0, 1, 2); + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, bob, "m", 10'000U)); + auto& homeserver = fixture.runtime().homeserver; + REQUIRE(hs::bootstrap_admin_user(homeserver, "admin", password).ok); + auto expected_errcode = std::string{}; + auto expected_status = std::uint16_t{0U}; + if (mode == 0) + { + REQUIRE(hs::set_local_user_moderation(homeserver, "@admin:example.org", bob, true, false, + "account.suspended", "suspended") + .ok); + expected_errcode = "M_USER_SUSPENDED"; + expected_status = 403U; + } + else if (mode == 1) + { + REQUIRE(hs::set_local_user_moderation(homeserver, "@admin:example.org", bob, false, true, "account.locked", + "locked") + .ok); + expected_errcode = "M_USER_LOCKED"; + expected_status = 401U; + } + else + { + for (auto& user : homeserver.database.users) + { + if (user.user_id == bob) + { + user.deactivated = true; + } + } + expected_errcode = "M_USER_DEACTIVATED"; + expected_status = 403U; + } + auto const before = fixture.event_count(); + + WHEN("the due time passes") + { + std::ignore = run_job(fixture, t0 + 10'000U); + + THEN("the event is failed with the account's errcode and nothing is sent") + { + CHECK(fixture.event_count() == before); + CHECK(status_of(fixture, bob, id) == db::DelayedEventStatus::failed); + auto const error = *object_member_as_object( + *object_member_as_object(looked_up(fixture, bob, id), "finalised"), "error"); + CHECK(*string_member(error, "errcode") == expected_errcode); + } + } + + WHEN("bob asks for the event to be sent now") + { + auto const result = manage(fixture, bob, id, hs::DelayedEventAction::send); + + THEN("the answer is the account refusal and the row stays scheduled") + { + CHECK_FALSE(result.ok); + CHECK(result.status == expected_status); + CHECK(result.errcode == expected_errcode); + CHECK(status_of(fixture, bob, id) == db::DelayedEventStatus::scheduled); + CHECK_FALSE(fixture.table().is_being_sent(id)); + } + } + } +} + +SCENARIO("The send-time rate limit fails a due event when the user's per-user policy for the route is spent", + "[homeserver][delayed-events][job][rate-limit]") +{ + GIVEN("a per-user policy of one send a minute for the send route, and two due events") + { + auto limits = merovingian::config::ClientRateLimitsConfig{}; + limits.per_user["/_matrix/client/v3/rooms/{roomId}/send/"] = merovingian::http::RateLimitPolicy{1U, 60U}; + auto fixture = Fixture{memory_config(std::move(limits))}; + auto const first = scheduled_id(fixture, message_request(fixture, alice, "a", 1'000U)); + auto const second = scheduled_id(fixture, message_request(fixture, alice, "b", 2'000U)); + + WHEN("the job runs when both are due") + { + std::ignore = run_job(fixture, t0 + 5'000U); + + THEN("the first is sent and the second fails with M_LIMIT_EXCEEDED") + { + CHECK(status_of(fixture, alice, first) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, second) == db::DelayedEventStatus::failed); + auto const error = *object_member_as_object( + *object_member_as_object(looked_up(fixture, alice, second), "finalised"), "error"); + CHECK(*string_member(error, "errcode") == "M_LIMIT_EXCEEDED"); + } + } + } + + GIVEN("no per-user policy and the same two due events") + { + auto fixture = Fixture{memory_config()}; + auto const first = scheduled_id(fixture, message_request(fixture, alice, "a", 1'000U)); + auto const second = scheduled_id(fixture, message_request(fixture, alice, "b", 2'000U)); + + WHEN("the job runs when both are due") + { + std::ignore = run_job(fixture, t0 + 5'000U); + + THEN("both are sent") + { + CHECK(status_of(fixture, alice, first) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, second) == db::DelayedEventStatus::sent); + } + } + } +} + +SCENARIO("A user holds at most the configured number of scheduled events, and is told when to retry", + "[homeserver][delayed-events][schedule][cap]") +{ + GIVEN("a cap of two and two events scheduled 500 ms apart") + { + auto fixture = Fixture{memory_config()}; + fixture.runtime().limits.max_scheduled_delayed_events_per_user = 2U; + std::ignore = scheduled_id(fixture, message_request(fixture, alice, "a", 60'000U)); + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 500U; + std::ignore = scheduled_id(fixture, message_request(fixture, alice, "b", 30'000U)); + // The earliest due time is t0 + 30'500; at t0 + 1'200 that is 29'300 ms away. + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 1'200U; + + WHEN("alice asks for a third, and bob for his first") + { + auto const refused = schedule(fixture, message_request(fixture, alice, "c")); + auto const other = schedule(fixture, message_request(fixture, bob, "x")); + + THEN("alice gets 429 with the wait rounded up to a whole second, and bob is unaffected") + { + CHECK_FALSE(refused.ok); + CHECK(refused.status == 429U); + CHECK(refused.errcode == "M_LIMIT_EXCEEDED"); + CHECK(refused.reason == "The maximum number of delayed events has been reached."); + CHECK(refused.retry_after_ms == 30'000U); + CHECK(fixture.table().scheduled_count(alice) == 2U); + CHECK(other.ok); + } + } + + WHEN("one of alice's events is cancelled") + { + auto const held = fixture.table().earliest_due_ms_of_user(alice); + REQUIRE(held.has_value()); + auto const cancelled = + manage(fixture, alice, fixture.table().earliest_due()->delay_id, hs::DelayedEventAction::cancel); + auto const third = schedule(fixture, message_request(fixture, alice, "c")); + + THEN("a finalised event does not count against the cap") + { + CHECK(cancelled.ok); + CHECK(third.ok); + } + } + } +} + +SCENARIO("The delay is bounded by the configured maximum, with the errcode of the route's prefix", + "[homeserver][delayed-events][schedule][limits]") +{ + GIVEN("a maximum of one hour") + { + auto fixture = Fixture{memory_config()}; + fixture.runtime().limits.max_delayed_event_delay_ms = 3'600'000U; + + WHEN("one millisecond more than the maximum is requested on each prefix, and the maximum itself") + { + auto stable = message_request(fixture, alice, "s", 3'600'001U); + stable.stable_errcode = true; + auto const unstable = message_request(fixture, alice, "u", 3'600'001U); + auto const at_maximum = message_request(fixture, alice, "m", 3'600'000U); + auto const stable_result = schedule(fixture, stable); + auto const unstable_result = schedule(fixture, unstable); + auto const maximum_result = schedule(fixture, at_maximum); + + THEN("both are 400 with their own errcode and the maximum is accepted") + { + CHECK(stable_result.status == 400U); + CHECK(stable_result.errcode == "M_DELAY_TOO_LARGE"); + CHECK(stable_result.reason == "The requested delay exceeds the allowed maximum."); + CHECK(unstable_result.status == 400U); + CHECK(unstable_result.errcode == "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE"); + CHECK(maximum_result.ok); + CHECK(fixture.table().size() == 1U); + } + } + } +} + +SCENARIO("With the feature disabled nothing new is scheduled, but what was scheduled is still sent and managed", + "[homeserver][delayed-events][schedule][disabled]") +{ + GIVEN("an event scheduled before the feature is turned off") + { + auto fixture = Fixture{memory_config()}; + auto const kept = scheduled_id(fixture, message_request(fixture, alice, "a", 10'000U)); + auto const cancelled = scheduled_id(fixture, message_request(fixture, alice, "b")); + fixture.runtime().limits.delayed_events_enabled = false; + auto const before = fixture.event_count(); + + WHEN("a new event is scheduled and the old ones are managed and sent") + { + auto const refused = schedule(fixture, message_request(fixture, alice, "c")); + auto const cancel = manage(fixture, alice, cancelled, hs::DelayedEventAction::cancel); + std::ignore = run_job(fixture, t0 + 10'000U); + + THEN("the schedule is 403 M_FORBIDDEN, the cancel works and the due event is sent") + { + CHECK(refused.status == 403U); + CHECK(refused.errcode == "M_FORBIDDEN"); + CHECK(refused.reason == "Delayed events are not enabled on this homeserver."); + CHECK(cancel.ok); + CHECK(status_of(fixture, alice, kept) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 1U); + CHECK(fixture.table().size() == 2U); + } + } + } +} + +SCENARIO("Scheduling the same transaction again returns the same delay ID and stores one row", + "[homeserver][delayed-events][schedule][idempotency]") +{ + GIVEN("an event scheduled with a transaction ID") + { + auto fixture = Fixture{memory_config()}; + auto const first = scheduled_id(fixture, message_request(fixture, alice, "txn-1")); + + WHEN("the same request is repeated, from another device, and with an empty transaction ID twice") + { + auto const again = schedule(fixture, message_request(fixture, alice, "txn-1")); + auto other_device = message_request(fixture, alice, "txn-1"); + other_device.device_id = "D9"; + auto const elsewhere = schedule(fixture, other_device); + auto const empty_one = schedule(fixture, state_request(fixture)); + auto const empty_two = schedule(fixture, state_request(fixture)); + + THEN("the repeat finds the first, another device stores a second, and an empty ID never matches") + { + CHECK(again.ok); + CHECK(again.value == first); + CHECK(elsewhere.ok); + CHECK(elsewhere.value != first); + CHECK(empty_one.ok); + CHECK(empty_two.ok); + CHECK(empty_one.value != empty_two.value); + CHECK(fixture.table().size() == 4U); + } + } + } +} + +SCENARIO("A delayed event the sender could not send now is refused when it is scheduled, as the ordinary routes refuse", + "[homeserver][delayed-events][schedule][refusal]") +{ + GIVEN("a room, a user who is not in it and a user without the power to send state") + { + auto fixture = Fixture{memory_config()}; + auto const carol_token = sign_in(fixture.runtime(), "carol", "D3"); + auto const unknown_room = std::string{"!nowhere:example.org"}; + + WHEN("each is scheduled and the same event is sent through the ordinary route") + { + auto unknown = message_request(fixture); + unknown.room_id = unknown_room; + auto const unknown_result = schedule(fixture, unknown); + auto const unknown_ordinary = request( + fixture.runtime(), "PUT", "/_matrix/client/v3/rooms/" + unknown_room + "/send/m.test.delayed/t1", + fixture.alice_token, R"({"body":"x"})"); + auto const stranger_result = schedule(fixture, message_request(fixture, "@carol:example.org")); + auto const stranger_ordinary = request( + fixture.runtime(), "PUT", "/_matrix/client/v3/rooms/" + fixture.room_id + "/send/m.test.delayed/t1", + carol_token, R"({"body":"x"})"); + auto const weak_result = schedule(fixture, state_request(fixture, bob)); + auto const weak_ordinary = + request(fixture.runtime(), "PUT", "/_matrix/client/v3/rooms/" + fixture.room_id + "/state/m.test.state", + fixture.bob_token, R"({"v":1})"); + + THEN("the status matches the ordinary route's, and nothing is stored") + { + CHECK_FALSE(unknown_result.ok); + CHECK(unknown_result.status == unknown_ordinary.status); + CHECK_FALSE(stranger_result.ok); + CHECK(stranger_result.status == stranger_ordinary.status); + CHECK(stranger_result.status == 403U); + CHECK_FALSE(weak_result.ok); + CHECK(weak_result.status == weak_ordinary.status); + CHECK(weak_result.status == 403U); + CHECK(fixture.table().size() == 0U); + } + } + + WHEN("an oversized type, transaction ID or content, or a NUL byte, is scheduled") + { + auto long_type = message_request(fixture); + long_type.event_type = std::string(256U, 't'); + auto long_txn = message_request(fixture, alice, std::string(256U, 'x')); + auto big_content = message_request(fixture); + big_content.content_json = R"({"body":")" + std::string(65'536U, 'c') + R"("})"; + auto nul_type = message_request(fixture); + nul_type.event_type = nul_text("m.test", "x"); + auto nul_txn = message_request(fixture, alice, nul_text("a", "b")); + auto nul_key = state_request(fixture); + nul_key.state_key = nul_text("a", "b"); + auto long_key = state_request(fixture); + long_key.state_key = std::string(256U, 'k'); + auto no_delay = message_request(fixture, alice, "z", 0U); + auto not_object = message_request(fixture); + not_object.content_json = "[1]"; + + THEN("each gets the ordinary answer and nothing is stored") + { + auto const type_result = schedule(fixture, long_type); + CHECK(type_result.status == 413U); + CHECK(type_result.errcode == "M_TOO_LARGE"); + CHECK(type_result.reason == "the event type is at most 255 bytes"); + auto const txn_result = schedule(fixture, long_txn); + CHECK(txn_result.status == 400U); + CHECK(txn_result.errcode == "M_INVALID_PARAM"); + CHECK(txn_result.reason == "the transaction ID is at most 255 bytes"); + auto const content_result = schedule(fixture, big_content); + CHECK(content_result.status == 413U); + CHECK(content_result.errcode == "M_TOO_LARGE"); + for (auto const* refused : {&nul_type, &nul_txn, &nul_key, &no_delay}) + { + auto const result = schedule(fixture, *refused); + CHECK(result.status == 400U); + CHECK(result.errcode == "M_INVALID_PARAM"); + } + CHECK(schedule(fixture, long_key).status == 413U); + CHECK(schedule(fixture, not_object).status == 400U); + CHECK(fixture.table().size() == 0U); + } + } + } +} + +SCENARIO("A delayed event survives a restart, and one that fell due while the server was down is sent first", + "[homeserver][delayed-events][persistence]") +{ + GIVEN("an event scheduled on a SQLite file and a runtime started again from the same file") + { + auto const path = unique_sqlite_path(); + std::filesystem::remove(path); + auto const config = sqlite_config(path); + auto const overdue = GENERATE(false, true); + auto id = std::string{}; + auto room_id = std::string{}; + { + auto first = Fixture{config}; + id = scheduled_id(first, message_request(first, alice, "txn-1", 30'000U)); + room_id = first.room_id; + } + auto restarted = hs::start_client_server(config); + REQUIRE(restarted.started); + auto& homeserver = restarted.runtime.homeserver; + homeserver.test_frozen_event_clock_ms = static_cast(t0); + + WHEN("the job runs just before the old deadline and then at it, or long after it") + { + auto const before = homeserver.database.persistent_store.events.size(); + auto const early = hs::send_due_delayed_events(restarted.runtime, t0 + 30'000U - 1U, {}); + auto const early_count = homeserver.database.persistent_store.events.size(); + auto const late_now = overdue ? t0 + 10'000'000U : t0 + 30'000U; + auto const late = hs::send_due_delayed_events(restarted.runtime, late_now, {}); + + THEN("the event kept its deadline, is not early and is sent by the first run that is due") + { + auto const* held = homeserver.database.persistent_store.delayed_events.find_owned(alice, id); + REQUIRE(held != nullptr); + CHECK(held->delayed_since_ms == t0); + CHECK(held->delay_ms == 30'000U); + CHECK(held->room_id == room_id); + CHECK(held->status == db::DelayedEventStatus::sent); + CHECK(early_count == before); + CHECK(homeserver.database.persistent_store.events.size() == before + 1U); + CHECK(early == std::chrono::milliseconds{1}); + CHECK(late == std::chrono::milliseconds{60'000}); + } + } + + std::filesystem::remove(path); + } +} + +SCENARIO("A failed outcome commit leaves the event sent once and never sent again", + "[homeserver][delayed-events][job][at-least-once]") +{ + GIVEN("a due event whose finalising write fails") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "txn-1", 10'000U)); + auto const before = fixture.event_count(); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "finalise_delayed_event"); + + WHEN("the job runs twice") + { + std::ignore = run_job(fixture, t0 + 10'000U); + std::ignore = run_job(fixture, t0 + 20'000U); + + THEN("the event was sent once and memory says it was sent") + { + CHECK(fixture.event_count() == before + 1U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK_FALSE(fixture.table().is_being_sent(id)); + } + } + } +} + +SCENARIO("Finalised events are kept per user up to a bound and for seven days", + "[homeserver][delayed-events][job][retention]") +{ + GIVEN("a bound of two finalised events per user") + { + auto fixture = Fixture{memory_config()}; + fixture.runtime().limits.max_finalised_delayed_events_per_user = 2U; + + WHEN("three of alice's events are finalised") + { + auto const one = scheduled_id(fixture, message_request(fixture, alice, "1")); + auto const two = scheduled_id(fixture, message_request(fixture, alice, "2")); + auto const three = scheduled_id(fixture, message_request(fixture, alice, "3")); + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 1U; + REQUIRE(manage(fixture, alice, one, hs::DelayedEventAction::cancel).ok); + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 2U; + REQUIRE(manage(fixture, alice, two, hs::DelayedEventAction::cancel).ok); + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 3U; + REQUIRE(manage(fixture, alice, three, hs::DelayedEventAction::cancel).ok); + + THEN("the oldest is gone and answers 404, the others remain") + { + CHECK(fixture.table().find_owned(alice, one) == nullptr); + CHECK(fixture.table().find_owned(alice, two) != nullptr); + CHECK(fixture.table().find_owned(alice, three) != nullptr); + CHECK(manage(fixture, alice, one, hs::DelayedEventAction::cancel).status == 404U); + } + } + } + + GIVEN("a cancelled event finalised at t0 and one finalised a day later") + { + auto fixture = Fixture{memory_config()}; + auto const old_id = scheduled_id(fixture, message_request(fixture, alice, "old")); + REQUIRE(manage(fixture, alice, old_id, hs::DelayedEventAction::cancel).ok); + auto constexpr day = std::uint64_t{24U * 60U * 60U * 1000U}; + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + day; + auto const new_id = scheduled_id(fixture, message_request(fixture, alice, "new")); + REQUIRE(manage(fixture, alice, new_id, hs::DelayedEventAction::cancel).ok); + + WHEN("the job runs just under seven days after the first, then at seven days and one millisecond") + { + std::ignore = run_job(fixture, t0 + 7U * day); + auto const kept_at_seven_days = fixture.table().find_owned(alice, old_id) != nullptr; + std::ignore = run_job(fixture, t0 + 7U * day + 1U); + + THEN("the older row goes only once it is past seven days and the newer one stays") + { + CHECK(kept_at_seven_days); + CHECK(fixture.table().find_owned(alice, old_id) == nullptr); + CHECK(fixture.table().find_owned(alice, new_id) != nullptr); + } + } + } +} + +SCENARIO("Deactivating an account deletes every delayed event of that user and nobody else's", + "[homeserver][delayed-events][deactivation]") +{ + GIVEN("scheduled and finalised delayed events of alice and one of bob") + { + auto fixture = Fixture{memory_config()}; + auto const live = scheduled_id(fixture, message_request(fixture, alice, "a")); + auto const done = scheduled_id(fixture, message_request(fixture, alice, "b")); + REQUIRE(manage(fixture, alice, done, hs::DelayedEventAction::cancel).ok); + auto const bobs = scheduled_id(fixture, message_request(fixture, bob, "c")); + + WHEN("alice's account is deactivated") + { + auto const guard = hs::RuntimeLock{fixture.runtime().homeserver.mutex}; + auto const result = hs::deactivate_user_account(fixture.runtime().homeserver, alice, + {std::string{alice}, std::string{alice}, "test"}); + + THEN("none of her rows remain and bob's is untouched") + { + CHECK(result.ok); + CHECK(fixture.table().find(live) == nullptr); + CHECK(fixture.table().find(done) == nullptr); + CHECK(fixture.table().find(bobs) != nullptr); + CHECK(fixture.table().size() == 1U); + } + } + } +} + +SCENARIO("The job's return value says when it should run next, and it stops when asked", + "[homeserver][delayed-events][job][pacing]") +{ + GIVEN("a runtime with nothing scheduled") + { + auto fixture = Fixture{memory_config()}; + + WHEN("the job runs") + { + auto const wait = run_job(fixture, t0); + + THEN("it asks for 60 seconds") + { + CHECK(wait == std::chrono::milliseconds{60'000}); + } + } + + WHEN("an event due in 12.5 s and one due in two minutes are scheduled") + { + std::ignore = scheduled_id(fixture, message_request(fixture, alice, "a", 12'500U)); + std::ignore = scheduled_id(fixture, message_request(fixture, alice, "b", 120'000U)); + auto const near = run_job(fixture, t0 + 500U); + auto const far = run_job(fixture, t0 - 1U); + REQUIRE( + manage(fixture, alice, fixture.table().earliest_due()->delay_id, hs::DelayedEventAction::cancel).ok); + auto const capped = run_job(fixture, t0); + + THEN("the wait is the exact time to the next deadline, capped at 60 seconds") + { + CHECK(near == std::chrono::milliseconds{12'000}); + CHECK(far == std::chrono::milliseconds{12'501}); + CHECK(capped == std::chrono::milliseconds{60'000}); + } + } + } + + GIVEN("102 events due at once") + { + auto fixture = Fixture{memory_config()}; + fixture.runtime().limits.max_scheduled_delayed_events_per_user = 1000U; + for (auto index = 0; index < 102; ++index) + { + std::ignore = scheduled_id(fixture, message_request(fixture, alice, std::to_string(index), 1'000U)); + } + auto const before = fixture.event_count(); + + WHEN("the job runs once") + { + auto const wait = run_job(fixture, t0 + 1'000U); + + THEN("it sent 100 and asks to run again at once, and the next run sends the other two") + { + CHECK(fixture.event_count() == before + 100U); + CHECK(wait == std::chrono::milliseconds{0}); + auto const second = run_job(fixture, t0 + 1'000U); + CHECK(fixture.event_count() == before + 102U); + CHECK(second == std::chrono::milliseconds{60'000}); + } + } + + WHEN("the job runs with its stop flag already set") + { + auto stop = hs::StopFlag{}; + stop.request_stop(); + std::ignore = run_job(fixture, t0 + 1'000U, stop); + + THEN("nothing is sent") + { + CHECK(fixture.event_count() == before); + CHECK(fixture.table().scheduled_count(alice) == 102U); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index 319f2e17a..3623b182b 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -1005,6 +1005,7 @@ shell = find_program('sh') 'integration/test_filter_list_cap_flow.cpp', 'integration/test_room_read_examination_budget_flow.cpp', 'integration/test_stored_invite_caps_flow.cpp', + 'integration/test_delayed_events_flow.cpp', 'integration/test_security_issues_460_463_flow.cpp', 'integration/test_server_signing_key_lifecycle_flow.cpp', 'integration/test_sync_dag_gap_flow.cpp', From 64ac82faf9a530833f98403ffa76de9eecefbbee Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 02:51:33 +0000 Subject: [PATCH 10/29] Document delayed events in the changelog, API reference, threat model, capability gaps and logging docs Corrects ADR-0823's account of how Element Call's pinned matrix-js-sdk schedules. Co-Authored-By: Claude Fable 5.2 --- .../references/module-invariants.md | 8 ++++++ CHANGELOG.md | 16 +++++++++--- ...ameter-forms-that-released-clients-send.md | 3 ++- docs/log-filtering.md | 1 + docs/matrix-v1.19-client-server-api.md | 1 + docs/observability-audit.md | 9 +++++++ docs/threat-model.md | 26 +++++++++++++++++++ docs/todos/capability-gaps.md | 3 ++- 8 files changed, 62 insertions(+), 5 deletions(-) diff --git a/.agent/skills/merovingian-review/references/module-invariants.md b/.agent/skills/merovingian-review/references/module-invariants.md index b5a26f86f..60ea15792 100644 --- a/.agent/skills/merovingian-review/references/module-invariants.md +++ b/.agent/skills/merovingian-review/references/module-invariants.md @@ -112,6 +112,9 @@ look at every time. Read the `AGENTS.md` itself before judging a change to that an OAuth client that has a session (ADR-0776, ADR-0777). Whatever replaces a device's credentials replaces its `oauth_sessions` row in the same transaction; a grant's audit row is in its transaction (ADR-0792). `src/database/AGENTS.md#OAuth 2.0 tables (migration 002, ADR-0776, ADR-0777)` +- A finalised-row bound and the 7-day prune apply to every `delayed_events` row; only `DelayedEventTable` touches the + rows, and the in-memory mirror changes only after the commit (one exception, ADR-0825). A path that deletes an + account's data deletes these rows too (ADR-0824). ## src/email @@ -197,6 +200,10 @@ look at every time. Read the `AGENTS.md` itself before judging a change to that API's own function; a password change, reset or deactivation calls `end_account_page_access`; a taken approval is spent only after the keys are stored (ADR-0796 to ADR-0798). `src/homeserver/AGENTS.md#OAuth 2.0 API (ADR-0812, ADR-0773)` +- A due delayed event is sent only by the `delayed-events` job or the `send` action, under the runtime lock, through + `send_event_as_user`, never before `delayed_since_ms + delay_ms`, after the account checks and the send-time rate + limit, with the "being sent" mark set across the send; a request carrying `org.matrix.msc4140.delay` never reaches an + immediate send; the content and state key are never logged (ADR-0823, ADR-0825, ADR-0826). - Worker frames are untrusted. Relayed PDUs are re-verified with main's resolver **before** taking `runtime.mutex`, and the envelope is rebuilt from the verified event (ADR-0171). `#Federation worker relays are untrusted input` @@ -339,6 +346,7 @@ look at every time. Read the `AGENTS.md` itself before judging a change to that equals the generated step (ADR-0778). `migrations/AGENTS.md#Migration 002: the OAuth 2.0 tables (1.1.0, ADR-0776, ADR-0778)` - Next number is the maximum plus one, contiguous. Check that parallel branches have not taken it. Format and header follow `#File format`. +- Migration 003 adds `delayed_events` the same way, in its own per-version array (ADR-0778, ADR-0824). - Every new table is classified for the federation worker. `#Federation worker classification` - `NOT NULL` additions need a `DEFAULT`. No drops without approval. Update `docs/database-persistence.md`. diff --git a/CHANGELOG.md b/CHANGELOG.md index a6745a339..6316aaaa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,16 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Features -* Delayed events (MSC4140): a client can schedule an event to be sent after a delay, and restart, cancel or send it, - so a MatrixRTC call membership ends when its client vanishes. +* Delayed events (MSC4140): schedule, restart, cancel, send and look up one, on the v1.20 paths, the + `org.matrix.msc4140` paths and `?org.matrix.msc4140.delay=` on `/send` and `/state` (ADR-0823). +* `/versions` and `/capabilities` advertise delayed events (`org.matrix.msc4140`, `m.delayed_events`) + (ADR-0823). +* A due delayed event is sent by the server as its user through the ordinary send path: authorised when sent, + never early, kept across a restart (ADR-0825). +* Migration 003 adds the `delayed_events` table; finalised events are kept per user up to a bound and 7 days + (ADR-0824). +* `server.delayed_events.enabled`, `max_delay_ms`, `max_scheduled_per_user` and `max_finalised_per_user`, with + built-in per-user rate limits (ADR-0826). ## Security Fixes @@ -20,7 +28,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Release notes -Delayed events for MatrixRTC, on top of 1.1.1. Upgrading runs migration 003, which adds one table. +Delayed events for MatrixRTC, on top of 1.1.1. Upgrading runs migration 003, which adds one table, and a downgrade +drops it with whatever is scheduled. The feature is on by default and `server.delayed_events.enabled = false` turns it +off. Operators running Element Call keep `max_delay_ms` at 3600000 or more. # Version 1.1.1 diff --git a/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md b/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md index 0fd2d0ebb..5c88e7883 100644 --- a/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md +++ b/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md @@ -19,7 +19,8 @@ The released clients, read at their release tags on 2026-11-17, do not send the * Element Web 1.12.35 (matrix-js-sdk 45.1.0) tries the unstable dedicated path `.../org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}` first and falls back to the query form on `M_UNRECOGNIZED`. It manages with `POST .../org.matrix.msc4140/delayed_events/{delayId}/{send|cancel|restart}`. - Element Call 0.28.2 pins matrix-js-sdk commit `3f05dc4d`, which does the same. + Element Call 0.28.2 pins matrix-js-sdk commit `3f05dc4d`, which schedules with the query form and manages with the + same path-form `POST`. * lk-jwt-service 0.10.0 manages a delegated leave as an application service acting for the user, on the same unstable management path. * Every client reads only the `org.matrix.msc4140` flag in `unstable_features`; matrix-js-sdk refuses to use the diff --git a/docs/log-filtering.md b/docs/log-filtering.md index 2176d309e..41969030d 100644 --- a/docs/log-filtering.md +++ b/docs/log-filtering.md @@ -133,6 +133,7 @@ used as the first argument to `observability::log_diagnostic` and `log_diagnosti | `stream_token_epoch` | The database's sync token epoch: created at first start, replaced by `--after-restore`, or refused | | `tls` | TLS context set-up and handshake accept or reject | | `oauth` | OAuth 2.0 client registration, eviction at the cap, and stored metadata that no longer validates | +| `delayed_events` | Delayed events: scheduled, restarted, cancelled, sent, failed, and a failed outcome write; never event content or a state key | Unrecognised module names are accepted and the level is recorded — there is no error. Restart the server to apply. diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index f54740812..167f465ec 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -540,6 +540,7 @@ In each case below the spec leaves the choice to the server; the owner decided t - **`POST /users/{userId}/report` (v1.14, ADR-0482).** Needs an access token and is rate-limited. The body is a JSON object whose `reason`, when present, is a string (`400 M_BAD_JSON` otherwise). A local user ID no account holds is `404 M_NOT_FOUND` and stores nothing; a user of another server is accepted without a lookup. It is stored as an audit row `trust_safety.account.accept_report` (reporter, reported user), one per pair, listed by `GET /admin/safety/reports`; a repeat answers `200` without a second row. The reporter need share no room with the reported user. - **Password reset by email (ADR-0481).** `POST /account/password/email/requestToken` (no token) answers `200` with `sid` and `submit_url` for every well-formed request, whether or not an account holds the address, and emails an 8-character code only to an active account's address; it never answers `M_THREEPID_NOT_FOUND` or `M_THREEPID_IN_USE`. A repeat with the same or a lower `send_attempt` returns the same `sid` and sends nothing. With no relay (`server.email.smtp_url`) it is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`. The client posts `{sid, client_secret, token}` to `submit_url` (`/_matrix/client/unstable/org.merovingian/account/password/email/submitToken`, this server's own path, the Identity Service `submitToken` shape): `{"success":true}`, or `400 M_TOKEN_INCORRECT`, `M_INVALID_PARAM` or `M_SESSION_EXPIRED` (five wrong codes end the session; sessions last 15 minutes). `POST /account/password` with no access token and `auth: {type: "m.login.email.identity", threepid_creds: {sid, client_secret}}` then sets the password of the account that holds the address (`401` with the flows for a missing or unvalidated stage, `400 M_WEAK_PASSWORD` for a weak password and the session is kept for a retry) and, with `logout_devices` (the default), revokes every token of the account. `POST /account/password/msisdn/requestToken` is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED` (no SMS). - **MatrixRTC transport discovery (MSC4143, not in v1.19; ADR-0822).** `GET /_matrix/client/v1/rtc/transports` answers `{"transports":[{"type":"livekit","livekit_service_url":""}]}` and `GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports` answers `{"rtc_transports":[...]}`, both needing an access token; with `server.rtc.livekit.service_url` empty they answer `{"transports":[]}` and `{}`. `/versions` always lists `org.matrix.msc4143` and `org.matrix.msc4143.stable`, and the well-known document gains `org.matrix.msc4143.rtc_foci` when a URL is set. There is no `m.livekit` transport and no `/rtc/livekit/*` route. +- **Delayed events (MSC4140; in the v1.20 spec text, not in v1.19; ADR-0823 to ADR-0826).** All nine routes need an access token and the standard account gate; `?user_id=` masquerade works as on other routes. Schedule: `PUT /v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}` and `PUT /unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}`; a request to `PUT /v3/rooms/{roomId}/send/{eventType}/{txnId}` or `PUT /v3/rooms/{roomId}/state/{eventType}/{stateKey}` (also the no-state-key and empty-state-key forms) that carries `?org.matrix.msc4140.delay=` is always scheduled, never sent at once, and answers `403 M_FORBIDDEN` when the feature is disabled; a delay that is not a decimal integer of at least 1, or is repeated, is `400 M_INVALID_PARAM`, and one over `server.delayed_events.max_delay_ms` is `400` (`M_DELAY_TOO_LARGE` on the v3 dedicated path, `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE` on the others); a user at `max_scheduled_per_user` gets `429 M_LIMIT_EXCEEDED`; content over 65,536 bytes is `413 M_TOO_LARGE`. Manage: `POST /v1/delayed_events/{delayId}/{send|cancel|restart}` and `POST /unstable/org.matrix.msc4140/delayed_events/{delayId}/{send|cancel|restart}`, and, under the unstable prefix only, `POST /unstable/org.matrix.msc4140/delayed_events/{delayId}` with `{"action":"send"|"cancel"|"restart"}` (there is no `POST /v1/delayed_events/{delayId}`; any other last segment is `404 M_UNRECOGNIZED`). Look up: `GET /v1/delayed_events/{delayId}` and `GET /unstable/org.matrix.msc4140/delayed_events/{delayId}`. A delay ID that is unknown and one that belongs to another user both answer `404 M_NOT_FOUND` with `Delayed event not found.`; a repeated action on a finalised event answers `200` or `409 M_UNKNOWN`, and one on an event being sent answers `409`. `/versions` lists `org.matrix.msc4140` and `org.matrix.msc4140.stable` (each true when the feature is enabled), and `/capabilities` lists `m.delayed_events` and `org.matrix.msc4140.delayed_events` (`max_delay_ms`, `max_scheduled`; zeros when disabled). The event a delayed event became carries `unsigned.delay_id` for the user who scheduled it. The listing endpoint (MSC4486) is not routed (`404 M_UNRECOGNIZED`); sticky events (MSC4354) are not supported. - **`GET /voip/turnServer` with `server.turn.uris` (ADR-0821)** answers one `uris` entry per configured URI, in the configured order; `server.turn.server` still gives one. - **`GET /voip/turnServer` with `server.turn.shared_secret_file` (ADR-0480)** answers per-user credentials in the coturn REST API format: `username` `:`, `password` the base64 HMAC-SHA1 of it under the secret, `ttl` `server.turn.ttl_seconds`. - **`GET /_merovingian/health` (ADR-0479)** is not a Matrix endpoint: an unauthenticated probe, `200 ok` or `503 unavailable`. diff --git a/docs/observability-audit.md b/docs/observability-audit.md index 8dc5c1a59..1138da4ea 100644 --- a/docs/observability-audit.md +++ b/docs/observability-audit.md @@ -627,6 +627,15 @@ template, no ID and no query) and `type` (the exception's mangled type name, `un The exception's message is never logged: it can quote request data. A `std::overflow_error` from the sync stream counter keeps its own answer and is not logged this way. +## Delayed event diagnostics (ADR-0823 to ADR-0826) + +The delayed-events handlers and the `delayed-events` job log under the `delayed_events` module, each with the fields +`delay_id`, `user_id`, `room_id` and `event_type`, never the content and never the state key (a call membership's state +key holds a device ID): `delayed_event.scheduled` (info, with `delay_ms`), `delayed_event.restarted` (info), +`delayed_event.cancelled` (info), `delayed_event.sent` (info, with `event_id`), `delayed_event.failed` (warning, with +`status` and `errcode`) and `delayed_event.finalise_commit_failed` (error, with `outcome`, `refused` or `commit_failed`: +the event was sent but its row could not be recorded, so it is sent again after a restart). + ## Registration token file diagnostics When `security.registration.token_file` is set but yields no token hash, every `POST /register` that needs the token diff --git a/docs/threat-model.md b/docs/threat-model.md index 1d8e45e31..55f112704 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -3337,3 +3337,29 @@ is an absolute `https` URL with a host, no userinfo, query or fragment, no white every body is built with the JSON serializer, never by concatenating the URL into a string; and the server never fetches the URL, so it adds no outbound request and no SSRF path. Residual risk: an operator who points it at a service they do not control hands that service the OpenID tokens clients send it. + +### Delayed events (1.1.2, ADR-0823 to ADR-0826) + +New surface: nine routes (schedule, restart, cancel, send and look up, on the v1.20, `org.matrix.msc4140` and query +forms), which need an access token; a table, `delayed_events`, of client-supplied event content held before it is +sent; a background thread in the main process, the `delayed-events` job, that sends events with no request, a stored, +deferred, server-initiated action; and four configuration keys, `server.delayed_events.*`. Threats and mitigations: +(a) acting after losing the right to (left the room, power level lowered, account suspended, locked or deactivated): the +event is authorised against the room state at send time through the ordinary send path, the job re-checks the account, +and deactivation deletes the user's rows; (b) reading or driving another user's delayed event: the ID is 128 random bits, +an identifier and never a credential, every use is authenticated, the lookup is by (user, ID), and an unknown and a +foreign ID get an identical `404 M_NOT_FOUND`; (c) exhausting storage or memory: a per-user cap on scheduled events +(the new one is refused, nothing is evicted), the 65,536-byte content limit, bounded retention of finalised rows +(`max_finalised_per_user` and 7 days) and `max_delay_ms`; (d) exhausting the runtime lock or the room with a burst of +due events: one send per lock hold, at most 100 per run, the scheduling and management rate limits and a per-user +send-time rate limit; (e) write load from restarts: each restart is one small write, bounded by the per-user management +rate limit; (f) an event sent early or twice: never early by construction, since the deadline is compared under the lock +at send time, and at least once by decision, so a crash between the send and its record sends it once more after +restart; (g) a request that carries the delay parameter being sent at once: the delayed branch is taken first and never +falls through, including when the feature is disabled; (h) an application service: it manages a delayed event only by +masquerading as a user in its namespace, as on every other route; (i) confidentiality: the content of a not-yet-sent or +cancelled event is in the database and in memory for up to the retention period, and neither it nor the state key is +ever logged; (j) the federation worker: it never reads the table and sends nothing. Residual risk: a duplicate event +after a crash in the window between the send and its record; a wall clock stepped forwards brings deadlines forward by +the step; a suspended account's delayed events fail rather than send, so the server does not end a suspended +participant's call membership; the feature is on by default. diff --git a/docs/todos/capability-gaps.md b/docs/todos/capability-gaps.md index 3d582cb33..586abd72d 100644 --- a/docs/todos/capability-gaps.md +++ b/docs/todos/capability-gaps.md @@ -48,7 +48,8 @@ When adding a capability, record what is *missing* as explicitly as what works. | Endpoint | Status | Needs | | --- | --- | --- | | `GET /_matrix/client/v1/auth_metadata` and the MSC2965 unstable path | `spec-covered` | The OAuth 2.0 server metadata, served while `server.oauth.enabled` is true (ADR-0773); `404 M_UNRECOGNIZED` while it is off. Registration, the authorisation pages, the token endpoint (code and refresh grants), revocation, the account pages at `/_merovingian/account` (all six account management actions), the `m.oauth` UIA stage, the `legacy_api_enabled=false` switch and the hand-off for OAuth 2.0 aware clients (`m.login.sso` with `oauth_aware_preferred`) are served (ADR-0779 to ADR-0802), and so is the device authorization grant with `server.oauth.device_grant_enabled=true` (ADR-0804 to ADR-0806). An OAuth session still cannot add an email address or mint a login token (no page; `403 M_FORBIDDEN`). Element X's QR-code login needs the MSC4108 rendezvous API, which is not in spec v1.19 and not part of 1.1.0 (owner decision), so it stays unavailable although the device authorization grant is served. Nothing here has yet been tried with Element X or in a real browser; see the open items in `docs/todos/oauth-2-0-api.md`. | -| `GET /_matrix/client/v1/rtc/transports` and the MSC4143 unstable path | `runtime-wired` | MatrixRTC transport discovery (MSC4143, not in the v1.19 spec): advertises the operator's `lk-jwt-service` (`server.rtc.livekit.service_url`) as type `livekit` (ADR-0822). Gaps: no delayed events (MSC4140), so a vanished participant can stay listed in a call; no built-in token service (`m.livekit`, `/rtc/livekit/*`). | +| `GET /_matrix/client/v1/rtc/transports` and the MSC4143 unstable path | `runtime-wired` | MatrixRTC transport discovery (MSC4143, not in the v1.19 spec): advertises the operator's `lk-jwt-service` (`server.rtc.livekit.service_url`) as type `livekit` (ADR-0822). Gap: no built-in token service (`m.livekit`, `/rtc/livekit/*`). | +| Delayed events: `PUT .../delayed_event/{eventType}/{txnId}`, `?org.matrix.msc4140.delay=` on `/send` and `/state`, `POST` and `GET .../delayed_events/{delayId}` (stable, `org.matrix.msc4140` and query forms) | `runtime-wired` | MSC4140, in the v1.20 spec text and not in v1.19: schedule, restart, cancel, send and look up a delayed event, kept in the `delayed_events` table (migration 003) and sent by the `delayed-events` job through the ordinary send path (ADR-0823 to ADR-0826). Not implemented: the listing endpoint (MSC4486, `404 M_UNRECOGNIZED`) and sticky events (MSC4354). No real client (Element X, Element Web, Element Call, lk-jwt-service) has been run against it yet. | | `GET /_matrix/client/v3/voip/turnServer` | `spec-covered` | Static TURN credentials from `server.turn.*` configuration, or per-user time-limited coturn shared-secret credentials when `server.turn.shared_secret_file` is set (`src/homeserver/TurnCredentials.cpp`, ADR-0480). `server.turn.uris` advertises several URIs (ADR-0821). | | Unrouted v1.19 operations | `not-started` | These answer `404 M_UNRECOGNIZED` (most are asserted as `(implementation gap)` scenarios in `tests/conformance/test_client_server_conformance.cpp`): legacy `GET /v3/events`, `GET /v3/events/{eventId}`, `GET /v3/initialSync`; `GET /.well-known/matrix/policy_server` and `GET /.well-known/matrix/support`. `POST /v1/login/get_token` (off unless `security.login_token.enabled`; UIA password every call; single-use in-memory tokens, ADR-0548) and `GET /v3/admin/whois/{userId}` (devices only: the server records no IP, user agent or last-seen time, so `connections` is always empty) are implemented (CS8). The two `/account/password/*/requestToken` routes are routed (email: ADR-0481; msisdn answers `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`). User reporting, redaction, the appservice ping and the appservice network directory have their own rows. | | Room summary over federation | `runtime-wired` | `GET /v1/room_summary/{roomIdOrAlias}?via=` asks up to 3 `via` servers over federation `GET /hierarchy/{roomId}` (the `room` chunk; v1.19 defines no federation summary route) for a room this server holds no state for (ADR-0556): one ADR-0079 slot (429 at once without one, also without a token), one deadline, unverified answer, shown only by its own `join_rule` / `world_readable` / `allowed_room_ids`. Local state always wins; a malformed `via` is `400 M_INVALID_PARAM`. Not implemented: resolving an alias that lives on another server (404 / 401), and `membership` for a room not held locally. | From 18c3f3a387464238fc269119d0dbb2d00507b0d2 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:04:36 +0000 Subject: [PATCH 11/29] Add the built-in delayed-event rate limits with their unit test, Complement override and tables Co-Authored-By: Claude Fable 5.2 --- docs/http-transport.md | 6 +- docs/user-manual.md | 2 + src/http/RateLimit.cpp | 16 ++++++ tests/complement/merovingian.conf.template | 4 ++ tests/unit/test_http_rate_limit_tiers.cpp | 64 ++++++++++++++++++++++ 5 files changed, 89 insertions(+), 3 deletions(-) diff --git a/docs/http-transport.md b/docs/http-transport.md index 99062b5c4..9c4e3f238 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -820,10 +820,10 @@ cap applies regardless of which room, device, or media ID appears in the URL. | `generic` | every other client-server route | `client_rate_limits.default_per_ip` (600/60s) | Built-in per-endpoint refinements inside a tier: device and key APIs at -120/60s, search at 20/60s, and thumbnails at 240/60s. Search requests do real +120/60s, search at 20/60s, thumbnails at 240/60s, and the delayed-event management and lookup paths (both prefixes, ADR-0826) at 3000/60s. Search requests do real work (a bounded in-memory scan, see `ClientApiLimits::max_search_events_scanned`) rather than a cheap -lookup. The built-in per-user caps are 5/60s on `/login`, 4/60s on `/login/get_token` (ADR-0548), 30/60s on the two `preview_url` routes (ADR-0213) and 30/60s on `/_matrix/media/v1/create` (ADR-0212). +lookup. The built-in per-user caps are 5/60s on `/login`, 4/60s on `/login/get_token` (ADR-0548), 30/60s on the two `preview_url` routes (ADR-0213) and 30/60s on `/_matrix/media/v1/create` (ADR-0212). The delayed-event management and lookup paths (`/_matrix/client/v1/delayed_events/` and `/_matrix/client/unstable/org.matrix.msc4140/delayed_events/`) are 120/60s per user, and the dedicated scheduling paths (`.../rooms/{roomId}/delayed_event/...`, both prefixes) 60/60s per user (ADR-0826); each path template is its own bucket. The per-user tier keys on the *authenticated* user, which before a login succeeds is nobody — so a `/login` guessing spree against one account spread @@ -849,7 +849,7 @@ Per-IP policy resolution is most-specific-first: `client_rate_limits.default_per_ip`. Per-user resolution: `client_rate_limits.per_user.` first, then -the built-in per-user caps (`/login`, `/login/get_token`, `preview_url`, media `create`); routes with neither have no per-user cap (the +the built-in per-user caps (`/login`, `/login/get_token`, `preview_url`, media `create`, the delayed-event paths); routes with neither have no per-user cap (the per-IP cap still applies). Tier names are `auth_sensitive`, `media`, `sync`, `federation`, `admin`, `generic`; an unknown name is a parse-time finding, not a silently ignored key. All `client_rate_limits.*` changes require a server diff --git a/docs/user-manual.md b/docs/user-manual.md index fd94524f7..d7fc98580 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -1888,6 +1888,7 @@ Default route-aware policies applied when no override is configured: |---|---| | Login / registration | 20/60s per IP; 5/60s per user on `/login` | | Device and key APIs | 120/60s per IP | +| Delayed-event management and lookup (`/_matrix/client/v1/delayed_events/`, `/_matrix/client/unstable/org.matrix.msc4140/delayed_events/`) | 3000/60s per IP; 120/60s per user | | Media uploads and downloads | 120/60s per IP | | Media thumbnails | 240/60s per IP | | Search (`/_matrix/client/v3/search`) | 20/60s per IP | @@ -1896,6 +1897,7 @@ Default route-aware policies applied when no override is configured: | Login token minting (`/_matrix/client/v1/login/get_token`) | 4/60s per user | | URL previews (`/_matrix/client/v1/media/preview_url`, `/_matrix/media/v3/preview_url`) | 30/60s per user | | Asynchronous upload reservations (`/_matrix/media/v1/create`) | 30/60s per user | +| Delayed-event scheduling paths (`.../rooms/{roomId}/delayed_event/{eventType}/{txnId}`, both prefixes) | 60/60s per user | | Generic client APIs | 600/60s per IP fallback | Example overrides: diff --git a/src/http/RateLimit.cpp b/src/http/RateLimit.cpp index d43829408..46836ea3a 100644 --- a/src/http/RateLimit.cpp +++ b/src/http/RateLimit.cpp @@ -40,6 +40,13 @@ namespace { return RateLimitPolicy{240U, 60U}; } + // ADR-0826: a client polls and restarts a delayed event on a timer, so the management and lookup paths + // take a higher per-address cap than the generic tier. + if (starts_with(target, "/_matrix/client/v1/delayed_events/") || + starts_with(target, "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/")) + { + return RateLimitPolicy{3000U, 60U}; + } return std::nullopt; } @@ -192,6 +199,9 @@ auto default_client_rate_limit_config() noexcept -> RateLimitConfig {"/_matrix/client/v3/search", {20U, 60U}}, {"/_matrix/client/v1/media/thumbnail/", {240U, 60U}}, {"/_matrix/media/v3/thumbnail/", {240U, 60U}}, + // ADR-0826: management and lookup of a delayed event, both prefixes. + {"/_matrix/client/v1/delayed_events/", {3000U, 60U}}, + {"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/", {3000U, 60U}}, }, .builtin_per_user = { @@ -210,6 +220,12 @@ auto default_client_rate_limit_config() noexcept -> RateLimitConfig // report is a durable audit row, so each user is held to it (per user, normalised route). {"/_matrix/client/v3/users/{userId}/report", {20U, 60U}}, {"/_matrix/client/v3/rooms/{roomId}/report/{eventId}", {20U, 60U}}, + // ADR-0826: a user restarts a delayed event about every 5 seconds per call, so 120 a + // minute is generous for management and lookup; scheduling is one request per call join. + {"/_matrix/client/v1/delayed_events/", {120U, 60U}}, + {"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/", {120U, 60U}}, + {"/_matrix/client/v3/rooms/{roomId}/delayed_event/", {60U, 60U}}, + {"/_matrix/client/unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/", {60U, 60U}}, }, .default_per_ip = {600U, 60U}, }; diff --git a/tests/complement/merovingian.conf.template b/tests/complement/merovingian.conf.template index 806cac605..7e73b9ea5 100644 --- a/tests/complement/merovingian.conf.template +++ b/tests/complement/merovingian.conf.template @@ -84,6 +84,10 @@ client_rate_limits.per_user./_matrix/media/v3/preview_url=1000000000/1s client_rate_limits.per_user./_matrix/media/v1/create=1000000000/1s client_rate_limits.per_user./_matrix/client/v3/rooms/{roomId}/report/{eventId}=1000000000/1s client_rate_limits.per_user./_matrix/client/v3/users/{userId}/report=1000000000/1s +client_rate_limits.per_user./_matrix/client/v1/delayed_events/=1000000000/1s +client_rate_limits.per_user./_matrix/client/unstable/org.matrix.msc4140/delayed_events/=1000000000/1s +client_rate_limits.per_user./_matrix/client/v3/rooms/{roomId}/delayed_event/=1000000000/1s +client_rate_limits.per_user./_matrix/client/unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/=1000000000/1s # Complement logs in and registers from many parallel subtests; four concurrent # Argon2id hashes answered some of them 429 (ADR-0246). Each takes 64 MiB, so 32 # bounds a homeserver container at 2 GiB of hashing. diff --git a/tests/unit/test_http_rate_limit_tiers.cpp b/tests/unit/test_http_rate_limit_tiers.cpp index 559e2f075..6dc983ba5 100644 --- a/tests/unit/test_http_rate_limit_tiers.cpp +++ b/tests/unit/test_http_rate_limit_tiers.cpp @@ -9,7 +9,9 @@ #include #include +#include #include +#include namespace { @@ -348,3 +350,65 @@ SCENARIO("Raised generic and media defaults still enforce their configured burst } } } + +SCENARIO("Delayed-event paths have built-in per-address and per-user refinements (ADR-0826)", + "[http][rate-limit][tier][delayed-events]") +{ + GIVEN("the default client rate-limit configuration") + { + auto clock = ManualClock{}; + auto const engine = RateLimitEngine{merovingian::http::default_client_rate_limit_config(), clock}; + auto const stable = std::string{"/_matrix/client/v1/delayed_events/"}; + auto const unstable = std::string{"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/"}; + auto const per_user = [&](std::string const& target) -> std::uint32_t { + auto const policy = engine.resolve_per_user_policy(target); + REQUIRE(policy.has_value()); + return policy->max_requests; + }; + + WHEN("management and lookup paths of both prefixes are resolved") + { + THEN("each takes 3000 per address and 120 per user, in a 60 second window") + { + for (auto const& base : {stable, unstable}) + { + for (auto const* suffix : {"{delayId}", "{delayId}/send", "{delayId}/cancel", "{delayId}/restart"}) + { + auto const template_path = base + suffix; + REQUIRE(resolved_max(engine, template_path) == 3000U); + REQUIRE(per_user(template_path) == 120U); + REQUIRE(engine.resolve_per_user_policy(template_path)->window_seconds == 60U); + } + } + } + } + + WHEN("the dedicated scheduling templates of both prefixes are resolved") + { + auto const s1 = std::string{"/_matrix/client/v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}"}; + auto const s2 = std::string{ + "/_matrix/client/unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}"}; + + THEN("each takes 60 per user and keeps the generic per-address tier") + { + REQUIRE(per_user(s1) == 60U); + REQUIRE(per_user(s2) == 60U); + REQUIRE(resolved_max(engine, s1) == 600U); + REQUIRE(resolved_max(engine, s2) == 600U); + } + } + + WHEN("the ordinary send and state templates are resolved") + { + THEN("they have no per-user refinement") + { + REQUIRE_FALSE( + engine.resolve_per_user_policy("/_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}") + .has_value()); + REQUIRE_FALSE( + engine.resolve_per_user_policy("/_matrix/client/v3/rooms/{roomId}/state/{eventType}/{stateKey}") + .has_value()); + } + } + } +} From 96150e6cc010175f48514202616cace4dcf833c4 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:04:36 +0000 Subject: [PATCH 12/29] Add the delayed-events routes, the /versions and /capabilities entries and their request-level tests Co-Authored-By: Claude Fable 5.2 --- .../homeserver/LocalHttpRouter.hpp | 6 + src/homeserver/AGENTS.md | 30 + src/homeserver/ClientServerRuntime.cpp | 506 ++++++- src/homeserver/LocalHttpRouter.cpp | 5 + .../test_delayed_events_routes_flow.cpp | 1339 +++++++++++++++++ .../test_security_audit_http_3_flow.cpp | 31 + tests/meson.build | 1 + 7 files changed, 1912 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_delayed_events_routes_flow.cpp diff --git a/include/merovingian/homeserver/LocalHttpRouter.hpp b/include/merovingian/homeserver/LocalHttpRouter.hpp index 644df78f5..b905421e9 100644 --- a/include/merovingian/homeserver/LocalHttpRouter.hpp +++ b/include/merovingian/homeserver/LocalHttpRouter.hpp @@ -7,6 +7,7 @@ #include "merovingian/homeserver/HomeserverRuntime.hpp" #include "merovingian/homeserver/LocalHttpRequest.hpp" #include "merovingian/homeserver/LocalHttpResponse.hpp" +#include "merovingian/homeserver/OperationResult.hpp" #include "merovingian/homeserver/OutboundDeadline.hpp" #include @@ -40,6 +41,11 @@ namespace merovingian::homeserver [[nodiscard]] auto rate_limit_client_key(LocalHttpRequest const& request, config::ServerConfig const& server) -> std::string; +// The response the local router gives a room operation's result: a failure with no status is a 403, and the errcode +// travels with it (`LocalHttpResponse::errcode`). It is the function `PUT /send` and `PUT /state` answers go through, +// shared so a delayed event's failed `send` action answers what an ordinary send would (ADR-0823). +[[nodiscard]] auto room_operation_response(OperationResult const& result) -> LocalHttpResponse; + [[nodiscard]] auto handle_local_http_request(HomeserverRuntime& runtime, LocalHttpRequest const& request) -> LocalHttpResponse; [[nodiscard]] auto handle_federation_http_request(HomeserverRuntime& runtime, LocalHttpRequest const& request) diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index 7d9d2c42b..2a4209ffd 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -28,6 +28,7 @@ This is the largest and most complex module — read this file carefully before | `ClientTxnPrune.cpp` | `prune_client_transactions`: the sweep the `client-txn-prune` `PeriodicJob` runs every hour, deleting to-device `client_txn_ids` records older than 7 days (F-54, ADR-0381) | | `RetentionSweep.cpp` | `run_retention_sweep`: the sweep the `retention-sweep` `PeriodicJob` runs every hour. Deletes sessions and tokens that can no longer authenticate and `audit_log` rows past `security.audit_log_retention_days`, OAuth 2.0 session rows whose device holds no live access or refresh token (judged again in the hold that deletes them, ADR-0808), and OAuth 2.0 clients with no session unused for 30 days (`oauth_client_retention`, least recently used first; never a client with a session, nor one `oauth_client_has_pending_authorisation` names: a live pending authorisation or an unredeemed code, answered in O(1) from the stores; ADR-0777, ADR-0782), in transactions of at most 500 rows (`retention_batch_rows`), at most 20 batches per kind per sweep. `runtime.mutex` is taken for one batch at a time, never across the whole sweep (ADR-0488, ADR-0489) | | `DelayedEvents.cpp` | Delayed events (MSC4140, ADR-0823 to ADR-0826): `schedule_delayed_event`, `manage_delayed_event` (send, cancel, restart), `delayed_event_lookup`, the clock `delayed_event_now_ms` (and its seam `test_delayed_event_clock_ms`), `wake_delayed_events`, and `send_due_delayed_events`, the body of the `delayed-events` `PeriodicJob` in `main.cpp`. The caller holds `runtime.mutex` for all but the job body, which takes it for one send at a time | +| `ClientServerRuntime.cpp` (delayed events) | The MSC4140 routes (ADR-0823): S1 and S2 `PUT .../delayed_event/...`, the `org.matrix.msc4140.delay` branch of `PUT /send` and `/state`, M1 to M3 `POST .../delayed_events/{delayId}[/{action}]` and G1 and G2 `GET .../delayed_events/{delayId}`, with the `/versions` flags and `/capabilities` entries. See "Delayed events" below | | `ClientServerRuntimeInternal.hpp` | Call-throughs from `ClientServerRuntime.cpp`'s anonymous namespace that `DelayedEvents.cpp` shares with the `/send` and `/state` routes: `rate_limit_route_template` (`normalized_target`), `route_event_body` (`event_body_from_content`), `event_txn_part_is_too_long`, `errcode_for_refusal_status`, and the two reasons the `/send` route gives for a too-long type or transaction ID. Not public API | | `OAuthClientRegistration.cpp` | `POST /_merovingian/oauth2/register` (`register_oauth_client`), the client ID (`oauth_client_id_of`, ADR-0779), the cap and its eviction (ADR-0780), and `find_registered_oauth_client` | | `OAuthJsonResponse.cpp` | `oauth_json_response` / `oauth_error_response` / `oauth_empty_response`: every OAuth JSON (or empty) answer, with `no-store` (ADR-0781) | @@ -1236,6 +1237,35 @@ next layer) only parse a request and call it. Invariants a change must keep: - `send_event_as_user`'s room and membership refusal is `room_send_refusal`, and its power refusal `event_authorization_refusal`, both shared with the schedule-time check; change them, not a copy. +The routes (`ClientServerRuntime.cpp`, ADR-0823) only parse a request and call the service. Rules a change must keep: + +- **The delayed branch of `/send` and `/state` comes first and never falls through.** A `PUT .../send/{eventType}/{txnId}` + or `PUT .../state/{eventType}[/{stateKey}]` that carries `org.matrix.msc4140.delay` (`delay_query_of`: the name is + percent-decoded too) is answered by `delayed_query_form` before the idempotency lookup, `?ts=` and anything that could + send. A repeated, empty or non-digit value, `0`, or one over 64 bits is `400 M_INVALID_PARAM`; a disabled server is + `403 M_FORBIDDEN` first. No error of such a request may reach the ordinary send. +- **The dedicated routes (S1, S2) are matched before the room routes**, whose `/send/` and `/state/` markers would + otherwise find them inside the path. Disabled is answered before any body error (`delayed_events_disabled_refusal` + calls the service with an empty request, so the text is written once); then `parse_delayed_schedule_body`. +- **One helper maps every answer**: `delayed_event_answer`. Success is the route's body; a failure goes through + `room_operation_response` (the function the local router's `/send` answer uses, exported in `LocalHttpRouter.hpp`) + and `error_code_for`, with `retry_after_ms` through `dispatch_err`. A route never writes an errcode of its own, so a + failed `send` action answers what `PUT /send` would have. +- **The 404 for an unknown or another user's ID is the service's, byte for byte, on every management and lookup + route** (M1, M2, M3, G1, G2). A route answers it with nothing added: no header, no body member. +- **M1 and M2 ignore the body.** An empty body, `{}` and unknown members pass; a body that is not JSON is the + dispatcher's `M_NOT_JSON` before the route (the gate skips an empty or all-whitespace body). M3 (unstable only) needs a + JSON object with a string `action`. `POST /v1/delayed_events/{id}` is no route: 405 `M_UNRECOGNIZED` after + authentication, like any served path with another method. The listing `GET .../delayed_events` is unmatched: 404. +- **Templates**: `normalized_target` gives S1 and S2 `.../rooms/{roomId}/delayed_event/{eventType}/{txnId}` and each + prefix `.../delayed_events/{delayId}` and `.../{delayId}/{send|cancel|restart}`; the action stays literal, so any + other last segment is unmatched (404 before authentication). Each template has a row in the HTTP-3 scenario "gives + every implemented dynamic client route its own coalesced bucket", and the built-in limits in + `default_client_rate_limit_config` (ADR-0826) are keyed on them; a built-in per-user key also needs its + `client_rate_limits.per_user.` line in `tests/complement/merovingian.conf.template`. +- `/versions` flags and `/capabilities` entries read `rt.limits.delayed_events_enabled`, + `max_delayed_event_delay_ms` and `max_scheduled_delayed_events_per_user`; disabled is false and zeros. + ## Typing expiry (R13, ADR-0199) Sending a message does not end the sender's typing (ADR-0298): typing ends only on the client's diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 7a0697976..1f9c4b1a6 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -90,6 +90,7 @@ #include "merovingian/homeserver/ClientServerStartOptions.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" #include "merovingian/homeserver/CreateRoomOptions.hpp" +#include "merovingian/homeserver/DelayedEvents.hpp" #include "merovingian/homeserver/EmailPush.hpp" #include "merovingian/homeserver/FetchRelationsRequest.hpp" #include "merovingian/homeserver/FetchThreadsRequest.hpp" @@ -3492,6 +3493,16 @@ namespace return std::string{unmatched}; } } + else if (action == "delayed_event") + { + // S1 (MSC4140, ADR-0823): PUT /rooms/{roomId}/delayed_event/{eventType}/{txnId}. + out = + replace_two_tail_segments(normalized_room + "/delayed_event/", tail, "{eventType}", "{txnId}"); + if (out.empty()) + { + return std::string{unmatched}; + } + } else if (action == "event") { if (!has_one_segment(tail)) @@ -3714,6 +3725,63 @@ namespace return std::string{unmatched}; } + // Delayed events (MSC4140, ADR-0823): the identifier of a delayed event, and for S2 the room, event type and + // transaction ID, are variable, so each template is its own bucket (ADR-0092). The management action stays + // literal: any other last segment is unmatched, so it is answered 404 M_UNRECOGNIZED before authentication. + auto const delayed_events_template = [&](std::string_view prefix, std::string_view rest) -> std::string { + auto const first_end = rest.find('/'); + if (!has_one_segment(rest.substr(0U, first_end))) + { + return std::string{unmatched}; + } + if (first_end == std::string_view::npos) + { + return std::string{prefix} + "delayed_events/{delayId}"; + } + auto const action = rest.substr(first_end + 1U); + if (action == "send" || action == "cancel" || action == "restart") + { + return std::string{prefix} + "delayed_events/{delayId}/" + std::string{action}; + } + return std::string{unmatched}; + }; + auto constexpr delayed_stable_prefix = std::string_view{"/_matrix/client/v1/delayed_events/"}; + if (starts_with(out, delayed_stable_prefix)) + { + return delayed_events_template("/_matrix/client/v1/", + std::string_view{out}.substr(delayed_stable_prefix.size())); + } + auto constexpr delayed_unstable_root = std::string_view{"/_matrix/client/unstable/org.matrix.msc4140/"}; + if (starts_with(out, delayed_unstable_root)) + { + auto const rest = std::string_view{out}.substr(delayed_unstable_root.size()); + auto constexpr rooms_part = std::string_view{"rooms/"}; + auto constexpr events_part = std::string_view{"delayed_events/"}; + if (starts_with(rest, events_part)) + { + return delayed_events_template(delayed_unstable_root, rest.substr(events_part.size())); + } + if (starts_with(rest, rooms_part)) + { + // rooms/{roomId}/delayed_event/{eventType}/{txnId} + auto const after_rooms = rest.substr(rooms_part.size()); + auto const room_end = after_rooms.find('/'); + auto constexpr marker = std::string_view{"/delayed_event/"}; + if (room_end != std::string_view::npos && room_end != 0U && + starts_with(after_rooms.substr(room_end), marker)) + { + auto const normalized = replace_two_tail_segments( + std::string{delayed_unstable_root} + "rooms/{roomId}/delayed_event/", + after_rooms.substr(room_end + marker.size()), "{eventType}", "{txnId}"); + if (!normalized.empty()) + { + return normalized; + } + } + } + return std::string{unmatched}; + } + // v1 room endpoints carry the room id (and, for relations, the event id) // in the path. Coalesce those parameters so rate-limit accounting is // stable across rooms and events. @@ -12352,6 +12420,376 @@ namespace return json_serialize(json_obj(std::move(members))); } + // ---- Delayed events (MSC4140, ADR-0823 to ADR-0826): the routes only parse a request and call DelayedEvents.cpp. + + inline constexpr auto delay_query_parameter = std::string_view{"org.matrix.msc4140.delay"}; + + // What a request's `org.matrix.msc4140.delay` query parameter says. `present` is true for a request that carries + // the parameter at all, with any value: such a request is a delayed one and never falls through to an immediate + // send. `delay_ms` is empty when the parameter is repeated, empty, or not a whole decimal number of at least 1 + // that fits 64 bits. + struct DelayQuery final + { + bool present{false}; + std::optional delay_ms{}; + }; + + // Reads the parameter, with the name percent-decoded as well as the value so that an encoded spelling of it cannot + // slip past into an immediate send. Looks at the first two occurrences only. + [[nodiscard]] auto delay_query_of(std::string_view target) -> DelayQuery + { + auto values = std::vector{}; + auto const query_pos = target.find('?'); + if (query_pos != std::string_view::npos) + { + auto query = target.substr(query_pos + 1U); + while (!query.empty() && values.size() < 2U) + { + auto const amp = query.find('&'); + auto const pair = query.substr(0U, amp); + auto const equals = pair.find('='); + if (core::percent_decode(pair.substr(0U, equals)) == delay_query_parameter) + { + values.push_back(core::percent_decode(equals == std::string_view::npos ? std::string_view{} + : pair.substr(equals + 1U))); + } + if (amp == std::string_view::npos) + { + break; + } + query.remove_prefix(amp + 1U); + } + } + if (values.empty()) + { + return {}; + } + auto result = DelayQuery{.present = true, .delay_ms = std::nullopt}; + if (values.size() != 1U || values.front().empty() || !std::ranges::all_of(values.front(), [](char c) { + return c >= '0' && c <= '9'; + })) + { + return result; + } + auto parsed = std::uint64_t{0}; + auto const text = std::string_view{values.front()}; + auto const [end, error] = std::from_chars(text.data(), text.data() + text.size(), parsed); + if (error == std::errc{} && end == text.data() + text.size() && parsed >= 1U) + { + result.delay_ms = parsed; + } + return result; + } + + // The 403 a disabled server answers, taken from the service so the text is written once; nullopt when delayed + // events are on. A disabled server answers it before any body or parameter error. + [[nodiscard]] auto delayed_events_disabled_refusal(ClientServerRuntime& rt) -> std::optional + { + if (rt.limits.delayed_events_enabled) + { + return std::nullopt; + } + return schedule_delayed_event(rt, DelayedEventScheduleRequest{}); + } + + // What a successful delayed-event operation answers: `{"delay_id": value}`, the service's JSON body as it is, or + // `{}`. + enum class DelayedEventSuccess : std::uint8_t + { + delay_id, + body, + empty, + }; + + // The one mapping from a delayed-event `OperationResult` to the response, used by every delayed-event route. A + // failure goes through `room_operation_response` and `error_code_for`, the functions the ordinary `/send` answer + // goes through (a failure with no status is 403, an empty errcode is the status's usual one), so a failed `send` + // action answers what `PUT /send` would have. Never writes an errcode of its own. + [[nodiscard]] auto delayed_event_answer(LocalHttpRequest const& req, ClientServerRuntime const& rt, + OperationResult const& result, DelayedEventSuccess success) + -> DispatchResult + { + if (result.ok) + { + switch (success) + { + case DelayedEventSuccess::delay_id: + return dispatch_resp(req, rt, 200U, + json_serialize(json_obj({json_member("delay_id", json_str(result.value))}))); + case DelayedEventSuccess::body: + return dispatch_resp(req, rt, 200U, result.value); + case DelayedEventSuccess::empty: + return dispatch_resp(req, rt, 200U, "{}"); + } + } + auto const failure = room_operation_response(result); + return dispatch_err(req, rt, failure.status, error_code_for(failure), failure.body, result.retry_after_ms); + } + + // Schedules `request` for the request's user and device, and answers it. Caller holds runtime.mutex. + [[nodiscard]] auto schedule_delayed_event_for_request(ClientServerRuntime& rt, LocalHttpRequest const& req, + std::string const& user, DelayedEventScheduleRequest request) + -> DispatchResult + { + request.user_id = user; + request.device_id = authenticated_request_device_id(rt, req.access_token); + request.via_appservice = appservice::decode_masquerade_token(req.access_token).has_value(); + auto const result = schedule_delayed_event(rt, request); + return delayed_event_answer(req, rt, result, DelayedEventSuccess::delay_id); + } + + // L1 and L2: a `/send` or `/state` request that carries `org.matrix.msc4140.delay` (ADR-0823). Answers nothing for + // a request without the parameter. With it, the request is always a delayed one: whatever is wrong with it, it is + // never sent now. + [[nodiscard]] auto delayed_query_form(ClientServerRuntime& rt, LocalHttpRequest const& req, std::string const& user, + std::string_view room_id, std::string_view event_type, + std::string_view txn_id, std::optional state_key) + -> std::optional + { + auto const delay = delay_query_of(req.target); + if (!delay.present) + { + return std::nullopt; + } + if (auto const refusal = delayed_events_disabled_refusal(rt); refusal.has_value()) + { + return delayed_event_answer(req, rt, *refusal, DelayedEventSuccess::delay_id); + } + if (!delay.delay_ms.has_value()) + { + return dispatch_err(req, rt, 400U, "M_INVALID_PARAM", + "org.matrix.msc4140.delay must be given once, as a whole number of milliseconds of at " + "least 1"); + } + auto request = DelayedEventScheduleRequest{}; + request.room_id = std::string{room_id}; + request.event_type = std::string{event_type}; + request.txn_id = std::string{txn_id}; + request.state_key = std::move(state_key); + request.content_json = req.body; + request.delay_ms = *delay.delay_ms; + request.stable_errcode = false; + return schedule_delayed_event_for_request(rt, req, user, std::move(request)); + } + + // The members of S1 and S2's body. + struct DelayedScheduleBody final + { + std::uint64_t delay_ms{0U}; + std::string content_json{}; + std::optional state_key{}; + }; + + // `delay_ms` (a JSON integer of at least 1) and `content` (a JSON object) are required; `state_key`, when present, + // must be a string. Other members are ignored. nullopt is 400 M_BAD_JSON. + [[nodiscard]] auto parse_delayed_schedule_body(std::string_view body) -> std::optional + { + auto const object = parsed_json_object(body); + if (!object.has_value()) + { + return std::nullopt; + } + auto const* delay = integer_member(*object, "delay_ms"); + auto const* content = object_member(*object, "content"); + if (delay == nullptr || *delay < 1 || content == nullptr || + std::get_if(&content->storage()) == nullptr) + { + return std::nullopt; + } + auto parsed = DelayedScheduleBody{}; + parsed.delay_ms = static_cast(*delay); + parsed.content_json = json_serialize(*content); + if (auto const* state_key = object_member(*object, "state_key"); state_key != nullptr) + { + auto const* text = std::get_if(&state_key->storage()); + if (text == nullptr) + { + return std::nullopt; + } + parsed.state_key = *text; + } + return parsed; + } + + // The parts of S1 (`/_matrix/client/v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}`) and S2 (the same under + // `/_matrix/client/unstable/org.matrix.msc4140/`). + struct DelayedSchedulePath final + { + std::string room_id{}; + std::string event_type{}; + std::string txn_id{}; + bool stable{false}; + }; + + [[nodiscard]] auto delayed_schedule_path(std::string_view request_path) -> std::optional + { + auto constexpr stable_prefix = std::string_view{"/_matrix/client/v3/rooms/"}; + auto constexpr unstable_prefix = std::string_view{"/_matrix/client/unstable/org.matrix.msc4140/rooms/"}; + auto const stable = starts_with(request_path, stable_prefix); + if (!stable && !starts_with(request_path, unstable_prefix)) + { + return std::nullopt; + } + auto rest = request_path.substr(stable ? stable_prefix.size() : unstable_prefix.size()); + auto segments = std::array{}; + for (auto index = std::size_t{0U}; index < segments.size(); ++index) + { + auto const slash = rest.find('/'); + auto const last = index + 1U == segments.size(); + if ((slash == std::string_view::npos) != last || slash == 0U || rest.empty()) + { + return std::nullopt; + } + segments[index] = rest.substr(0U, slash); + if (!last) + { + rest.remove_prefix(slash + 1U); + } + } + if (segments[1] != "delayed_event" || segments[2].empty() || segments[3].empty()) + { + return std::nullopt; + } + return DelayedSchedulePath{core::percent_decode_path_component(segments[0]), + core::percent_decode_path_component(segments[2]), + core::percent_decode_path_component(segments[3]), stable}; + } + + // S1 and S2. Order: the feature switch, then the body, then the service. + [[nodiscard]] auto handle_delayed_event_schedule(ClientServerRuntime& rt, LocalHttpRequest const& req, + std::string const& user, DelayedSchedulePath const& path) + -> DispatchResult + { + if (auto const refusal = delayed_events_disabled_refusal(rt); refusal.has_value()) + { + return delayed_event_answer(req, rt, *refusal, DelayedEventSuccess::delay_id); + } + auto const body = parse_delayed_schedule_body(req.body); + if (!body.has_value()) + { + return dispatch_err(req, rt, 400U, "M_BAD_JSON", + "the body must be an object with an integer delay_ms of at least 1, an object content " + "and, optionally, a string state_key"); + } + auto request = DelayedEventScheduleRequest{}; + request.room_id = path.room_id; + request.event_type = path.event_type; + request.txn_id = path.txn_id; + request.state_key = body->state_key; + request.content_json = body->content_json; + request.delay_ms = body->delay_ms; + request.stable_errcode = path.stable; + return schedule_delayed_event_for_request(rt, req, user, std::move(request)); + } + + // The parts of the management and lookup paths: `.../delayed_events/{delayId}` and `.../{delayId}/{action}`, on + // `/_matrix/client/v1/` (stable) or `/_matrix/client/unstable/org.matrix.msc4140/`. + struct DelayedManagePath final + { + std::string delay_id{}; + std::optional action{}; + bool stable{false}; + }; + + [[nodiscard]] auto delayed_manage_path(std::string_view request_path) -> std::optional + { + auto constexpr stable_prefix = std::string_view{"/_matrix/client/v1/delayed_events/"}; + auto constexpr unstable_prefix = + std::string_view{"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/"}; + auto const stable = starts_with(request_path, stable_prefix); + if (!stable && !starts_with(request_path, unstable_prefix)) + { + return std::nullopt; + } + auto const rest = request_path.substr(stable ? stable_prefix.size() : unstable_prefix.size()); + auto const slash = rest.find('/'); + auto const delay_id = rest.substr(0U, slash); + if (delay_id.empty()) + { + return std::nullopt; + } + auto path = DelayedManagePath{core::percent_decode_path_component(delay_id), std::nullopt, stable}; + if (slash == std::string_view::npos) + { + return path; + } + auto const action = rest.substr(slash + 1U); + if (action == "send") + { + path.action = DelayedEventAction::send; + } + else if (action == "cancel") + { + path.action = DelayedEventAction::cancel; + } + else if (action == "restart") + { + path.action = DelayedEventAction::restart; + } + else + { + return std::nullopt; + } + return path; + } + + // M1, M2, M3, G1 and G2. nullopt for a method the path does not serve (M3 has no stable form), which the caller + // leaves to the dispatcher's 405. M1 and M2 ignore the body. + [[nodiscard]] auto handle_delayed_event_management(ClientServerRuntime& rt, LocalHttpRequest const& req, + std::string const& user, DelayedManagePath const& path) + -> std::optional + { + if (req.method == "GET" && !path.action.has_value()) + { + return delayed_event_answer(req, rt, delayed_event_lookup(rt, user, path.delay_id), + DelayedEventSuccess::body); + } + if (req.method != "POST") + { + return std::nullopt; + } + if (path.action.has_value()) + { + return delayed_event_answer(req, rt, manage_delayed_event(rt, user, path.delay_id, *path.action), + DelayedEventSuccess::empty); + } + if (path.stable) + { + return std::nullopt; + } + // M3: the action is in the body. + auto const object = parsed_json_object(req.body); + if (!object.has_value()) + { + return dispatch_err(req, rt, 400U, "M_BAD_JSON", "the request body must be a JSON object"); + } + auto const* member = object_member(*object, "action"); + if (member == nullptr) + { + return dispatch_err(req, rt, 400U, "M_MISSING_PARAM", "action is required"); + } + auto const* text = std::get_if(&member->storage()); + auto action = std::optional{}; + if (text != nullptr && *text == "send") + { + action = DelayedEventAction::send; + } + else if (text != nullptr && *text == "cancel") + { + action = DelayedEventAction::cancel; + } + else if (text != nullptr && *text == "restart") + { + action = DelayedEventAction::restart; + } + if (!action.has_value()) + { + return dispatch_err(req, rt, 400U, "M_INVALID_PARAM", "action must be send, cancel or restart"); + } + return delayed_event_answer(req, rt, manage_delayed_event(rt, user, path.delay_id, *action), + DelayedEventSuccess::empty); + } + // A refused upload, create or reserved upload as the client sees it. The // media service names the spec's errcode where one applies; the internal // statuses for a disallowed file type (415) and an exhausted quota (507) @@ -15389,12 +15827,16 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp req, rt, 200U, json_serialize(json_obj({ json_member("versions", json_arr(std::move(versions))), - json_member("unstable_features", json_obj({ - json_member("org.matrix.msc4186", json_bool(true)), - json_member("org.matrix.msc4143", json_bool(true)), - json_member("org.matrix.msc4143.stable", json_bool(true)), - json_member("org.matrix.simplified_msc3575", json_bool(true)), - })), + json_member("unstable_features", + json_obj({ + json_member("org.matrix.msc4186", json_bool(true)), + json_member("org.matrix.msc4143", json_bool(true)), + json_member("org.matrix.msc4143.stable", json_bool(true)), + // MSC4140 (ADR-0823): both flags say whether scheduling is on. + json_member("org.matrix.msc4140", json_bool(rt.limits.delayed_events_enabled)), + json_member("org.matrix.msc4140.stable", json_bool(rt.limits.delayed_events_enabled)), + json_member("org.matrix.simplified_msc3575", json_bool(true)), + })), }))); } @@ -18815,6 +19257,21 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp json_member("available", json_obj(std::move(available_versions))), })), }; + // MSC4140 (ADR-0823, ADR-0826): the longest delay and the most scheduled events a user may hold; zeros when + // scheduling is off. Both the stable and the unstable name are served. + { + auto const enabled = rt.limits.delayed_events_enabled; + auto const max_delay = enabled ? static_cast(rt.limits.max_delayed_event_delay_ms) : 0; + auto const max_scheduled = + enabled ? static_cast(rt.limits.max_scheduled_delayed_events_per_user) : 0; + for (auto const* name : {"m.delayed_events", "org.matrix.msc4140.delayed_events"}) + { + capabilities.push_back(json_member(name, json_obj({ + json_member("max_delay_ms", json_int(max_delay)), + json_member("max_scheduled", json_int(max_scheduled)), + }))); + } + } // Spec (m.account_moderation, v1.18): whether the caller can lock and suspend accounts, "omitted // altogether if `suspend` and `lock` would be `false`". Both endpoints need a server admin, so // the two flags are always equal. @@ -19698,6 +20155,28 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp }))); } + // Delayed events (MSC4140, ADR-0823): S1 and S2 schedule, M1 to M3 manage and G1 and G2 look up. The listing + // `GET .../delayed_events` is MSC4486 and no route (`normalized_target` leaves it unmatched). Placed before the + // room routes, whose `/send/` and `/state/` markers must never be offered S1's path. A method a path does not + // serve (M3 under `/v1`, a GET of S1) is left to the dispatcher's 405. + if (req.method == "PUT") + { + if (auto const path = delayed_schedule_path(request_path); path.has_value()) + { + return handle_delayed_event_schedule(rt, req, *user, *path); + } + } + if (req.method == "GET" || req.method == "POST") + { + if (auto const path = delayed_manage_path(request_path); path.has_value()) + { + if (auto answer = handle_delayed_event_management(rt, req, *user, *path); answer.has_value()) + { + return std::move(*answer); + } + } + } + auto const key_route = auth::match_key_api_route(req.method, req.target); if (key_route.matched) { @@ -20631,6 +21110,14 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp } if (auto const path = room_send_path_parts(req.target); path.has_value()) { + // L1 (MSC4140, ADR-0823): the delayed branch comes before the idempotency lookup, `?ts=` and + // everything that could send, and a request that carries the parameter never falls through. + if (auto delayed = + delayed_query_form(rt, req, *user, path->room_id, path->event_type, path->txn_id, std::nullopt); + delayed.has_value()) + { + return std::move(*delayed); + } auto const massaged = massaged_timestamp(req); if (!massaged.valid) { @@ -20713,6 +21200,13 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp } if (auto const path = room_state_path_parts(req.target); path.has_value()) { + // L2 (MSC4140, ADR-0823): as L1, for every form of the state path (a state key, an empty one, none). + if (auto delayed = + delayed_query_form(rt, req, *user, path->room_id, path->event_type, {}, path->state_key); + delayed.has_value()) + { + return std::move(*delayed); + } auto const massaged = massaged_timestamp(req); if (!massaged.valid) { diff --git a/src/homeserver/LocalHttpRouter.cpp b/src/homeserver/LocalHttpRouter.cpp index ed54d6073..30ce90d0e 100644 --- a/src/homeserver/LocalHttpRouter.cpp +++ b/src/homeserver/LocalHttpRouter.cpp @@ -6053,6 +6053,11 @@ namespace return result; } +auto room_operation_response(OperationResult const& result) -> LocalHttpResponse +{ + return response_from_room_operation(result); +} + auto pdu_event_already_stored(HomeserverRuntime& runtime, std::string_view room_id, std::string_view event_id) -> bool { auto const guard = RuntimeLock{runtime.mutex}; diff --git a/tests/integration/test_delayed_events_routes_flow.cpp b/tests/integration/test_delayed_events_routes_flow.cpp new file mode 100644 index 000000000..98130eed4 --- /dev/null +++ b/tests/integration/test_delayed_events_routes_flow.cpp @@ -0,0 +1,1339 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// Delayed events (MSC4140, ADR-0823, ADR-0826): the client-server routes that call the service layer, the /versions +// and /capabilities entries and the built-in rate limits. Every request goes through +// `handle_client_server_request` on a real runtime. No scenario sleeps or starts the job's thread: time is +// `HomeserverRuntime::test_delayed_event_clock_ms` and the job body is called with its own `now_ms`. The feature is not +// in the Matrix v1.19 specification, so there is no conformance scenario. + +#include "../support/AppserviceTestConfig.hpp" +#include "../support/InMemoryDatabaseConfig.hpp" +#include "../support/JsonTestSupport.hpp" +#include "../support/MasterKey.hpp" +#include "../support/RegistrationToken.hpp" +#include "merovingian/canonicaljson/Value.hpp" +#include "merovingian/config/ClientRateLimitsConfig.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ListenersConfig.hpp" +#include "merovingian/config/LogModulesConfig.hpp" +#include "merovingian/config/SecurityConfig.hpp" +#include "merovingian/config/ServerConfig.hpp" +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/DelayedEventTable.hpp" +#include "merovingian/database/PersistentStore.hpp" +#include "merovingian/homeserver/AuthService.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerStartResult.hpp" +#include "merovingian/homeserver/DelayedEvents.hpp" +#include "merovingian/homeserver/LocalHttpResponse.hpp" +#include "merovingian/homeserver/StopFlag.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +namespace hs = merovingian::homeserver; +namespace cj = merovingian::canonicaljson; +namespace db = merovingian::database; +using namespace merovingian::tests; + +constexpr auto t0 = std::uint64_t{1'700'000'000'000U}; +constexpr auto password = std::string_view{"CorrectHorse7!"}; +constexpr auto alice = std::string_view{"@alice:example.org"}; +constexpr auto bob = std::string_view{"@bob:example.org"}; +constexpr auto v1 = std::string_view{"/_matrix/client/v1/delayed_events/"}; +constexpr auto unstable = std::string_view{"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/"}; +constexpr auto delay_param = std::string_view{"org.matrix.msc4140.delay"}; +constexpr auto message_content = std::string_view{R"({"body":"later"})"}; + +[[nodiscard]] auto memory_config() -> merovingian::config::Config +{ + auto security = merovingian::config::SecurityConfig{}; + security.secrets.master_key_file = shared_master_key_file(); + enable_token_registration(security); + return { + merovingian::config::ServerConfig{}, + merovingian::config::ListenersConfig{}, + in_memory_database_config(), + std::move(security), + merovingian::config::ClientRateLimitsConfig{}, + merovingian::config::LogModulesConfig{}, + }; +} + +[[nodiscard]] auto request(hs::ClientServerRuntime& runtime, std::string method, std::string target, + std::string const& token = {}, std::string body = {}) -> hs::LocalHttpResponse +{ + return hs::handle_client_server_request(runtime, {std::move(method), std::move(target), token, std::move(body)}) + .response; +} + +[[nodiscard]] auto errcode_of(hs::LocalHttpResponse const& response) -> std::string +{ + auto const body = parse_object(response.body); + auto const* code = string_member(body, "errcode"); + return code == nullptr ? std::string{} : *code; +} + +[[nodiscard]] auto header_of(hs::LocalHttpResponse const& response, std::string_view name) -> std::string +{ + for (auto const& [key, value] : response.headers) + { + if (key == name) + { + return value; + } + } + return {}; +} + +// Registers `name` and signs it in on device `device`; the access token. +[[nodiscard]] auto sign_in(hs::ClientServerRuntime& runtime, std::string const& name, std::string const& device) + -> std::string +{ + REQUIRE(request(runtime, "POST", "/_matrix/client/v3/register", {}, registration_json(name, password)).status == + 200U); + auto const login = + request(runtime, "POST", "/_matrix/client/v3/login", {}, + R"({"type":"m.login.password","identifier":{"type":"m.id.user","user":"@)" + name + + R"(:example.org"},"password":")" + std::string{password} + R"(","device_id":")" + device + R"("})"); + REQUIRE(login.status == 200U); + auto const body = parse_object(login.body); + auto const* token = string_member(body, "access_token"); + REQUIRE(token != nullptr); + return *token; +} + +// A running homeserver with alice and bob in one public room. Alice created it. +class Fixture final +{ +public: + explicit Fixture(merovingian::config::Config const& config = memory_config()) + : m_started{hs::start_client_server(config)} + { + REQUIRE(m_started.started); + alice_token = sign_in(runtime(), "alice", "D1"); + bob_token = sign_in(runtime(), "bob", "D2"); + auto const created = + request(runtime(), "POST", "/_matrix/client/v3/createRoom", alice_token, R"({"preset":"public_chat"})"); + REQUIRE(created.status == 200U); + room_id = *string_member(parse_object(created.body), "room_id"); + REQUIRE(request(runtime(), "POST", "/_matrix/client/v3/join/" + room_id, bob_token, "{}").status == 200U); + runtime().homeserver.test_delayed_event_clock_ms = t0; + runtime().homeserver.test_frozen_event_clock_ms = static_cast(t0); + } + + [[nodiscard]] auto runtime() noexcept -> hs::ClientServerRuntime& + { + return m_started.runtime; + } + + [[nodiscard]] auto table() noexcept -> db::DelayedEventTable& + { + return m_started.runtime.homeserver.database.persistent_store.delayed_events; + } + + [[nodiscard]] auto event_count() noexcept -> std::size_t + { + return m_started.runtime.homeserver.database.persistent_store.events.size(); + } + + // The room's path, `/_matrix/client/v3/rooms/{roomId}`. + [[nodiscard]] auto room_path() const -> std::string + { + return "/_matrix/client/v3/rooms/" + room_id; + } + + // Moves both the delayed-event clock and the frozen event clock. + auto set_time(std::uint64_t now_ms) -> void + { + runtime().homeserver.test_delayed_event_clock_ms = now_ms; + runtime().homeserver.test_frozen_event_clock_ms = static_cast(now_ms); + } + + // The job body at `now_ms`. + auto run_job(std::uint64_t now_ms) -> void + { + std::ignore = hs::send_due_delayed_events(runtime(), now_ms, hs::StopFlag{}); + } + + std::string alice_token{}; + std::string bob_token{}; + std::string room_id{}; + +private: + hs::ClientServerStartResult m_started; +}; + +// S1 and S2's body. +[[nodiscard]] auto schedule_body(std::uint64_t delay_ms, std::string_view content = message_content) -> std::string +{ + return R"({"delay_ms":)" + std::to_string(delay_ms) + R"(,"content":)" + std::string{content} + "}"; +} + +// PUT of S1 (`stable`) or S2 for `type` and `txn`. +[[nodiscard]] auto put_dedicated(Fixture& fixture, bool stable, std::string const& token, std::string_view type, + std::string_view txn, std::string body) -> hs::LocalHttpResponse +{ + auto const path = stable + ? fixture.room_path() + "/delayed_event/" + : "/_matrix/client/unstable/org.matrix.msc4140/rooms/" + fixture.room_id + "/delayed_event/"; + return request(fixture.runtime(), "PUT", path + std::string{type} + "/" + std::string{txn}, token, std::move(body)); +} + +// PUT of L1: the ordinary send route with the delay in the query. +[[nodiscard]] auto put_send_delayed(Fixture& fixture, std::string const& token, std::string_view type, + std::string_view txn, std::string_view query, + std::string body = std::string{message_content}) -> hs::LocalHttpResponse +{ + return request(fixture.runtime(), "PUT", + fixture.room_path() + "/send/" + std::string{type} + "/" + std::string{txn} + "?" + + std::string{query}, + token, std::move(body)); +} + +// PUT of L2: the ordinary state route; `tail` is what follows the event type, `/key`, `/` or nothing. +[[nodiscard]] auto put_state_delayed(Fixture& fixture, std::string const& token, std::string_view type, + std::string_view tail, std::string_view query, std::string body = R"({"v":1})") + -> hs::LocalHttpResponse +{ + return request(fixture.runtime(), "PUT", + fixture.room_path() + "/state/" + std::string{type} + std::string{tail} + "?" + std::string{query}, + token, std::move(body)); +} + +[[nodiscard]] auto delay_id_of(hs::LocalHttpResponse const& response) -> std::string +{ + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + auto const* id = string_member(body, "delay_id"); + REQUIRE(id != nullptr); + REQUIRE_FALSE(id->empty()); + return *id; +} + +// Schedules a message for alice by S2 and returns its delay ID. +[[nodiscard]] auto schedule_message(Fixture& fixture, std::string_view txn, std::uint64_t delay_ms = 60'000U, + std::string const& token = {}) -> std::string +{ + return delay_id_of(put_dedicated(fixture, false, token.empty() ? fixture.alice_token : token, "m.test.delayed", txn, + schedule_body(delay_ms))); +} + +[[nodiscard]] auto post_action(Fixture& fixture, bool stable, std::string_view delay_id, std::string_view action, + std::string const& token, std::string body = {}, std::string_view query = {}) + -> hs::LocalHttpResponse +{ + return request(fixture.runtime(), "POST", + std::string{stable ? v1 : unstable} + std::string{delay_id} + "/" + std::string{action} + + std::string{query}, + token, std::move(body)); +} + +[[nodiscard]] auto post_body_action(Fixture& fixture, std::string_view delay_id, std::string body, + std::string const& token) -> hs::LocalHttpResponse +{ + return request(fixture.runtime(), "POST", std::string{unstable} + std::string{delay_id}, token, std::move(body)); +} + +[[nodiscard]] auto get_delayed(Fixture& fixture, bool stable, std::string_view delay_id, std::string const& token) + -> hs::LocalHttpResponse +{ + return request(fixture.runtime(), "GET", std::string{stable ? v1 : unstable} + std::string{delay_id}, token); +} + +[[nodiscard]] auto status_of(Fixture& fixture, std::string_view user, std::string_view delay_id) + -> db::DelayedEventStatus +{ + auto const* row = fixture.table().find_owned(user, delay_id); + REQUIRE(row != nullptr); + return row->status; +} + +// True when the room's timeline in alice's initial `/sync` holds an event of `type`. +[[nodiscard]] auto timeline_has(Fixture& fixture, std::string_view type) -> bool +{ + auto const response = request(fixture.runtime(), "GET", "/_matrix/client/v3/sync", fixture.alice_token); + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + auto const* rooms = object_member_as_object(body, "rooms"); + auto const* joined = rooms == nullptr ? nullptr : object_member_as_object(*rooms, "join"); + auto const* room = joined == nullptr ? nullptr : object_member_as_object(*joined, fixture.room_id); + auto const* timeline = room == nullptr ? nullptr : object_member_as_object(*room, "timeline"); + auto const* events = timeline == nullptr ? nullptr : object_member_as_array(*timeline, "events"); + if (events == nullptr) + { + return false; + } + for (auto const& entry : *events) + { + auto const* object = std::get_if(&entry.storage()); + if (object != nullptr && string_member(*object, "type") != nullptr && *string_member(*object, "type") == type) + { + return true; + } + } + return false; +} + +[[nodiscard]] auto state_content(Fixture& fixture, std::string_view type, std::string_view key = {}) + -> hs::LocalHttpResponse +{ + return request(fixture.runtime(), "GET", + fixture.room_path() + "/state/" + std::string{type} + (key.empty() ? "" : "/" + std::string{key}), + fixture.alice_token); +} + +} // namespace + +SCENARIO("Every scheduling form answers a delay ID, sends nothing early and sends the event at its due time", + "[homeserver][delayed-events][routes][schedule]") +{ + GIVEN("alice in a room, and one of the six forms that schedule an event") + { + auto const form = GENERATE(0, 1, 2, 3, 4, 5); + auto fixture = Fixture{}; + auto const is_state = form >= 3; + auto const event_type = std::string{is_state ? "m.test.state" : "m.test.delayed"}; + auto const before = fixture.event_count(); + auto const dedicated_body = schedule_body(30'000U, is_state ? std::string_view{R"({"v":1})"} : message_content); + + WHEN("it is scheduled for 30 seconds") + { + auto const response = [&]() -> hs::LocalHttpResponse { + switch (form) + { + case 0: + return put_dedicated(fixture, true, fixture.alice_token, event_type, "t1", dedicated_body); + case 1: + return put_dedicated(fixture, false, fixture.alice_token, event_type, "t1", dedicated_body); + case 2: + return put_send_delayed(fixture, fixture.alice_token, event_type, "t1", + std::string{delay_param} + "=30000"); + case 3: + return put_state_delayed(fixture, fixture.alice_token, event_type, "/key", + std::string{delay_param} + "=30000"); + case 4: + return put_state_delayed(fixture, fixture.alice_token, event_type, "/", + std::string{delay_param} + "=30000"); + default: + return put_state_delayed(fixture, fixture.alice_token, event_type, "", + std::string{delay_param} + "=30000"); + } + }(); + auto const id = delay_id_of(response); + + THEN("the answer is a delay_id, and nothing is sent before the due time") + { + CHECK(response.body.find("event_id") == std::string::npos); + CHECK(fixture.event_count() == before); + fixture.run_job(t0 + 29'999U); + CHECK(fixture.event_count() == before); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + AND_THEN("the job sends it at the due time and it is in /sync") + { + fixture.set_time(t0 + 30'000U); + fixture.run_job(t0 + 30'000U); + CHECK(fixture.event_count() == before + 1U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(timeline_has(fixture, event_type)); + } + } + } +} + +SCENARIO("A dedicated route with a state_key schedules a state event, even with an empty key", + "[homeserver][delayed-events][routes][schedule]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + auto const state_key = GENERATE(std::string{"k"}, std::string{}); + + WHEN("S1 is given a state_key member") + { + auto const body = R"({"delay_ms":5000,"content":{"v":2},"state_key":")" + state_key + R"("})"; + auto const id = delay_id_of(put_dedicated(fixture, true, fixture.alice_token, "m.test.state", "s1", body)); + + THEN("the lookup shows a state event, and it replaces the state when due") + { + auto const found = parse_object(get_delayed(fixture, true, id, fixture.alice_token).body); + REQUIRE(string_member(found, "state_key") != nullptr); + CHECK(*string_member(found, "state_key") == state_key); + fixture.set_time(t0 + 5'000U); + fixture.run_job(t0 + 5'000U); + auto const current = state_content(fixture, "m.test.state", state_key); + CHECK(current.status == 200U); + CHECK(current.body.find("\"v\":2") != std::string::npos); + } + } + } +} + +SCENARIO("A call membership leave scheduled on the state route survives the join and is moved by a restart", + "[homeserver][delayed-events][routes][element-x]") +{ + GIVEN("alice scheduling the empty call.member for her device with Element X's requests") + { + auto fixture = Fixture{}; + auto const type = std::string{"org.matrix.msc3401.call.member"}; + auto const key = std::string{"_%40alice%3Aexample.org_D1"}; + auto const scheduled = + put_state_delayed(fixture, fixture.alice_token, type, "/" + key, std::string{delay_param} + "=30000", "{}"); + auto const id = delay_id_of(scheduled); + + WHEN("the join is sent normally and the delayed event is restarted twenty seconds in") + { + auto const join = request(fixture.runtime(), "PUT", fixture.room_path() + "/state/" + type + "/" + key, + fixture.alice_token, R"({"application":"m.call","call_id":""})"); + fixture.set_time(t0 + 20'000U); + auto const restart = post_body_action(fixture, id, R"({"action":"restart"})", fixture.alice_token); + + THEN("the join did not cancel the leave, and the leave comes at the new due time") + { + CHECK(join.status == 200U); + CHECK(restart.status == 200U); + CHECK(restart.body == "{}"); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + fixture.run_job(t0 + 30'000U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + CHECK(state_content(fixture, type, "_%40alice%3Aexample.org_D1").body.find("m.call") != + std::string::npos); + fixture.set_time(t0 + 50'000U); + fixture.run_job(t0 + 50'000U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + auto const current = state_content(fixture, type, "_%40alice%3Aexample.org_D1"); + CHECK(current.status == 200U); + CHECK(current.body == "{}"); + } + } + } +} + +SCENARIO("The Element Web flow schedules on the unstable path, restarts and then sends on the path form", + "[homeserver][delayed-events][routes][element-web]") +{ + GIVEN("alice with a message scheduled by the unstable dedicated path") + { + auto fixture = Fixture{}; + auto const id = schedule_message(fixture, "web"); + auto const before = fixture.event_count(); + + WHEN("she restarts it and then sends it") + { + auto const restart = post_action(fixture, false, id, "restart", fixture.alice_token); + auto const sent = post_action(fixture, false, id, "send", fixture.alice_token); + + THEN("both answer 200 {} and the event is sent at once") + { + CHECK(restart.status == 200U); + CHECK(restart.body == "{}"); + CHECK(sent.status == 200U); + CHECK(sent.body == "{}"); + CHECK(fixture.event_count() == before + 1U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(timeline_has(fixture, "m.test.delayed")); + } + } + } +} + +SCENARIO("A request without the delay parameter sends at once as before", "[homeserver][delayed-events][routes][send]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + auto const before = fixture.event_count(); + + WHEN("she sends a message and a state event with no delay parameter") + { + auto const sent = request(fixture.runtime(), "PUT", fixture.room_path() + "/send/m.test.delayed/p1", + fixture.alice_token, std::string{message_content}); + auto const state = request(fixture.runtime(), "PUT", fixture.room_path() + "/state/m.test.state/k", + fixture.alice_token, R"({"v":1})"); + + THEN("each answers an event_id and is stored") + { + CHECK(sent.status == 200U); + CHECK(sent.body.find("event_id") != std::string::npos); + CHECK(state.status == 200U); + CHECK(state.body.find("event_id") != std::string::npos); + CHECK(fixture.event_count() == before + 2U); + } + } + } +} + +SCENARIO("With delayed events disabled every scheduling form is 403 and nothing is sent", + "[homeserver][delayed-events][routes][disabled]") +{ + GIVEN("a server with the feature switched off") + { + auto fixture = Fixture{}; + fixture.runtime().limits.delayed_events_enabled = false; + auto const before = fixture.event_count(); + auto const form = GENERATE(0, 1, 2, 3, 4); + + WHEN("a form is used, with a body or parameter that would also be refused") + { + auto const response = [&]() -> hs::LocalHttpResponse { + switch (form) + { + case 0: + return put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "t", "not even json{"); + case 1: + return put_dedicated(fixture, false, fixture.alice_token, "m.test.delayed", "t", "{}"); + case 2: + return put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "t", + std::string{delay_param} + "=banana"); + case 3: + return put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", + std::string{delay_param} + "=5000"); + default: + return put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "t", + std::string{delay_param} + "=5000"); + } + }(); + + THEN("the answer is 403 M_FORBIDDEN, or the gate's M_NOT_JSON for a body that is not JSON, and no " + "event is sent") + { + if (form == 0) + { + CHECK(response.status == 400U); + CHECK(errcode_of(response) == "M_NOT_JSON"); + } + else + { + CHECK(response.status == 403U); + CHECK(errcode_of(response) == "M_FORBIDDEN"); + } + CHECK(fixture.event_count() == before); + CHECK(state_content(fixture, "m.test.state", "k").status == 404U); + } + } + + WHEN("a disabled server is asked for its dedicated routes with a valid body") + { + auto const s1 = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "d1", schedule_body(5'000U)); + auto const s2 = + put_dedicated(fixture, false, fixture.alice_token, "m.test.delayed", "d2", schedule_body(5'000U)); + + THEN("both are 403 M_FORBIDDEN") + { + CHECK(s1.status == 403U); + CHECK(errcode_of(s1) == "M_FORBIDDEN"); + CHECK(s2.status == 403U); + CHECK(errcode_of(s2) == "M_FORBIDDEN"); + } + } + } +} + +SCENARIO("A delay parameter that is not one whole number of at least 1 is refused and nothing is sent", + "[homeserver][delayed-events][routes][invalid-param]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + auto const before = fixture.event_count(); + auto const query = GENERATE( + std::string{"org.matrix.msc4140.delay="}, std::string{"org.matrix.msc4140.delay"}, + std::string{"org.matrix.msc4140.delay=abc"}, std::string{"org.matrix.msc4140.delay=+5"}, + std::string{"org.matrix.msc4140.delay=5.0"}, std::string{"org.matrix.msc4140.delay=%205"}, + std::string{"org.matrix.msc4140.delay=0x10"}, std::string{"org.matrix.msc4140.delay=-1"}, + std::string{"org.matrix.msc4140.delay=0"}, std::string{"org.matrix.msc4140.delay=99999999999999999999999"}, + std::string{"org.matrix.msc4140.delay=18446744073709551616"}, + std::string{"org.matrix.msc4140.delay=5000&org.matrix.msc4140.delay=6000"}, + std::string{"org.matrix.msc4140.delay=5000&ts=1&org.matrix.msc4140.delay=5000"}, + std::string{"org%2Ematrix.msc4140.delay=banana"}); + + WHEN("the send route and the state route are called with it") + { + auto const send = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "bad", query); + auto const state = put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", query); + + THEN("both answer 400 M_INVALID_PARAM, schedule nothing and send nothing") + { + INFO(query); + CHECK(send.status == 400U); + CHECK(errcode_of(send) == "M_INVALID_PARAM"); + CHECK(state.status == 400U); + CHECK(errcode_of(state) == "M_INVALID_PARAM"); + CHECK(fixture.event_count() == before); + } + } + } +} + +SCENARIO("The delay parameter is read with leading zeros and an encoded name, and the body must still be an object", + "[homeserver][delayed-events][routes][invalid-param]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + auto const before = fixture.event_count(); + + WHEN("the delay is written 007 and the parameter name percent-encoded") + { + auto const padded = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "z1", + std::string{delay_param} + "=0007"); + auto const encoded = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "z2", + "org%2Ematrix.msc4140.delay=5000"); + auto const not_object = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "z3", + std::string{delay_param} + "=5000", "[]"); + + THEN("the first two are scheduled, the third is M_BAD_JSON, and nothing is sent") + { + CHECK(delay_id_of(padded).size() > 0U); + CHECK(delay_id_of(encoded).size() > 0U); + CHECK(not_object.status == 400U); + CHECK(errcode_of(not_object) == "M_BAD_JSON"); + CHECK(fixture.event_count() == before); + } + } + } +} + +SCENARIO("The dedicated routes refuse a body that is not the documented object with M_BAD_JSON", + "[homeserver][delayed-events][routes][bad-body]") +{ + GIVEN("alice in a room and a body") + { + auto fixture = Fixture{}; + auto const body = GENERATE( + std::string{}, std::string{" "}, std::string{"{}"}, std::string{"[]"}, std::string{"5"}, + std::string{R"("text")"}, std::string{R"({"content":{}})"}, std::string{R"({"delay_ms":5000})"}, + std::string{R"({"delay_ms":"5000","content":{}})"}, std::string{R"({"delay_ms":5000.5,"content":{}})"}, + std::string{R"({"delay_ms":5000.0,"content":{}})"}, std::string{R"({"delay_ms":true,"content":{}})"}, + std::string{R"({"delay_ms":null,"content":{}})"}, std::string{R"({"delay_ms":-5,"content":{}})"}, + std::string{R"({"delay_ms":0,"content":{}})"}, std::string{R"({"delay_ms":5000,"content":[]})"}, + std::string{R"({"delay_ms":5000,"content":"x"})"}, + std::string{R"({"delay_ms":5000,"content":{},"state_key":5})"}, + std::string{R"({"delay_ms":5000,"content":{},"state_key":null})"}); + auto const before = fixture.event_count(); + + WHEN("each dedicated route is called with it") + { + auto const s1 = put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "b1", body); + auto const s2 = put_dedicated(fixture, false, fixture.alice_token, "m.test.delayed", "b2", body); + + THEN("both answer 400 M_BAD_JSON and nothing is stored") + { + INFO(body); + CHECK(s1.status == 400U); + CHECK(errcode_of(s1) == "M_BAD_JSON"); + CHECK(s2.status == 400U); + CHECK(errcode_of(s2) == "M_BAD_JSON"); + CHECK(fixture.event_count() == before); + } + } + } +} + +SCENARIO("The dedicated routes answer the gate's M_NOT_JSON for a body that is not JSON and ignore unknown members", + "[homeserver][delayed-events][routes][bad-body]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + + WHEN("a body that is not JSON, and a body with unknown members, are sent") + { + auto const garbage = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "n1", "{\"delay_ms\""); + auto const extra = put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "n2", + R"({"delay_ms":5000,"content":{},"sticky_duration_ms":7,"extra":[1]})"); + + THEN("the first is 400 M_NOT_JSON and the second schedules") + { + CHECK(garbage.status == 400U); + CHECK(errcode_of(garbage) == "M_NOT_JSON"); + CHECK(delay_id_of(extra).size() > 0U); + } + } + } +} + +SCENARIO("A delay over the maximum names the stable errcode on S1 and the unstable one everywhere else", + "[homeserver][delayed-events][routes][limits]") +{ + GIVEN("a maximum delay of one hour") + { + auto fixture = Fixture{}; + fixture.runtime().limits.max_delayed_event_delay_ms = 3'600'000U; + auto const before = fixture.event_count(); + + WHEN("each form asks for an hour and a millisecond") + { + auto const s1 = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "l1", schedule_body(3'600'001U)); + auto const s2 = + put_dedicated(fixture, false, fixture.alice_token, "m.test.delayed", "l2", schedule_body(3'600'001U)); + auto const l1 = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "l3", + std::string{delay_param} + "=3600001"); + auto const l2 = put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", + std::string{delay_param} + "=3600001"); + auto const exact = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "l4", + std::string{delay_param} + "=3600000"); + + THEN("each is 400 with its errcode, and the maximum itself is accepted") + { + CHECK(s1.status == 400U); + CHECK(errcode_of(s1) == "M_DELAY_TOO_LARGE"); + for (auto const* other : {&s2, &l1, &l2}) + { + CHECK(other->status == 400U); + CHECK(errcode_of(*other) == "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE"); + } + CHECK(delay_id_of(exact).size() > 0U); + CHECK(fixture.event_count() == before); + } + } + } +} + +SCENARIO("A user at the scheduled-event cap is answered 429 with Retry-After and retry_after_ms", + "[homeserver][delayed-events][routes][limits]") +{ + GIVEN("a cap of one scheduled event and alice holding one") + { + auto fixture = Fixture{}; + fixture.runtime().limits.max_scheduled_delayed_events_per_user = 1U; + std::ignore = schedule_message(fixture, "c1", 60'000U); + + WHEN("she schedules another by each form") + { + auto const s1 = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "c2", schedule_body(5'000U)); + auto const l1 = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "c3", + std::string{delay_param} + "=5000"); + + THEN("both are 429 M_LIMIT_EXCEEDED with the header and the body field") + { + for (auto const* denied : {&s1, &l1}) + { + CHECK(denied->status == 429U); + CHECK(errcode_of(*denied) == "M_LIMIT_EXCEEDED"); + auto const denied_body = parse_object(denied->body); + auto const* retry = int_member(denied_body, "retry_after_ms"); + REQUIRE(retry != nullptr); + CHECK(*retry == 60'000); + CHECK(header_of(*denied, "Retry-After") == "60"); + } + } + } + } +} + +SCENARIO("Management routes take an empty body, an empty object, unknown members, and answer the gate for non-JSON", + "[homeserver][delayed-events][routes][manage]") +{ + GIVEN("alice with a scheduled message and a body") + { + auto fixture = Fixture{}; + auto const id = schedule_message(fixture, "m1"); + auto const stable = GENERATE(true, false); + auto const body = + GENERATE(std::string{}, std::string{"{}"}, std::string{R"({"unknown":[1,2],"action":"cancel"})"}); + + WHEN("restart is posted with that body") + { + auto const response = post_action(fixture, stable, id, "restart", fixture.alice_token, body); + + THEN("it is 200 {} and the action is the path's: the event is still scheduled") + { + CHECK(response.status == 200U); + CHECK(response.body == "{}"); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + + WHEN("restart is posted with a body that is not JSON") + { + auto const response = post_action(fixture, stable, id, "restart", fixture.alice_token, "not json"); + + THEN("the gate answers 400 M_NOT_JSON before the route runs") + { + CHECK(response.status == 400U); + CHECK(errcode_of(response) == "M_NOT_JSON"); + } + } + } +} + +SCENARIO("Each management action follows the outcome table on both prefixes", + "[homeserver][delayed-events][routes][manage]") +{ + GIVEN("alice with delayed messages in each state and a prefix") + { + auto fixture = Fixture{}; + auto const stable = GENERATE(true, false); + auto const scheduled = schedule_message(fixture, "o1"); + auto const sent = schedule_message(fixture, "o2"); + auto const cancelled = schedule_message(fixture, "o3"); + REQUIRE(post_action(fixture, stable, sent, "send", fixture.alice_token).status == 200U); + REQUIRE(post_action(fixture, stable, cancelled, "cancel", fixture.alice_token).status == 200U); + auto const& token = fixture.alice_token; + + WHEN("each action is posted to each") + { + THEN("the answers are the ADR-0823 table's, 409 with M_UNKNOWN where it says so") + { + CHECK(post_action(fixture, stable, scheduled, "restart", token).status == 200U); + CHECK(post_action(fixture, stable, sent, "send", token).status == 200U); + for (auto const* action : {"restart", "cancel"}) + { + auto const response = post_action(fixture, stable, sent, action, token); + CHECK(response.status == 409U); + CHECK(errcode_of(response) == "M_UNKNOWN"); + } + CHECK(post_action(fixture, stable, cancelled, "cancel", token).status == 200U); + for (auto const* action : {"restart", "send"}) + { + auto const response = post_action(fixture, stable, cancelled, action, token); + CHECK(response.status == 409U); + CHECK(errcode_of(response) == "M_UNKNOWN"); + } + CHECK(post_action(fixture, stable, scheduled, "cancel", token).status == 200U); + CHECK(status_of(fixture, alice, scheduled) == db::DelayedEventStatus::cancelled); + } + } + } +} + +SCENARIO("A last path segment that is not an action is not a route", "[homeserver][delayed-events][routes][manage]") +{ + GIVEN("alice with a scheduled message") + { + auto fixture = Fixture{}; + auto const id = schedule_message(fixture, "u1"); + + WHEN("an unknown action, a trailing slash and a deeper path are posted on both prefixes") + { + THEN("each is the dispatcher's 404 M_UNRECOGNIZED and the event is untouched") + { + for (auto const base : {v1, unstable}) + { + for (auto const* tail : {"/destroy", "/send/", "/send/more", "/", ""}) + { + auto const path = std::string{base} + id + tail; + auto const response = request(fixture.runtime(), "POST", path, fixture.alice_token, "{}"); + INFO(path); + if (std::string_view{tail}.empty()) + { + // `/v1/delayed_events/{id}` is no route (405) and the unstable one is M3 (400 missing + // action). + CHECK(response.status == (base == v1 ? 405U : 400U)); + } + else + { + CHECK(response.status == 404U); + CHECK(errcode_of(response) == "M_UNRECOGNIZED"); + } + } + } + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + } +} + +SCENARIO("The action in the body of the unstable POST selects the operation", + "[homeserver][delayed-events][routes][m3]") +{ + GIVEN("alice with delayed messages") + { + auto fixture = Fixture{}; + auto const restart_id = schedule_message(fixture, "p1"); + auto const send_id = schedule_message(fixture, "p2"); + auto const cancel_id = schedule_message(fixture, "p3"); + auto const before = fixture.event_count(); + + WHEN("each action is posted") + { + auto const restart = post_body_action(fixture, restart_id, R"({"action":"restart"})", fixture.alice_token); + auto const sent = post_body_action(fixture, send_id, R"({"action":"send","extra":1})", fixture.alice_token); + auto const cancel = post_body_action(fixture, cancel_id, R"({"action":"cancel"})", fixture.alice_token); + + THEN("each acts and answers 200 {}") + { + CHECK(restart.status == 200U); + CHECK(restart.body == "{}"); + CHECK(sent.status == 200U); + CHECK(cancel.status == 200U); + CHECK(status_of(fixture, alice, restart_id) == db::DelayedEventStatus::scheduled); + CHECK(status_of(fixture, alice, send_id) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, cancel_id) == db::DelayedEventStatus::cancelled); + CHECK(fixture.event_count() == before + 1U); + } + } + + WHEN("the body has no action, an unknown action, a non-string action or is not an object") + { + auto const missing = post_body_action(fixture, restart_id, R"({"other":"send"})", fixture.alice_token); + auto const unknown = post_body_action(fixture, restart_id, R"({"action":"explode"})", fixture.alice_token); + auto const number = post_body_action(fixture, restart_id, R"({"action":5})", fixture.alice_token); + auto const array = post_body_action(fixture, restart_id, R"([])", fixture.alice_token); + auto const empty = post_body_action(fixture, restart_id, "", fixture.alice_token); + + THEN("the answers are M_MISSING_PARAM, M_INVALID_PARAM and M_BAD_JSON, and nothing happens") + { + CHECK(missing.status == 400U); + CHECK(errcode_of(missing) == "M_MISSING_PARAM"); + CHECK(unknown.status == 400U); + CHECK(errcode_of(unknown) == "M_INVALID_PARAM"); + CHECK(number.status == 400U); + CHECK(errcode_of(number) == "M_INVALID_PARAM"); + CHECK(array.status == 400U); + CHECK(errcode_of(array) == "M_BAD_JSON"); + CHECK(empty.status == 400U); + CHECK(errcode_of(empty) == "M_BAD_JSON"); + CHECK(status_of(fixture, alice, restart_id) == db::DelayedEventStatus::scheduled); + CHECK(fixture.event_count() == before); + } + } + + WHEN("the same body is posted to the stable prefix") + { + auto const response = request(fixture.runtime(), "POST", std::string{v1} + restart_id, fixture.alice_token, + R"({"action":"restart"})"); + + THEN("there is no such route: 405 M_UNRECOGNIZED, the dispatcher's answer for a method a path lacks") + { + CHECK(response.status == 405U); + CHECK(errcode_of(response) == "M_UNRECOGNIZED"); + } + } + } +} + +SCENARIO("A failed send action answers what the ordinary send answers and leaves the event scheduled", + "[homeserver][delayed-events][routes][manage]") +{ + GIVEN("bob with a delayed message, who then leaves the room") + { + auto fixture = Fixture{}; + auto const message_id = schedule_message(fixture, "f1", 60'000U, fixture.bob_token); + REQUIRE(request(fixture.runtime(), "POST", fixture.room_path() + "/leave", fixture.bob_token, "{}").status == + 200U); + + WHEN("the send action is posted on each route form, and an ordinary send of the same event is made") + { + auto const stable = post_action(fixture, true, message_id, "send", fixture.bob_token); + auto const unstable_form = post_action(fixture, false, message_id, "send", fixture.bob_token); + auto const body_form = post_body_action(fixture, message_id, R"({"action":"send"})", fixture.bob_token); + auto const ordinary = request(fixture.runtime(), "PUT", fixture.room_path() + "/send/m.test.delayed/o-1", + fixture.bob_token, std::string{message_content}); + + THEN("the status and errcode are the ordinary route's, and the event is still scheduled") + { + CHECK(ordinary.status >= 400U); + for (auto const* delayed : {&stable, &unstable_form, &body_form}) + { + CHECK(delayed->status == ordinary.status); + CHECK(errcode_of(*delayed) == errcode_of(ordinary)); + } + CHECK(status_of(fixture, bob, message_id) == db::DelayedEventStatus::scheduled); + } + } + } +} + +SCENARIO("Lookup shows a scheduled, a sent, a cancelled and a failed event on both prefixes", + "[homeserver][delayed-events][routes][lookup]") +{ + GIVEN("alice's events in the four states, the failed one bob's") + { + auto fixture = Fixture{}; + auto const scheduled = schedule_message(fixture, "g1", 60'000U); + auto const sent = schedule_message(fixture, "g2", 60'000U); + auto const cancelled = schedule_message(fixture, "g3", 60'000U); + auto const failed = schedule_message(fixture, "g4", 10'000U, fixture.bob_token); + REQUIRE(post_action(fixture, true, sent, "send", fixture.alice_token).status == 200U); + REQUIRE(post_action(fixture, true, cancelled, "cancel", fixture.alice_token).status == 200U); + REQUIRE(request(fixture.runtime(), "POST", fixture.room_path() + "/leave", fixture.bob_token, "{}").status == + 200U); + fixture.set_time(t0 + 10'000U); + fixture.run_job(t0 + 10'000U); + + WHEN("each is looked up on both prefixes") + { + auto const body = [&](std::string const& id, std::string const& token) { + auto const stable = get_delayed(fixture, true, id, token); + auto const other = get_delayed(fixture, false, id, token); + REQUIRE(stable.status == 200U); + REQUIRE(other.status == 200U); + REQUIRE(stable.body == other.body); + return parse_object(stable.body); + }; + auto const found_scheduled = body(scheduled, fixture.alice_token); + auto const found_sent = body(sent, fixture.alice_token); + auto const found_cancelled = body(cancelled, fixture.alice_token); + auto const found_failed = body(failed, fixture.bob_token); + + THEN("the members are the service's") + { + CHECK(*string_member(found_scheduled, "delay_id") == scheduled); + CHECK(*string_member(found_scheduled, "room_id") == fixture.room_id); + CHECK(*string_member(found_scheduled, "type") == "m.test.delayed"); + CHECK(string_member(found_scheduled, "state_key") == nullptr); + CHECK(*int_member(found_scheduled, "delay_ms") == 60'000); + CHECK(*int_member(found_scheduled, "delayed_since_ts") == static_cast(t0)); + CHECK(object_member_as_object(found_scheduled, "content") != nullptr); + CHECK(object_member(found_scheduled, "finalised") == nullptr); + + auto const* sent_final = object_member_as_object(found_sent, "finalised"); + REQUIRE(sent_final != nullptr); + CHECK(string_member(*sent_final, "event_id") != nullptr); + CHECK(int_member(*sent_final, "finalised_ts") != nullptr); + CHECK(string_member(*sent_final, "error") == nullptr); + + auto const* cancelled_final = object_member_as_object(found_cancelled, "finalised"); + REQUIRE(cancelled_final != nullptr); + CHECK(string_member(*cancelled_final, "event_id") == nullptr); + CHECK(object_member(*cancelled_final, "error") == nullptr); + + auto const* failed_final = object_member_as_object(found_failed, "finalised"); + REQUIRE(failed_final != nullptr); + CHECK(string_member(*failed_final, "event_id") == nullptr); + CHECK(object_member_as_object(*failed_final, "error") != nullptr); + } + } + } +} + +SCENARIO("An unknown delay ID and another user's get the same 404 on every management and lookup route", + "[homeserver][delayed-events][routes][not-found]") +{ + GIVEN("alice's scheduled event and bob, who asks about it and about an ID that does not exist") + { + auto fixture = Fixture{}; + auto const alice_id = schedule_message(fixture, "x1"); + auto const due_before = db::delayed_event_due_ms(*fixture.table().find_owned(alice, alice_id)); + struct Route final + { + std::string method; + std::string tail; + std::string body; + bool stable; + }; + auto const routes = std::vector{ + {"POST", "/send", "", true }, + {"POST", "/cancel", "", true }, + {"POST", "/restart", "{}", true }, + {"POST", "/send", "", false}, + {"POST", "/cancel", "{}", false}, + {"POST", "/restart", "", false}, + {"POST", "", R"({"action":"send"})", false}, + {"POST", "", R"({"action":"cancel"})", false}, + {"POST", "", R"({"action":"restart"})", false}, + {"GET", "", "", true }, + {"GET", "", "", false}, + }; + + WHEN("bob calls every route with each ID") + { + THEN("the two answers are equal in status, body and headers, and alice's event is untouched") + { + for (auto const& route : routes) + { + auto const base = std::string{route.stable ? v1 : unstable}; + auto const unknown = request(fixture.runtime(), route.method, + base + "no-such-delay-id" + route.tail, fixture.bob_token, route.body); + auto const foreign = request(fixture.runtime(), route.method, base + alice_id + route.tail, + fixture.bob_token, route.body); + INFO(route.method << " " << route.tail << (route.stable ? " stable" : " unstable")); + CHECK(unknown.status == 404U); + CHECK(errcode_of(unknown) == "M_NOT_FOUND"); + CHECK(foreign.status == unknown.status); + CHECK(foreign.body == unknown.body); + CHECK(foreign.headers == unknown.headers); + } + CHECK(status_of(fixture, alice, alice_id) == db::DelayedEventStatus::scheduled); + CHECK(db::delayed_event_due_ms(*fixture.table().find_owned(alice, alice_id)) == due_before); + } + } + } +} + +SCENARIO("Every route needs a valid token and a suspended or locked account gets the gate's answer", + "[homeserver][delayed-events][routes][auth]") +{ + GIVEN("alice with a scheduled event") + { + auto fixture = Fixture{}; + auto const id = schedule_message(fixture, "a1"); + struct Route final + { + std::string method; + std::string target; + std::string body; + }; + auto const room = fixture.room_path(); + auto const routes = std::vector{ + {"PUT", room + "/delayed_event/m.test.delayed/a2", schedule_body(5'000U) }, + {"PUT", + "/_matrix/client/unstable/org.matrix.msc4140/rooms/" + fixture.room_id + + "/delayed_event/m.test.delayed/a3", schedule_body(5'000U) }, + {"PUT", room + "/send/m.test.delayed/a4?org.matrix.msc4140.delay=5000", std::string{message_content}}, + {"PUT", room + "/state/m.test.state/k?org.matrix.msc4140.delay=5000", R"({"v":1})" }, + {"POST", std::string{v1} + id + "/restart", "" }, + {"POST", std::string{unstable} + id + "/cancel", "" }, + {"POST", std::string{unstable} + id, R"({"action":"restart"})" }, + {"GET", std::string{v1} + id, "" }, + {"GET", std::string{unstable} + id, "" }, + }; + + WHEN("the routes are called with no token and with a token the server does not know") + { + THEN("they are 401 M_MISSING_TOKEN and 401 M_UNKNOWN_TOKEN") + { + for (auto const& route : routes) + { + INFO(route.method << " " << route.target); + auto const none = request(fixture.runtime(), route.method, route.target, {}, route.body); + auto const bad = + request(fixture.runtime(), route.method, route.target, "not-a-real-token", route.body); + CHECK(none.status == 401U); + CHECK(errcode_of(none) == "M_MISSING_TOKEN"); + CHECK(bad.status == 401U); + CHECK(errcode_of(bad) == "M_UNKNOWN_TOKEN"); + } + } + } + + WHEN("alice is suspended, or locked") + { + auto const locked = GENERATE(false, true); + auto& homeserver = fixture.runtime().homeserver; + REQUIRE(hs::bootstrap_admin_user(homeserver, "admin", password).ok); + REQUIRE( + hs::set_local_user_moderation(homeserver, "@admin:example.org", alice, !locked, locked, "t", "t").ok); + + THEN("every route is the moderation gate's refusal and nothing changes") + { + for (auto const& route : routes) + { + INFO(route.method << " " << route.target); + auto const response = + request(fixture.runtime(), route.method, route.target, fixture.alice_token, route.body); + CHECK(response.status == (locked ? 401U : 403U)); + CHECK(errcode_of(response) == (locked ? "M_USER_LOCKED" : "M_USER_SUSPENDED")); + } + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + } +} + +SCENARIO("An application service acting for a user manages that user's delayed event and no one else's", + "[homeserver][delayed-events][routes][appservice]") +{ + GIVEN("an application service whose namespace holds alice and bob, and alice's delayed event") + { + auto bridge = TestAppservice{.id = "bridge", .users_regex = "@.*:example.org"}; + auto fixture = Fixture{appservice_test_config({bridge})}; + auto const token = bridge.as_token(); + auto const id = schedule_message(fixture, "as1"); + auto const as_alice = std::string{"?user_id=@alice:example.org"}; + auto const as_bob = std::string{"?user_id=@bob:example.org"}; + auto const before = fixture.event_count(); + + WHEN("it restarts and then sends alice's event as alice") + { + auto const restart = + request(fixture.runtime(), "POST", std::string{unstable} + id + "/restart" + as_alice, token); + auto const sent = + request(fixture.runtime(), "POST", std::string{unstable} + id + "/send" + as_alice, token); + + THEN("both are 200 and the event is sent as alice") + { + CHECK(restart.status == 200U); + CHECK(sent.status == 200U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 1U); + } + } + + WHEN("it acts as bob on alice's event, and as alice on a lookup") + { + auto const restart = + request(fixture.runtime(), "POST", std::string{unstable} + id + "/restart" + as_bob, token); + auto const cancel = + request(fixture.runtime(), "POST", std::string{unstable} + id + "/cancel" + as_bob, token); + auto const lookup = request(fixture.runtime(), "GET", std::string{unstable} + id + as_alice, token); + + THEN("bob's attempts are 404 M_NOT_FOUND and alice's event is untouched, while her lookup works") + { + CHECK(restart.status == 404U); + CHECK(errcode_of(restart) == "M_NOT_FOUND"); + CHECK(cancel.status == 404U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + CHECK(lookup.status == 200U); + } + } + } +} + +SCENARIO("/versions and /capabilities advertise delayed events with the configured limits", + "[homeserver][delayed-events][routes][advertisement]") +{ + GIVEN("a server with the feature on, or off") + { + auto fixture = Fixture{}; + auto const enabled = GENERATE(true, false); + fixture.runtime().limits.delayed_events_enabled = enabled; + fixture.runtime().limits.max_delayed_event_delay_ms = 7'200'000U; + fixture.runtime().limits.max_scheduled_delayed_events_per_user = 17U; + + WHEN("/versions and /capabilities are read") + { + auto const versions = parse_object(request(fixture.runtime(), "GET", "/_matrix/client/versions").body); + auto const capabilities_response = + request(fixture.runtime(), "GET", "/_matrix/client/v3/capabilities", fixture.alice_token); + REQUIRE(capabilities_response.status == 200U); + auto const capabilities = + *object_member_as_object(parse_object(capabilities_response.body), "capabilities"); + + THEN("both flags equal enabled, and both capabilities carry the limits, or zeros when disabled") + { + auto const* features = object_member_as_object(versions, "unstable_features"); + REQUIRE(features != nullptr); + for (auto const* flag : {"org.matrix.msc4140", "org.matrix.msc4140.stable"}) + { + auto const* value = bool_member(*features, flag); + REQUIRE(value != nullptr); + CHECK(*value == enabled); + } + for (auto const* name : {"m.delayed_events", "org.matrix.msc4140.delayed_events"}) + { + auto const* entry = object_member_as_object(capabilities, name); + REQUIRE(entry != nullptr); + REQUIRE(int_member(*entry, "max_delay_ms") != nullptr); + REQUIRE(int_member(*entry, "max_scheduled") != nullptr); + CHECK(*int_member(*entry, "max_delay_ms") == (enabled ? 7'200'000 : 0)); + CHECK(*int_member(*entry, "max_scheduled") == (enabled ? 17 : 0)); + } + } + } + } +} + +SCENARIO("The built-in per-user limits stop the 121st management request and the 61st scheduling request", + "[homeserver][delayed-events][routes][rate-limit]") +{ + GIVEN("alice and bob on one server with the default rate limits") + { + auto fixture = Fixture{}; + fixture.runtime().limits.max_scheduled_delayed_events_per_user = 1000U; + + WHEN("alice makes 121 lookups of an ID that does not exist") + { + auto statuses = std::vector{}; + for (auto index = 0; index < 121; ++index) + { + statuses.push_back(get_delayed(fixture, true, "none", fixture.alice_token).status); + } + auto const bob_answer = get_delayed(fixture, true, "none", fixture.bob_token); + + THEN("the first 120 are 404, the 121st is 429 M_LIMIT_EXCEEDED, and bob is not limited") + { + for (auto index = std::size_t{0U}; index < 120U; ++index) + { + CHECK(statuses[index] == 404U); + } + CHECK(statuses[120] == 429U); + CHECK(bob_answer.status == 404U); + } + } + + WHEN("alice schedules 61 events on the dedicated route") + { + auto statuses = std::vector{}; + auto last = hs::LocalHttpResponse{}; + for (auto index = 0; index < 61; ++index) + { + last = put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "r" + std::to_string(index), + schedule_body(60'000U)); + statuses.push_back(last.status); + } + auto const bob_answer = + put_dedicated(fixture, true, fixture.bob_token, "m.test.delayed", "r0", schedule_body(60'000U)); + + THEN("the first 60 are scheduled, the 61st is 429 M_LIMIT_EXCEEDED, and bob is not limited") + { + for (auto index = std::size_t{0U}; index < 60U; ++index) + { + CHECK(statuses[index] == 200U); + } + CHECK(statuses[60] == 429U); + CHECK(errcode_of(last) == "M_LIMIT_EXCEEDED"); + CHECK(bob_answer.status == 200U); + } + } + } +} + +SCENARIO("A method a delayed-event path does not serve, and the listing, get the dispatcher's standard answers", + "[homeserver][delayed-events][routes][methods]") +{ + GIVEN("alice with a scheduled event") + { + auto fixture = Fixture{}; + auto const id = schedule_message(fixture, "w1"); + + WHEN("the wrong methods are used, and the listing is asked for") + { + auto const get_schedule = + request(fixture.runtime(), "GET", fixture.room_path() + "/delayed_event/m.test.delayed/w2", + fixture.alice_token); + auto const post_schedule = + request(fixture.runtime(), "POST", fixture.room_path() + "/delayed_event/m.test.delayed/w2", + fixture.alice_token, schedule_body(5'000U)); + auto const delete_event = request(fixture.runtime(), "DELETE", std::string{v1} + id, fixture.alice_token); + auto const put_event = + request(fixture.runtime(), "PUT", std::string{unstable} + id, fixture.alice_token, "{}"); + auto const get_action = + request(fixture.runtime(), "GET", std::string{v1} + id + "/send", fixture.alice_token); + auto const put_action = + request(fixture.runtime(), "PUT", std::string{v1} + id + "/send", fixture.alice_token, "{}"); + auto const list_stable = + request(fixture.runtime(), "GET", std::string{v1}.substr(0U, v1.size() - 1U), fixture.alice_token); + auto const list_unstable = request( + fixture.runtime(), "GET", std::string{unstable}.substr(0U, unstable.size() - 1U), fixture.alice_token); + auto const list_slash = request(fixture.runtime(), "GET", std::string{v1}, fixture.alice_token); + + THEN("a served path with another method is 405 M_UNRECOGNIZED and the listing is 404 M_UNRECOGNIZED") + { + for (auto const* wrong : + {&get_schedule, &post_schedule, &delete_event, &put_event, &get_action, &put_action}) + { + CHECK(wrong->status == 405U); + CHECK(errcode_of(*wrong) == "M_UNRECOGNIZED"); + } + for (auto const* listing : {&list_stable, &list_unstable, &list_slash}) + { + CHECK(listing->status == 404U); + CHECK(errcode_of(*listing) == "M_UNRECOGNIZED"); + } + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + } +} diff --git a/tests/integration/test_security_audit_http_3_flow.cpp b/tests/integration/test_security_audit_http_3_flow.cpp index 83c398ef4..48c33e03b 100644 --- a/tests/integration/test_security_audit_http_3_flow.cpp +++ b/tests/integration/test_security_audit_http_3_flow.cpp @@ -417,6 +417,37 @@ SCENARIO("HTTP-3 gives every implemented dynamic client route its own coalesced // ADR-0212: the reserved-ID upload. {"/_matrix/media/v3/upload/{server}/{mediaId}", "/_matrix/media/v3/upload/example.org/one", "/_matrix/media/v3/upload/example.org/two" }, + // ADR-0826: delayed events (MSC4140). Every template is its own bucket; the action stays literal. + {"/_matrix/client/v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}", + "/_matrix/client/v3/rooms/%21a%3Aexample.org/delayed_event/m.one/t1", + "/_matrix/client/v3/rooms/%21b%3Aexample.org/delayed_event/m.two/t2"}, + {"/_matrix/client/unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}", + "/_matrix/client/unstable/org.matrix.msc4140/rooms/%21a%3Aexample.org/delayed_event/m.one/t1", + "/_matrix/client/unstable/org.matrix.msc4140/rooms/%21b%3Aexample.org/delayed_event/m.two/t2"}, + {"/_matrix/client/v1/delayed_events/{delayId}", + "/_matrix/client/v1/delayed_events/idone", + "/_matrix/client/v1/delayed_events/idtwo"}, + {"/_matrix/client/v1/delayed_events/{delayId}/send", + "/_matrix/client/v1/delayed_events/idone/send", + "/_matrix/client/v1/delayed_events/idtwo/send"}, + {"/_matrix/client/v1/delayed_events/{delayId}/cancel", + "/_matrix/client/v1/delayed_events/idone/cancel", + "/_matrix/client/v1/delayed_events/idtwo/cancel"}, + {"/_matrix/client/v1/delayed_events/{delayId}/restart", + "/_matrix/client/v1/delayed_events/idone/restart", + "/_matrix/client/v1/delayed_events/idtwo/restart"}, + {"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idone", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idtwo"}, + {"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}/send", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idone/send", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idtwo/send"}, + {"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}/cancel", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idone/cancel", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idtwo/cancel"}, + {"/_matrix/client/unstable/org.matrix.msc4140/delayed_events/{delayId}/restart", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idone/restart", + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/idtwo/restart"}, }; GIVEN("a one-request operator cap keyed on each route template") diff --git a/tests/meson.build b/tests/meson.build index 3623b182b..8143d3982 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -1006,6 +1006,7 @@ shell = find_program('sh') 'integration/test_room_read_examination_budget_flow.cpp', 'integration/test_stored_invite_caps_flow.cpp', 'integration/test_delayed_events_flow.cpp', + 'integration/test_delayed_events_routes_flow.cpp', 'integration/test_security_issues_460_463_flow.cpp', 'integration/test_server_signing_key_lifecycle_flow.cpp', 'integration/test_sync_dag_gap_flow.cpp', From a8a6bcc0568c069bb92bd242073412f3b4bfbe15 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:07:58 +0000 Subject: [PATCH 13/29] Describe delayed events in the user manual's calls section Co-Authored-By: Claude Fable 5.2 --- docs/user-manual.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/user-manual.md b/docs/user-manual.md index d7fc98580..f6ac9c013 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -858,11 +858,42 @@ start-up; anything outside the range, or not a boolean or unsigned integer, refu Not implemented yet: -- **Delayed events (MSC4140).** A participant whose client vanishes (a crash or lost network) is not removed from the - call's membership state automatically and can stay listed until they rejoin or someone removes them. - **A built-in token service.** There is no `/_matrix/client/v1/rtc/livekit/*` endpoint and the transport type `m.livekit` is not advertised; you must run `lk-jwt-service` yourself. +##### Delayed events and call membership + +A call member who loses connection should stop being listed. With delayed events (MSC4140) the client schedules the +"leave" of its own call membership and keeps restarting that timer while it is connected. If the client vanishes +(a crash or lost network), the timer runs out and the server sends the leave, so the participant is no longer listed. +Element Call asks for a delay of 18 seconds and restarts it every few seconds. Delayed events are on by default and +need no configuration. The keys that bound them are in the +[`server.delayed_events.*` table](#delayed-events-serverdelayed_events) above: `enabled`, `max_delay_ms`, +`max_scheduled_per_user` and `max_finalised_per_user`. Keep `max_delay_ms` at `3600000` (one hour) or more, because +Element Call uses a one-hour delay when the LiveKit service takes over the leave; a client whose delay is refused +joins without a delayed leave. The endpoints are listed in +[the client-server API reference](matrix-v1.19-client-server-api.md). + +The wire shapes were taken from the source of Element Web 1.12.35, Element X 26.11.0, Element Call 0.28.2 and +`lk-jwt-service` 0.10.0 at those releases (ADR-0823). No real client has been run against this server yet. +`lk-jwt-service` 0.10.0 manages a delegated leave as an application service acting for the user, so an operator who +wants that registers it as an application service whose user namespace covers the local users; see the +[application service section](#application-service-api-appservice). + +What you will notice in operation (ADR-0824, ADR-0825): + +- A due event is sent as the user and is authorised against the room when it is sent, not when it was scheduled. +- Events that fell due while the server was stopped are sent when it starts. A restart longer than a client's delay + therefore ends call memberships, and the clients rejoin. +- A failed send is recorded and not retried. A suspended or locked account's delayed events fail. +- Deactivating an account deletes its delayed events. +- Setting `server.delayed_events.enabled` to `false` stops new scheduling; events already scheduled are still sent. +- Scheduling and management are rate limited per user and per address; the built-in values can be overridden with + `client_rate_limits.*` (see [client rate limits](#client-rate-limits-client_rate_limits)). + +Not provided: the listing endpoint (MSC4486) and sticky events (MSC4354). Element Call's `matrix_2_0` mode is +therefore not supported; the default `compatibility` mode is the one this serves. + #### OAuth 2.0 API — `server.oauth.*` Merovingian has a built-in OAuth 2.0 API (Matrix v1.19 "OAuth 2.0 API"), which From 0c16373264faa07b759aa2acd616f877e3c4f1da Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:10:41 +0000 Subject: [PATCH 14/29] Make the being-sent mark exception-safe, log a failed prune, and refuse the delay parameter on the other sending routes Co-Authored-By: Claude Fable 5.2 --- docs/observability-audit.md | 2 +- src/homeserver/AGENTS.md | 7 +- src/homeserver/ClientServerRuntime.cpp | 23 +++++ src/homeserver/DelayedEvents.cpp | 93 +++++++++++++++++-- .../integration/test_delayed_events_flow.cpp | 74 +++++++++++++++ .../test_delayed_events_routes_flow.cpp | 89 ++++++++++++++++++ 6 files changed, 280 insertions(+), 8 deletions(-) diff --git a/docs/observability-audit.md b/docs/observability-audit.md index 1138da4ea..879a0f311 100644 --- a/docs/observability-audit.md +++ b/docs/observability-audit.md @@ -633,7 +633,7 @@ The delayed-events handlers and the `delayed-events` job log under the `delayed_ `delay_id`, `user_id`, `room_id` and `event_type`, never the content and never the state key (a call membership's state key holds a device ID): `delayed_event.scheduled` (info, with `delay_ms`), `delayed_event.restarted` (info), `delayed_event.cancelled` (info), `delayed_event.sent` (info, with `event_id`), `delayed_event.failed` (warning, with -`status` and `errcode`) and `delayed_event.finalise_commit_failed` (error, with `outcome`, `refused` or `commit_failed`: +`status` and `errcode`; error, with `status` and the exception's `type`, when the job's send threw), `delayed_event.prune_failed` (warning, no row fields: the age prune's commit failed and the rows stay for the next run) and `delayed_event.finalise_commit_failed` (error, with `outcome`, `refused` or `commit_failed`: the event was sent but its row could not be recorded, so it is sent again after a restart). ## Registration token file diagnostics diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index 2a4209ffd..84b55d186 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -1208,6 +1208,10 @@ from the table's redacted JSON; do not add a second copy. `DelayedEvents.cpp` schedules, manages, looks up and sends an event a user asked to have sent later; the routes (the next layer) only parse a request and call it. Invariants a change must keep: +- **The mark is owned by `SendMark` (RAII).** It is cleared on every path, an exception included. The job's send is + caught whole (a job item has no dispatch backstop, and an unrecorded attempt would be repeated every few seconds): it + is recorded `failed` 500 `M_UNKNOWN`. The `send` action does not catch: the exception reaches the backstop (500) and + the row stays scheduled. - **One sender at a time, marked.** A send takes `runtime.mutex` for one event, marks the row as being sent (`DelayedEventTable::mark_being_sent`, memory only) before `send_event_as_resolved_user`, and finalising clears the mark. A marked row is offered by nobody: restart, cancel and send answer 409 "being sent". The job sends ONE event per @@ -1232,7 +1236,8 @@ next layer) only parse a request and call it. Invariants a change must keep: - **Lookup is by `find_owned` only**, and an unknown, another user's, empty or over-long ID is the one 404 `M_NOT_FOUND` "Delayed event not found.", byte for byte. - **Logging** is under `delayed_events` (`delayed_event.scheduled`, `.restarted`, `.cancelled`, `.sent`, `.failed`, - `.finalise_commit_failed`) with `delay_id`, `user_id`, `room_id`, `event_type`, and `event_id`, `status`, `errcode` + `.finalise_commit_failed`, and `.prune_failed`, a warning with no row fields) with `delay_id`, `user_id`, `room_id`, + `event_type`, and `event_id`, `status`, `errcode`, and for an exception in the job's send `type` (never `what()`), where they apply. NEVER log `content_json` or a state key (a call-membership state key holds a device ID). - `send_event_as_user`'s room and membership refusal is `room_send_refusal`, and its power refusal `event_authorization_refusal`, both shared with the schedule-time check; change them, not a copy. diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 1f9c4b1a6..9f519a9fd 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -12572,6 +12572,21 @@ namespace return schedule_delayed_event_for_request(rt, req, user, std::move(request)); } + // A request that carries `org.matrix.msc4140.delay` is never sent at once by any route (ADR-0823). PUT /send and + // PUT /state schedule it; every other route that sends a room event from the caller's body (POST .../send, PUT + // .../redact/...) refuses it here, before anything is sent and whether or not delayed events are enabled. Uses + // `delay_query_of`, the one reader of the parameter. + [[nodiscard]] auto delay_parameter_refusal(LocalHttpRequest const& req, ClientServerRuntime const& rt) + -> std::optional + { + if (!delay_query_of(req.target).present) + { + return std::nullopt; + } + return dispatch_err(req, rt, 400U, "M_INVALID_PARAM", + "delayed events are scheduled with PUT /send, PUT /state or the delayed_event endpoint"); + } + // The members of S1 and S2's body. struct DelayedScheduleBody final { @@ -21034,6 +21049,10 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp // ../../docs/matrix-v1.19-spec/client-server-api.md#redactions if (auto const redact = room_redact_path_parts(req.target); redact.has_value()) { + if (auto refused = delay_parameter_refusal(req, rt); refused.has_value()) + { + return std::move(*refused); + } // The transaction ID is scoped to this endpoint and target, so the same txnId used // on /send, or to redact another event, is a different request. auto const txn_scope = std::string{"redact/"} + redact->event_id; @@ -21279,6 +21298,10 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp if (req.method == "POST" && suffix.size() > send_s.size() && suffix.substr(suffix.size() - send_s.size()) == send_s) { + if (auto refused = delay_parameter_refusal(req, rt); refused.has_value()) + { + return std::move(*refused); + } auto const room_id = core::percent_decode_path_component(suffix.substr(0U, suffix.size() - send_s.size())); log_diagnostic("room.send_event.internal", { {"actor", *user, false}, diff --git a/src/homeserver/DelayedEvents.cpp b/src/homeserver/DelayedEvents.cpp index e4010703c..a62b26d10 100644 --- a/src/homeserver/DelayedEvents.cpp +++ b/src/homeserver/DelayedEvents.cpp @@ -8,6 +8,7 @@ #include "merovingian/canonicaljson/Parser.hpp" #include "merovingian/canonicaljson/Serializer.hpp" #include "merovingian/canonicaljson/Value.hpp" +#include "merovingian/core/CurrentException.hpp" #include "merovingian/core/QueryParams.hpp" #include "merovingian/crypto/Encoding.hpp" #include "merovingian/crypto/Random.hpp" @@ -316,6 +317,52 @@ namespace } } + // Owns the "being sent" mark of one row (memory only) for the length of one send: made with the mark taken, and + // when it goes out of scope, on every path including an exception, the mark is cleared. Finalising the row already + // clears it and clearing an unmarked row does nothing, so the destructor clears unconditionally. A record the store + // refuses is the one case that must keep the mark (the row is then never sent again by this process): that path + // calls `keep_mark()`. + class SendMark final + { + public: + SendMark(database::DelayedEventTable& table, std::string delay_id) + : m_table{&table} + , m_delay_id{std::move(delay_id)} + , m_held{table.mark_being_sent(m_delay_id)} + { + } + + SendMark(SendMark const&) = delete; + auto operator=(SendMark const&) -> SendMark& = delete; + SendMark(SendMark&&) = delete; + auto operator=(SendMark&&) -> SendMark& = delete; + + ~SendMark() noexcept + { + if (m_held) + { + std::ignore = m_table->clear_being_sent(m_delay_id); + } + } + + // True when this guard took the mark: nobody else was sending the row. + [[nodiscard]] auto held() const noexcept -> bool + { + return m_held; + } + + // The record of the outcome was refused: leave the mark, so the row is not sent again by this process. + auto keep_mark() noexcept -> void + { + m_held = false; + } + + private: + database::DelayedEventTable* m_table; + std::string m_delay_id; + bool m_held; + }; + // Sends the earliest-due row if it is due at `now_ms`, under its own hold of the runtime lock, and records the // outcome (any failure is `failed`, never retried). False when nothing was due. [[nodiscard]] auto send_next_due(ClientServerRuntime& runtime, std::uint64_t now_ms) -> bool @@ -329,14 +376,38 @@ namespace } // A copy: the send and the outcome change the table, which invalidates the pointer. auto const row = *next; - if (!table.mark_being_sent(row.delay_id)) + auto mark = SendMark{table, row.delay_id}; + if (!mark.held()) { return false; } - auto const result = attempt_send(runtime, row); + // The catch-all is here because this is a job item, not a request handler: the dispatch boundary's backstop + // does not cover it, and an attempt that is not recorded is repeated, because PeriodicJob runs a job that + // throws again a few seconds later, and a throw after the event was stored would send it every time. So an + // exception is a failed send with status 500, recorded like any other failure. Only the type is logged, never + // what(), which can carry content. + auto result = std::optional{}; + try + { + result = attempt_send(runtime, row); + } + catch (...) + { + log_event("delayed_event.failed", row, + { + {"status", "500", false}, + {"type", core::current_exception_type_name(), false}, + }, + observability::LogEventSeverity::error); + result = refusal(500U, "M_UNKNOWN", "internal server error"); + } // Finalising clears the mark. A record that was refused leaves the row marked, so it is never sent again by // this process (and is logged), never in a loop. - record_send_outcome(runtime, row, result, now_ms); + record_send_outcome(runtime, row, *result, now_ms); + if (table.is_being_sent(row.delay_id)) + { + mark.keep_mark(); + } return true; } @@ -516,7 +587,10 @@ auto manage_delayed_event(ClientServerRuntime& runtime, std::string_view user_id log_event("delayed_event.cancelled", row); return ok_result(); case DelayedEventAction::send: { - if (!store.delayed_events.mark_being_sent(row.delay_id)) + // An exception from the send unwinds through the guard (the mark is cleared, the row stays scheduled with + // its timer unchanged) to the dispatch boundary's backstop, which answers 500. + auto mark = SendMark{store.delayed_events, row.delay_id}; + if (!mark.held()) { return conflict_result("The delayed event is being sent."); } @@ -524,10 +598,13 @@ auto manage_delayed_event(ClientServerRuntime& runtime, std::string_view user_id if (!result.ok) { // The row stays scheduled with its timer unchanged; the caller answers as the send did. - std::ignore = store.delayed_events.clear_being_sent(row.delay_id); return result; } record_send_outcome(runtime, row, result, now_ms); + if (store.delayed_events.is_being_sent(row.delay_id)) + { + mark.keep_mark(); + } return ok_result(); } } @@ -608,7 +685,11 @@ auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, { // One batch of the age prune, in this hold. A failed commit leaves the rows for the next run. auto const cutoff_ms = now_ms > finalised_retention_ms ? now_ms - finalised_retention_ms : std::uint64_t{0U}; - std::ignore = database::prune_finalised_delayed_events(store, cutoff_ms, prune_batch_rows); + if (!database::prune_finalised_delayed_events(store, cutoff_ms, prune_batch_rows).has_value()) + { + observability::log_diagnostic(log_module, "delayed_event.prune_failed", {}, + observability::LogEventSeverity::warning); + } } auto const* next = store.delayed_events.earliest_due(); if (next == nullptr) diff --git a/tests/integration/test_delayed_events_flow.cpp b/tests/integration/test_delayed_events_flow.cpp index eba8be54e..aee6f4f2a 100644 --- a/tests/integration/test_delayed_events_flow.cpp +++ b/tests/integration/test_delayed_events_flow.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -1289,3 +1290,76 @@ SCENARIO("The job's return value says when it should run next, and it stops when } } } + +SCENARIO("A send that throws in the job is recorded failed with M_UNKNOWN and is not sent again", + "[homeserver][delayed-events][job][exception]") +{ + GIVEN("a due delayed event whose send throws from the store's commit") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "boom", 10'000U)); + auto armed = true; + fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = [&armed](std::size_t) { + if (armed) + { + armed = false; + throw std::runtime_error{"simulated commit failure"}; + } + }; + + WHEN("the job runs at the due time, and again afterwards") + { + CHECK_NOTHROW(std::ignore = run_job(fixture, t0 + 10'000U)); + auto const after_first = fixture.event_count(); + CHECK_NOTHROW(std::ignore = run_job(fixture, t0 + 20'000U)); + + THEN("the row is failed with M_UNKNOWN, unmarked, and the second run sends nothing") + { + REQUIRE(armed == false); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::failed); + CHECK_FALSE(fixture.table().is_being_sent(id)); + auto const error = *object_member_as_object( + *object_member_as_object(looked_up(fixture, alice, id), "finalised"), "error"); + CHECK(*string_member(error, "errcode") == "M_UNKNOWN"); + CHECK(fixture.event_count() == after_first); + } + } + fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = nullptr; + } +} + +SCENARIO("A send action that throws is answered 500 by the backstop and leaves the event scheduled and unmarked", + "[homeserver][delayed-events][manage][exception]") +{ + GIVEN("a scheduled delayed event whose send throws from the store's commit") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "boom", 60'000U)); + auto armed = true; + fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = [&armed](std::size_t) { + if (armed) + { + armed = false; + throw std::runtime_error{"simulated commit failure"}; + } + }; + + WHEN("the send action is requested, and then a restart") + { + auto const sent = request(fixture.runtime(), "POST", "/_matrix/client/v1/delayed_events/" + id + "/send", + fixture.alice_token); + auto const restart = request(fixture.runtime(), "POST", + "/_matrix/client/v1/delayed_events/" + id + "/restart", fixture.alice_token); + + THEN("the first is 500, the row is scheduled and not marked, and the restart is 200") + { + REQUIRE(armed == false); + CHECK(sent.status == 500U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + CHECK_FALSE(fixture.table().is_being_sent(id)); + CHECK(restart.status == 200U); + } + } + fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = nullptr; + } +} diff --git a/tests/integration/test_delayed_events_routes_flow.cpp b/tests/integration/test_delayed_events_routes_flow.cpp index 98130eed4..d829e064e 100644 --- a/tests/integration/test_delayed_events_routes_flow.cpp +++ b/tests/integration/test_delayed_events_routes_flow.cpp @@ -1337,3 +1337,92 @@ SCENARIO("A method a delayed-event path does not serve, and the listing, get the } } } + +SCENARIO("A NUL in the content of a delayed event is refused as a raw byte and kept as an escape", + "[homeserver][delayed-events][routes][nul]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + auto const before = fixture.event_count(); + auto const raw_nul = std::string{"{\"a\":\"x"} + std::string(1U, '\0') + "y\"}"; + auto const escaped_nul = std::string{R"({"a":"x\u0000y"})"}; + + WHEN("the legacy forms carry a raw NUL byte inside a string") + { + auto const l1 = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "n1", + std::string{delay_param} + "=5000", raw_nul); + auto const l2 = put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", + std::string{delay_param} + "=5000", raw_nul); + + THEN("each is 400 M_NOT_JSON from the gate, and nothing is stored or sent") + { + for (auto const* response : {&l1, &l2}) + { + CHECK(response->status == 400U); + CHECK(errcode_of(*response) == "M_NOT_JSON"); + } + CHECK(fixture.event_count() == before); + } + } + + WHEN("each form carries the escape \u0000 in a string") + { + auto const l1 = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "n2", + std::string{delay_param} + "=5000", escaped_nul); + auto const l2 = put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", + std::string{delay_param} + "=5000", escaped_nul); + auto const s1 = put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "n3", + schedule_body(5'000U, escaped_nul)); + auto const s2 = put_dedicated(fixture, false, fixture.alice_token, "m.test.delayed", "n4", + schedule_body(5'000U, escaped_nul)); + + THEN("each is accepted and no stored content holds a raw NUL byte") + { + for (auto const* response : {&l1, &l2, &s1, &s2}) + { + auto const id = delay_id_of(*response); + auto const* row = fixture.table().find_owned(alice, id); + REQUIRE(row != nullptr); + CHECK(row->content_json.find('\0') == std::string::npos); + CHECK(row->content_json.find(R"(\u0000)") != std::string::npos); + CHECK(get_delayed(fixture, true, id, fixture.alice_token).status == 200U); + } + fixture.set_time(t0 + 5'000U); + fixture.run_job(t0 + 5'000U); + } + } + } +} + +SCENARIO("Any other route that sends from the caller's body refuses the delay parameter", + "[homeserver][delayed-events][routes][invalid-param]") +{ + GIVEN("alice in a room, with delayed events on or off") + { + auto fixture = Fixture{}; + fixture.runtime().limits.delayed_events_enabled = GENERATE(true, false); + auto const before = fixture.event_count(); + + WHEN("POST .../send and PUT .../redact/... carry the parameter") + { + auto const sent = + request(fixture.runtime(), "POST", fixture.room_path() + "/send?" + std::string{delay_param} + "=5000", + fixture.alice_token, R"({"type":"m.test.delayed","content":{"body":"now"}})"); + auto const redact = + request(fixture.runtime(), "PUT", + fixture.room_path() + "/redact/$somewhere/t1?" + std::string{delay_param} + "=5000", + fixture.alice_token, "{}"); + + THEN("each is 400 M_INVALID_PARAM and nothing is sent") + { + CHECK(sent.status == 400U); + CHECK(errcode_of(sent) == "M_INVALID_PARAM"); + CHECK(redact.status == 400U); + CHECK(errcode_of(redact) == "M_INVALID_PARAM"); + CHECK(fixture.event_count() == before); + CHECK_FALSE(timeline_has(fixture, "m.test.delayed")); + } + } + } +} From fe5d5ea5628b961391104dbf111adce938233fff Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:10:47 +0000 Subject: [PATCH 15/29] Document that every body-sending route refuses the delay parameter Co-Authored-By: Claude Fable 5.2 --- src/homeserver/AGENTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index 84b55d186..2d4494737 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -1249,6 +1249,9 @@ The routes (`ClientServerRuntime.cpp`, ADR-0823) only parse a request and call t percent-decoded too) is answered by `delayed_query_form` before the idempotency lookup, `?ts=` and anything that could send. A repeated, empty or non-digit value, `0`, or one over 64 bits is `400 M_INVALID_PARAM`; a disabled server is `403 M_FORBIDDEN` first. No error of such a request may reach the ordinary send. +- **Every other route that sends a room event from the caller's body** (`POST .../send`, `PUT .../redact/...`) answers a + request carrying the parameter `400 M_INVALID_PARAM` through `delay_parameter_refusal`, before anything is sent and + whether or not the feature is on; a new such route calls it too. - **The dedicated routes (S1, S2) are matched before the room routes**, whose `/send/` and `/state/` markers would otherwise find them inside the path. Disabled is answered before any body error (`delayed_events_disabled_refusal` calls the service with an empty request, so the text is written once); then `parse_delayed_schedule_body`. From a79a7151852919c0f80ec0acf82d82d84e94d35e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:24:31 +0000 Subject: [PATCH 16/29] Bring the delayed-events documents in line with the review's rulings Co-Authored-By: Claude Fable 5.2 --- ...ameter-forms-that-released-clients-send.md | 8 ++++ ...d-per-user-and-deleted-with-the-account.md | 8 +++- ...send-path-never-early-and-at-least-once.md | 26 ++++++++++--- docs/database-persistence.md | 8 +++- docs/log-filtering.md | 2 +- docs/matrix-v1.19-client-server-api.md | 2 +- docs/observability-audit.md | 10 +++-- docs/security-coding-rules.md | 39 ++++++++++++++++++- docs/threat-model.md | 8 ++-- 9 files changed, 92 insertions(+), 19 deletions(-) diff --git a/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md b/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md index 5c88e7883..50595cc3c 100644 --- a/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md +++ b/docs/adr/0823-delayed-events-are-served-on-the-stable-unstable-and-query-parameter-forms-that-released-clients-send.md @@ -55,6 +55,13 @@ route's idempotency lookup and before any send. It never falls through to an imm feature is disabled (then it is `403 M_FORBIDDEN`). A delay that is not a decimal integer of at least 1, or is repeated, is `400 M_INVALID_PARAM`. +The parameter is honoured only on `PUT .../send/{eventType}/{txnId}` and `PUT .../state/...`. Every other client +route that sends a room event from the caller's body (`POST /_matrix/client/v3/rooms/{roomId}/send` and +`PUT .../redact/{eventId}/{txnId}`) answers `400 M_INVALID_PARAM` when the parameter is present, before anything is +sent and whether or not the feature is enabled, so that no route can send at once a request that asked for a delay. +The parameter's name is percent-decoded before it is compared, so an encoded spelling is recognised. The +application-service `?ts=` parameter is ignored on a delayed request; the event is stamped with the time it is sent. + Management looks an event up by the authenticated user and the delay ID. A delay ID that does not exist and one that belongs to another user get the same answer, byte for byte: `404 M_NOT_FOUND`, `Delayed event not found.` @@ -129,6 +136,7 @@ Earlier drafts did this. Synapse stopped in 1.127 and 1.164. * The listing endpoint and sticky events (MSC4354) remain out of scope; the sticky member of the dedicated body is ignored. * Application-service masquerade on the management routes must keep working. +* A new route that sends an event from the caller's body must refuse the delay parameter the same way. * The user manual tells operators to keep the maximum delay at one hour or more. ## Links diff --git a/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md b/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md index e3f075948..1f1b13096 100644 --- a/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md +++ b/docs/adr/0824-a-delayed-event-is-a-stored-row-kept-after-it-is-finalised-bounded-per-user-and-deleted-with-the-account.md @@ -39,10 +39,15 @@ survives restarts with one mechanism. per user by `max_finalised_per_user` (default 100): when a row is finalised and the user is at the bound, that user's oldest finalised rows are deleted in the same transaction. They are bounded in age by a fixed 7 days, pruned by the delayed-events job, at most 500 rows per transaction, oldest first. +* A row the loader cannot read is skipped, never repaired and never guessed: the three numeric columns must be decimal + digits, the status and the two booleans exactly the texts the store writes, a scheduled row's delay within 1 to the + maximum, and a `sent` row's event ID not already claimed by another row. The number of rows skipped is logged once + at start-up. A number read as 0 would make a scheduled row due at once, which is sending early. * A restart of a scheduled event is persisted: it updates `delayed_since_ms`. * Deactivating an account deletes all of the user's rows, scheduled and finalised, in one transaction, as a step of `deactivate_user_account` beside the pusher cleanup. A failure is logged and deactivation continues; the check at - send time (ADR-0825) is the guarantee. + send time (ADR-0825) is the guarantee. The delete always runs in the database, so that a skipped row does not + outlive its account. * Logout and device deletion do not cancel. A delayed event belongs to the user; any of their devices, or an application service acting for them, may manage it. * The federation worker never reads the table. It is classified in the never-reads set of @@ -87,6 +92,7 @@ Persist only the schedule, not the postponement. * The federation worker does not read the table; a change that makes it read the table must change its classification deliberately. * A new code path that deletes an account's data must delete these rows too. +* A new column gets its check in both loaders. ## Links diff --git a/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md b/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md index cdc0f9447..6641413ee 100644 --- a/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md +++ b/docs/adr/0825-a-due-delayed-event-is-sent-by-one-job-through-the-ordinary-send-path-never-early-and-at-least-once.md @@ -20,11 +20,13 @@ and its authorisation, and the job machinery already sleeps until a deadline. hardening, stopped on shutdown in the same sequence as the other jobs, and refuses to serve if its thread cannot start. `HomeserverRuntime::delayed_events_wake` (null in tests, installed and removed under the runtime lock like `typing_expiry_wake`) is called when an event is scheduled. A restart only moves a deadline later and needs no wake. -* The body is a function taking the runtime, `now_ms` and the stop flag. Per run: take the runtime lock, ask the table - for the earliest-due scheduled row with due time at or before `now_ms`, send it, release the lock; one send per - lock hold, the stop flag checked between items, at most 100 sends per run (then it runs again at once). One age - prune batch, in its own lock hold. It returns the time until the next due row, clamped to 0 to 60 seconds, and 60 - seconds when nothing is scheduled. +* The body is a function taking the runtime, `now_ms` and the stop flag. Each run sends at most one due event: take + the runtime lock, ask the table for the earliest-due scheduled row with due time at or before `now_ms`, send it, + release the lock. Then one age prune batch, in its own lock hold. While another event is already due the job + returns a pace of 5 ms with the lock free (the profile propagation job's pace, + [ADR-0353](0353-a-profile-change-holds-the-runtime-lock-for-a-fixed-share-of-the-users-rooms.md)), so a backlog + cannot starve requests of a lock that is not fair. Otherwise it returns the time to the next deadline, at most 60 + seconds, and 60 seconds when nothing is scheduled. It never returns 0. * Time is wall-clock milliseconds, because the due time is persisted and must mean the same after a restart. Handlers and the job take "now" from one function that honours a `HomeserverRuntime` test seam. Tests drive the job body with a chosen `now_ms` and never sleep; no test depends on a wall-clock margin. @@ -44,6 +46,13 @@ and its authorisation, and the job machinery already sleeps until a deadline. under it. The row is also marked "being sent" (memory only) before `send_event_as_user` is called and unmarked when it returns, and while it is marked the row is out of the due order and a second send, restart or cancel answers 409. +* The "being sent" mark is held by a scope guard, so it is cleared on every path. In the job, an exception from the + send is recorded as a failed send (500, `M_UNKNOWN`) and logged by type only, because a job that throws is run + again a few seconds later and an unrecorded attempt would be repeated; if recording the outcome itself throws, the + mark is kept so the row is not sent again by this process. In a `send` request there is no catch: the exception + reaches the dispatch boundary's backstop (500) and the event stays scheduled, so if the exception came after the + event was stored the event can be sent again later. That is the at-least-once rule applying inside one process, in + this one case. * At least once: the order is send, then finalise. The event's commit and the row's update are two transactions; a crash between them sends the event again after restart. For a call membership leave a duplicate is harmless and a lost one is the bug being fixed. @@ -54,6 +63,11 @@ and its authorisation, and the job machinery already sleeps until a deadline. ## Pros and Cons of the Options +### Send every due event in one run, back to back + +* Bad, because each send releases the lock, but the mutex is not fair and the job thread retakes it at once, so a + backlog after downtime could hold requests off for its whole length. + ### Record as sent, then send (at most once) * Bad, because a crash between the two loses a call's leave. @@ -99,6 +113,8 @@ Compose and sign, then discard. ## Consequences * The delayed-events job is the only background sender of delayed events. +* A `send` request that throws after the event was stored leaves the row scheduled, so the event can be sent again + later: the at-least-once rule applying inside one process, in this one case. * Anything inside the send path that releases the runtime lock is covered by the "being sent" mark. A change that makes the send path release the lock for a new event type must keep the mark set across it. * A clock stepped backwards delays sends; stepped forwards, it brings them forward by the step. The server has no diff --git a/docs/database-persistence.md b/docs/database-persistence.md index 3dba52374..ad6aeeaeb 100644 --- a/docs/database-persistence.md +++ b/docs/database-persistence.md @@ -98,11 +98,15 @@ value. Test fixtures that do not exercise persistence select it through delay_ms`, then delay ID, so the earliest due is O(1)), by (user, device, room, event type, transaction ID) for idempotency, by event ID for sent rows, and every finalised row by finalised time for the age prune. A memory-only "being sent" mark hides a row from the due order while a send is in progress. No reader scans the table. A row - whose `status` text is not one of the four is skipped when hydrating, never repaired. + the loader cannot read is skipped when hydrating, never repaired and never guessed: the three numeric columns must + be decimal digits, `status` and the two booleans exactly the texts the store writes, a scheduled row's delay within + 1 to the maximum, and a `sent` row's event ID not already claimed by another row. The count skipped is logged once + at start-up. A number read as 0 would make a scheduled row due at once, which is sending early. - Rows leave in four ways, all in one transaction: the user's cancel and the outcome of a send each evict that user's oldest finalised rows beyond `server.delayed_events.max_finalised_per_user`; the age prune deletes finalised rows older than 7 days (oldest first, at most 500 a transaction); and account deactivation deletes all - of a user's rows (`DELETE ... WHERE user_id = $1`). Logout and device deletion delete nothing. + of a user's rows (`DELETE ... WHERE user_id = $1`, always run in the database, so a skipped row does not outlive its + account). Logout and device deletion delete nothing. - The one place memory changes without a commit: `record_delayed_event_outcome`. The event was already sent, so memory is finalised even when the finalising commit fails (the function says so), and the database row still reads `scheduled`, so the next start-up sends it again (at least once, ADR-0825). Every other function changes diff --git a/docs/log-filtering.md b/docs/log-filtering.md index 41969030d..a94c443d2 100644 --- a/docs/log-filtering.md +++ b/docs/log-filtering.md @@ -133,7 +133,7 @@ used as the first argument to `observability::log_diagnostic` and `log_diagnosti | `stream_token_epoch` | The database's sync token epoch: created at first start, replaced by `--after-restore`, or refused | | `tls` | TLS context set-up and handshake accept or reject | | `oauth` | OAuth 2.0 client registration, eviction at the cap, and stored metadata that no longer validates | -| `delayed_events` | Delayed events: scheduled, restarted, cancelled, sent, failed, and a failed outcome write; never event content or a state key | +| `delayed_events` | Delayed events: scheduled, restarted, cancelled, sent, failed, a failed outcome write, a failed prune, and rows skipped at load; never event content or a state key | Unrecognised module names are accepted and the level is recorded — there is no error. Restart the server to apply. diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index 167f465ec..a3291849f 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -540,7 +540,7 @@ In each case below the spec leaves the choice to the server; the owner decided t - **`POST /users/{userId}/report` (v1.14, ADR-0482).** Needs an access token and is rate-limited. The body is a JSON object whose `reason`, when present, is a string (`400 M_BAD_JSON` otherwise). A local user ID no account holds is `404 M_NOT_FOUND` and stores nothing; a user of another server is accepted without a lookup. It is stored as an audit row `trust_safety.account.accept_report` (reporter, reported user), one per pair, listed by `GET /admin/safety/reports`; a repeat answers `200` without a second row. The reporter need share no room with the reported user. - **Password reset by email (ADR-0481).** `POST /account/password/email/requestToken` (no token) answers `200` with `sid` and `submit_url` for every well-formed request, whether or not an account holds the address, and emails an 8-character code only to an active account's address; it never answers `M_THREEPID_NOT_FOUND` or `M_THREEPID_IN_USE`. A repeat with the same or a lower `send_attempt` returns the same `sid` and sends nothing. With no relay (`server.email.smtp_url`) it is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`. The client posts `{sid, client_secret, token}` to `submit_url` (`/_matrix/client/unstable/org.merovingian/account/password/email/submitToken`, this server's own path, the Identity Service `submitToken` shape): `{"success":true}`, or `400 M_TOKEN_INCORRECT`, `M_INVALID_PARAM` or `M_SESSION_EXPIRED` (five wrong codes end the session; sessions last 15 minutes). `POST /account/password` with no access token and `auth: {type: "m.login.email.identity", threepid_creds: {sid, client_secret}}` then sets the password of the account that holds the address (`401` with the flows for a missing or unvalidated stage, `400 M_WEAK_PASSWORD` for a weak password and the session is kept for a retry) and, with `logout_devices` (the default), revokes every token of the account. `POST /account/password/msisdn/requestToken` is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED` (no SMS). - **MatrixRTC transport discovery (MSC4143, not in v1.19; ADR-0822).** `GET /_matrix/client/v1/rtc/transports` answers `{"transports":[{"type":"livekit","livekit_service_url":""}]}` and `GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports` answers `{"rtc_transports":[...]}`, both needing an access token; with `server.rtc.livekit.service_url` empty they answer `{"transports":[]}` and `{}`. `/versions` always lists `org.matrix.msc4143` and `org.matrix.msc4143.stable`, and the well-known document gains `org.matrix.msc4143.rtc_foci` when a URL is set. There is no `m.livekit` transport and no `/rtc/livekit/*` route. -- **Delayed events (MSC4140; in the v1.20 spec text, not in v1.19; ADR-0823 to ADR-0826).** All nine routes need an access token and the standard account gate; `?user_id=` masquerade works as on other routes. Schedule: `PUT /v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}` and `PUT /unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}`; a request to `PUT /v3/rooms/{roomId}/send/{eventType}/{txnId}` or `PUT /v3/rooms/{roomId}/state/{eventType}/{stateKey}` (also the no-state-key and empty-state-key forms) that carries `?org.matrix.msc4140.delay=` is always scheduled, never sent at once, and answers `403 M_FORBIDDEN` when the feature is disabled; a delay that is not a decimal integer of at least 1, or is repeated, is `400 M_INVALID_PARAM`, and one over `server.delayed_events.max_delay_ms` is `400` (`M_DELAY_TOO_LARGE` on the v3 dedicated path, `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE` on the others); a user at `max_scheduled_per_user` gets `429 M_LIMIT_EXCEEDED`; content over 65,536 bytes is `413 M_TOO_LARGE`. Manage: `POST /v1/delayed_events/{delayId}/{send|cancel|restart}` and `POST /unstable/org.matrix.msc4140/delayed_events/{delayId}/{send|cancel|restart}`, and, under the unstable prefix only, `POST /unstable/org.matrix.msc4140/delayed_events/{delayId}` with `{"action":"send"|"cancel"|"restart"}` (there is no `POST /v1/delayed_events/{delayId}`; any other last segment is `404 M_UNRECOGNIZED`). Look up: `GET /v1/delayed_events/{delayId}` and `GET /unstable/org.matrix.msc4140/delayed_events/{delayId}`. A delay ID that is unknown and one that belongs to another user both answer `404 M_NOT_FOUND` with `Delayed event not found.`; a repeated action on a finalised event answers `200` or `409 M_UNKNOWN`, and one on an event being sent answers `409`. `/versions` lists `org.matrix.msc4140` and `org.matrix.msc4140.stable` (each true when the feature is enabled), and `/capabilities` lists `m.delayed_events` and `org.matrix.msc4140.delayed_events` (`max_delay_ms`, `max_scheduled`; zeros when disabled). The event a delayed event became carries `unsigned.delay_id` for the user who scheduled it. The listing endpoint (MSC4486) is not routed (`404 M_UNRECOGNIZED`); sticky events (MSC4354) are not supported. +- **Delayed events (MSC4140; in the v1.20 spec text, not in v1.19; ADR-0823 to ADR-0826).** All nine routes need an access token and the standard account gate; `?user_id=` masquerade works as on other routes. Schedule: `PUT /v3/rooms/{roomId}/delayed_event/{eventType}/{txnId}` and `PUT /unstable/org.matrix.msc4140/rooms/{roomId}/delayed_event/{eventType}/{txnId}`; a request to `PUT /v3/rooms/{roomId}/send/{eventType}/{txnId}` or `PUT /v3/rooms/{roomId}/state/{eventType}/{stateKey}` (also the no-state-key and empty-state-key forms) that carries `?org.matrix.msc4140.delay=` is always scheduled, never sent at once, and answers `403 M_FORBIDDEN` when the feature is disabled (the other body-sending routes, `POST .../send` and `PUT .../redact/...`, answer `400 M_INVALID_PARAM` when the parameter is present, and the application-service `?ts=` is ignored on a delayed request); a delay that is not a decimal integer of at least 1, or is repeated, is `400 M_INVALID_PARAM`, and one over `server.delayed_events.max_delay_ms` is `400` (`M_DELAY_TOO_LARGE` on the v3 dedicated path, `ORG.MATRIX.MSC4140_DELAY_TOO_LARGE` on the others); a user at `max_scheduled_per_user` gets `429 M_LIMIT_EXCEEDED`; content over 65,536 bytes is `413 M_TOO_LARGE`. Manage: `POST /v1/delayed_events/{delayId}/{send|cancel|restart}` and `POST /unstable/org.matrix.msc4140/delayed_events/{delayId}/{send|cancel|restart}`, and, under the unstable prefix only, `POST /unstable/org.matrix.msc4140/delayed_events/{delayId}` with `{"action":"send"|"cancel"|"restart"}` (there is no `POST /v1/delayed_events/{delayId}`; any other last segment is `404 M_UNRECOGNIZED`). Look up: `GET /v1/delayed_events/{delayId}` and `GET /unstable/org.matrix.msc4140/delayed_events/{delayId}`. A delay ID that is unknown and one that belongs to another user both answer `404 M_NOT_FOUND` with `Delayed event not found.`; a repeated action on a finalised event answers `200` or `409 M_UNKNOWN`, and one on an event being sent answers `409`. `/versions` lists `org.matrix.msc4140` and `org.matrix.msc4140.stable` (each true when the feature is enabled), and `/capabilities` lists `m.delayed_events` and `org.matrix.msc4140.delayed_events` (`max_delay_ms`, `max_scheduled`; zeros when disabled). The event a delayed event became carries `unsigned.delay_id` for the user who scheduled it. The listing endpoint (MSC4486) is not routed (`404 M_UNRECOGNIZED`); sticky events (MSC4354) are not supported. - **`GET /voip/turnServer` with `server.turn.uris` (ADR-0821)** answers one `uris` entry per configured URI, in the configured order; `server.turn.server` still gives one. - **`GET /voip/turnServer` with `server.turn.shared_secret_file` (ADR-0480)** answers per-user credentials in the coturn REST API format: `username` `:`, `password` the base64 HMAC-SHA1 of it under the secret, `ttl` `server.turn.ttl_seconds`. - **`GET /_merovingian/health` (ADR-0479)** is not a Matrix endpoint: an unauthenticated probe, `200 ok` or `503 unavailable`. diff --git a/docs/observability-audit.md b/docs/observability-audit.md index 879a0f311..716e861b8 100644 --- a/docs/observability-audit.md +++ b/docs/observability-audit.md @@ -58,7 +58,8 @@ retry is skipped because the gateway's circuit is open; the skipped notification half the total timeout, counted against the circuit (ADR-0409); `push.delivery.deferred` (warning, `recipient`) marks a pusher a batch queued unattempted after its deadline; `account.deactivated.pusher_removal_failed` (warning, `user_id`) a pusher of a deactivated account that -could not be deleted (ADR-0411). +could not be deleted (ADR-0411); `account.deactivated.delayed_events_removal_failed` (warning, `user_id`) the +delayed events of a deactivated account that could not be deleted (ADR-0824). Email pushers (ADR-0219, ADR-0220) log under the `rooms` logger with `recipient` (the user ID), never the email address, the relay URL or credentials: `push.email.sent` (debug, with `unread`); @@ -633,8 +634,11 @@ The delayed-events handlers and the `delayed-events` job log under the `delayed_ `delay_id`, `user_id`, `room_id` and `event_type`, never the content and never the state key (a call membership's state key holds a device ID): `delayed_event.scheduled` (info, with `delay_ms`), `delayed_event.restarted` (info), `delayed_event.cancelled` (info), `delayed_event.sent` (info, with `event_id`), `delayed_event.failed` (warning, with -`status` and `errcode`; error, with `status` and the exception's `type`, when the job's send threw), `delayed_event.prune_failed` (warning, no row fields: the age prune's commit failed and the rows stay for the next run) and `delayed_event.finalise_commit_failed` (error, with `outcome`, `refused` or `commit_failed`: -the event was sent but its row could not be recorded, so it is sent again after a restart). +`status` and `errcode`; error, with `status` and the exception's `type`, when the job's send threw), `delayed_event.prune_failed` (warning, no row fields: the age prune's commit failed and the rows stay for the next run), +`delayed_events.rows_skipped_at_load` (warning, once per start-up when rows were skipped, with the count only: the +loader skips a row it cannot read and never repairs it, ADR-0824) and `delayed_event.finalise_commit_failed` (error, +with `outcome`, `commit_failed`, `refused` or `exception`: the event was sent but its row could not be recorded, so it +is sent again after a restart). ## Registration token file diagnostics diff --git a/docs/security-coding-rules.md b/docs/security-coding-rules.md index 8365b75c8..7ae44bf49 100644 --- a/docs/security-coding-rules.md +++ b/docs/security-coding-rules.md @@ -1833,6 +1833,41 @@ media-fetch-pool task. (CWE-359, ADR-0219). Source: `src/email/AGENTS.md`. +## Delayed events + +- **A delayed event is authorised, and its account re-checked, when it is sent, never trusted from when it was + scheduled.** The job and the `send` action apply the account gate (deactivated, locked, suspended), the per-user rate + limit and the room authorisation of the ordinary send path at the moment of sending. + Why: the user may have left the room, lost power, been suspended or deactivated since the event was scheduled, and a + stored decision would let them act after losing the right to (CWE-285). + Source: `src/homeserver/AGENTS.md`; ADR-0825. + +- **A delay ID is an identifier, never a credential, and an unknown ID and another user's ID get an identical + answer.** Every use is authenticated and the lookup is by (user, ID); both cases answer `404 M_NOT_FOUND` with the + same body, byte for byte. + Why: a different answer for a foreign ID would tell a caller which IDs exist, and accepting the ID alone would let + whoever learns it drive another user's event (CWE-639). + Source: `src/homeserver/AGENTS.md`; ADR-0823. + +- **A request that carries `org.matrix.msc4140.delay` is never sent at once by any route.** The delayed branch is taken + before anything is sent, also when the feature is disabled; a route that does not honour the parameter refuses it with + `400 M_INVALID_PARAM`, and the name is percent-decoded before it is compared. A new route that sends an event from the + caller's body must refuse it the same way. + Why: a request that asked for a delay and was sent at once is an event the user did not mean to send yet, and an + encoded spelling must not slip past the check. + Source: `src/homeserver/AGENTS.md`; ADR-0823. + +- **Event content and state keys of delayed events are never logged.** + Why: a call membership's state key holds a device ID and the content is whatever the client supplied. + Source: `src/homeserver/AGENTS.md`; ADR-0825. + +- **A stored delayed-event row the loader cannot read is skipped, never repaired and never guessed.** The numeric + columns must be decimal digits, the status and booleans exactly the stored texts, a scheduled row's delay in range and + a sent row's event ID unclaimed; the number skipped is logged once. A new column gets its check in both loaders. + Why: a number read as 0 would make a scheduled row due at once, which is sending early, and a tampered row must not + decide when an event goes out. + Source: `src/database/AGENTS.md`; ADR-0824. + ## Trust and safety - **Never hard-code moderation decisions — all rules come from config or an @@ -2008,9 +2043,9 @@ For finding everything a specific file contributed, without re-reading the whole | `src/http/AGENTS.md` | HTTP and network boundary | | `src/net/AGENTS.md` | Memory safety; HTTP and network boundary | | `src/oauth/AGENTS.md` | OAuth 2.0 API | -| `src/homeserver/AGENTS.md` | Authentication and authorization; OAuth 2.0 API; HTTP and network boundary; Federation; Media; Sync | +| `src/homeserver/AGENTS.md` | Authentication and authorization; OAuth 2.0 API; HTTP and network boundary; Federation; Media; Sync; Delayed events | | `src/media/AGENTS.md` | HTTP and network boundary; Media | -| `src/database/AGENTS.md` | OAuth 2.0 API; Database | +| `src/database/AGENTS.md` | OAuth 2.0 API; Database; Delayed events | | `migrations/AGENTS.md` | Database | | `src/config/AGENTS.md` | Secrets and logging | | `src/observability/AGENTS.md` | Secrets and logging | diff --git a/docs/threat-model.md b/docs/threat-model.md index 55f112704..045eb40fe 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -3346,20 +3346,20 @@ sent; a background thread in the main process, the `delayed-events` job, that se deferred, server-initiated action; and four configuration keys, `server.delayed_events.*`. Threats and mitigations: (a) acting after losing the right to (left the room, power level lowered, account suspended, locked or deactivated): the event is authorised against the room state at send time through the ordinary send path, the job re-checks the account, -and deactivation deletes the user's rows; (b) reading or driving another user's delayed event: the ID is 128 random bits, +and deactivation deletes the user's rows in the database always, so a skipped row does not outlive its account; the loader skips a row it cannot read (non-decimal numbers, unknown status or boolean text, a delay outside 1 to the maximum, a duplicate sent event ID) and never guesses, so a tampered or corrupt row cannot make an event go out early; (b) reading or driving another user's delayed event: the ID is 128 random bits, an identifier and never a credential, every use is authenticated, the lookup is by (user, ID), and an unknown and a foreign ID get an identical `404 M_NOT_FOUND`; (c) exhausting storage or memory: a per-user cap on scheduled events (the new one is refused, nothing is evicted), the 65,536-byte content limit, bounded retention of finalised rows (`max_finalised_per_user` and 7 days) and `max_delay_ms`; (d) exhausting the runtime lock or the room with a burst of -due events: one send per lock hold, at most 100 per run, the scheduling and management rate limits and a per-user +due events: each run sends at most one due event, in its own lock hold, and while more are due it paces 5 ms with the lock free, so a backlog cannot starve requests; the scheduling and management rate limits and a per-user send-time rate limit; (e) write load from restarts: each restart is one small write, bounded by the per-user management rate limit; (f) an event sent early or twice: never early by construction, since the deadline is compared under the lock at send time, and at least once by decision, so a crash between the send and its record sends it once more after restart; (g) a request that carries the delay parameter being sent at once: the delayed branch is taken first and never -falls through, including when the feature is disabled; (h) an application service: it manages a delayed event only by +falls through, including when the feature is disabled, and the other routes that send from the caller's body refuse the parameter with `400 M_INVALID_PARAM`, so no route sends at once a request that asked for a delay; (h) an application service: it manages a delayed event only by masquerading as a user in its namespace, as on every other route; (i) confidentiality: the content of a not-yet-sent or cancelled event is in the database and in memory for up to the retention period, and neither it nor the state key is ever logged; (j) the federation worker: it never reads the table and sends nothing. Residual risk: a duplicate event -after a crash in the window between the send and its record; a wall clock stepped forwards brings deadlines forward by +after a crash in the window between the send and its record, or when a `send` request throws after the event was stored (it has no catch, the event stays scheduled and can be sent again later); a wall clock stepped forwards brings deadlines forward by the step; a suspended account's delayed events fail rather than send, so the server does not end a suspended participant's call membership; the feature is on by default. From 87700e24710725bb158a94d7d903f7b469cae035 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:41:04 +0000 Subject: [PATCH 17/29] Pace the delayed-events job to one send a run and keep the mark when recording the outcome throws Co-Authored-By: Claude Fable 5.2 --- .../merovingian/homeserver/DelayedEvents.hpp | 12 +++--- src/homeserver/AGENTS.md | 11 +++-- src/homeserver/DelayedEvents.cpp | 41 ++++++++++++++----- src/main.cpp | 14 ++++++- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/include/merovingian/homeserver/DelayedEvents.hpp b/include/merovingian/homeserver/DelayedEvents.hpp index 4e1cf2dc1..568c9833f 100644 --- a/include/merovingian/homeserver/DelayedEvents.hpp +++ b/include/merovingian/homeserver/DelayedEvents.hpp @@ -63,11 +63,13 @@ auto wake_delayed_events(HomeserverRuntime& runtime) -> void; [[nodiscard]] auto delayed_event_lookup(ClientServerRuntime& runtime, std::string_view user_id, std::string_view delay_id) -> OperationResult; -// The "delayed-events" job's body (main.cpp; tests call it directly). Takes runtime.homeserver.mutex itself, one hold -// per delayed event: sends each scheduled event due at `now_ms` or earlier, earliest first, at most 100 a run and none -// once `stop` is requested, never before its due time; then deletes at most 500 finalised rows older than 7 days -// before `now_ms`. A failed send is recorded and never retried. Returns how long to wait before the next run: 0 ms -// when due events remain, else the time to the next deadline, at most 60 s (60 s when nothing is scheduled). +// The "delayed-events" job's body (main.cpp; tests call it directly). Takes runtime.homeserver.mutex itself: sends the +// earliest scheduled event due at `now_ms` or earlier, if any, in one hold, and none once `stop` is requested, never +// before its due time; then, in another hold, deletes at most 500 finalised rows older than 7 days before `now_ms`. +// At most ONE send a run, so the runtime lock is free between sends (the mutex is not fair; ADR-0353 paces the profile +// job the same way). A failed send is recorded and never retried. Returns how long to wait before the next run, never +// 0: 5 ms (`delayed_events_pace`) when another event is already due at `now_ms`; else the time to the next deadline, +// at least 5 ms and at most 60 s; 60 s when nothing is scheduled. A caller that wants a backlog drained calls it again. [[nodiscard]] auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, StopFlag const& stop) -> std::chrono::milliseconds; diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index 2d4494737..ad187e3f0 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -27,7 +27,7 @@ This is the largest and most complex module — read this file carefully before | `AppserviceDelivery.cpp` | Application Service API outbound queue (ADR-0163): `enqueue_appservice_delivery` (request path, no network) and `drain_appservice_queues` (the one drainer thread, run by a `PeriodicJob` from `main.cpp`) | | `ClientTxnPrune.cpp` | `prune_client_transactions`: the sweep the `client-txn-prune` `PeriodicJob` runs every hour, deleting to-device `client_txn_ids` records older than 7 days (F-54, ADR-0381) | | `RetentionSweep.cpp` | `run_retention_sweep`: the sweep the `retention-sweep` `PeriodicJob` runs every hour. Deletes sessions and tokens that can no longer authenticate and `audit_log` rows past `security.audit_log_retention_days`, OAuth 2.0 session rows whose device holds no live access or refresh token (judged again in the hold that deletes them, ADR-0808), and OAuth 2.0 clients with no session unused for 30 days (`oauth_client_retention`, least recently used first; never a client with a session, nor one `oauth_client_has_pending_authorisation` names: a live pending authorisation or an unredeemed code, answered in O(1) from the stores; ADR-0777, ADR-0782), in transactions of at most 500 rows (`retention_batch_rows`), at most 20 batches per kind per sweep. `runtime.mutex` is taken for one batch at a time, never across the whole sweep (ADR-0488, ADR-0489) | -| `DelayedEvents.cpp` | Delayed events (MSC4140, ADR-0823 to ADR-0826): `schedule_delayed_event`, `manage_delayed_event` (send, cancel, restart), `delayed_event_lookup`, the clock `delayed_event_now_ms` (and its seam `test_delayed_event_clock_ms`), `wake_delayed_events`, and `send_due_delayed_events`, the body of the `delayed-events` `PeriodicJob` in `main.cpp`. The caller holds `runtime.mutex` for all but the job body, which takes it for one send at a time | +| `DelayedEvents.cpp` | Delayed events (MSC4140, ADR-0823 to ADR-0826): `schedule_delayed_event`, `manage_delayed_event` (send, cancel, restart), `delayed_event_lookup`, the clock `delayed_event_now_ms` (and its seam `test_delayed_event_clock_ms`), `wake_delayed_events`, and `send_due_delayed_events`, the body of the `delayed-events` `PeriodicJob` in `main.cpp`. The caller holds `runtime.mutex` for all but the job body, which takes it for one send a run | | `ClientServerRuntime.cpp` (delayed events) | The MSC4140 routes (ADR-0823): S1 and S2 `PUT .../delayed_event/...`, the `org.matrix.msc4140.delay` branch of `PUT /send` and `/state`, M1 to M3 `POST .../delayed_events/{delayId}[/{action}]` and G1 and G2 `GET .../delayed_events/{delayId}`, with the `/versions` flags and `/capabilities` entries. See "Delayed events" below | | `ClientServerRuntimeInternal.hpp` | Call-throughs from `ClientServerRuntime.cpp`'s anonymous namespace that `DelayedEvents.cpp` shares with the `/send` and `/state` routes: `rate_limit_route_template` (`normalized_target`), `route_event_body` (`event_body_from_content`), `event_txn_part_is_too_long`, `errcode_for_refusal_status`, and the two reasons the `/send` route gives for a too-long type or transaction ID. Not public API | | `OAuthClientRegistration.cpp` | `POST /_merovingian/oauth2/register` (`register_oauth_client`), the client ID (`oauth_client_id_of`, ADR-0779), the cap and its eviction (ADR-0780), and `find_registered_oauth_client` | @@ -1214,9 +1214,12 @@ next layer) only parse a request and call it. Invariants a change must keep: the row stays scheduled. - **One sender at a time, marked.** A send takes `runtime.mutex` for one event, marks the row as being sent (`DelayedEventTable::mark_being_sent`, memory only) before `send_event_as_resolved_user`, and finalising clears the - mark. A marked row is offered by nobody: restart, cancel and send answer 409 "being sent". The job sends ONE event per - hold and at most 100 a run; it checks its `StopFlag` between events. A record the store refuses leaves the row - marked, so it is never sent again by that process; never clear the mark to retry. + mark. A marked row is offered by nobody: restart, cancel and send answer 409 "being sent". The job sends AT MOST ONE event + a run, in one hold, and returns `delayed_events_pace` (5 ms) when another is already due, so the lock is free between + sends as in the profile propagation job (ADR-0353); it never returns 0, and it checks its `StopFlag` before the send. + A record the store refuses, or that throws (`delayed_event.finalise_commit_failed`, `outcome` `exception`, the type + only), leaves the row marked, so it is never sent again by that process (the next start sends it once more); never + clear the mark to retry. - **Never early.** The job sends only a row whose `delayed_event_due_ms` is at or before the `now_ms` it was given, re-read under the lock it sends under. "Now" is `delayed_event_now_ms` (production and handlers) or the job's parameter (tests); never read the clock anywhere else, and never sleep in a test. diff --git a/src/homeserver/DelayedEvents.cpp b/src/homeserver/DelayedEvents.cpp index a62b26d10..fb700b7e7 100644 --- a/src/homeserver/DelayedEvents.cpp +++ b/src/homeserver/DelayedEvents.cpp @@ -54,8 +54,10 @@ namespace constexpr auto log_module = std::string_view{"delayed_events"}; - // At most this many events are sent in one run of the job, so other threads get the runtime lock between runs. - constexpr auto max_sends_per_run = std::size_t{100U}; + // The wait after a run that sent an event while another is already due: the runtime lock is free for this long + // between two sends, as `profile_propagation_pace` leaves it between two rooms (ADR-0353). The mutex is not fair, + // so a job that retook it at once could starve request threads for a whole backlog. + constexpr auto delayed_events_pace = std::uint64_t{5U}; // The age prune deletes at most this many finalised rows per run, in one transaction (ADR-0824). constexpr auto prune_batch_rows = std::size_t{500U}; // A finalised row is kept at most this long: 7 days. @@ -401,9 +403,24 @@ namespace observability::LogEventSeverity::error); result = refusal(500U, "M_UNKNOWN", "internal server error"); } - // Finalising clears the mark. A record that was refused leaves the row marked, so it is never sent again by - // this process (and is logged), never in a loop. - record_send_outcome(runtime, row, *result, now_ms); + // Finalising clears the mark. A record the store refused leaves the row marked, so it is never sent again by + // this process (and is logged), never in a loop. A record that threw is the same: the mark is kept and the + // next start sends the event once more (at least once, ADR-0825); clearing it would send it on every run. + try + { + record_send_outcome(runtime, row, *result, now_ms); + } + catch (...) + { + mark.keep_mark(); + log_event("delayed_event.finalise_commit_failed", row, + { + {"outcome", "exception", false}, + {"type", core::current_exception_type_name(), false}, + }, + observability::LogEventSeverity::error); + return true; + } if (table.is_being_sent(row.delay_id)) { mark.keep_mark(); @@ -673,11 +690,10 @@ auto delayed_event_lookup(ClientServerRuntime& runtime, std::string_view user_id auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, StopFlag const& stop) -> std::chrono::milliseconds { - // One send per hold of the runtime lock, the stop flag read between items, at most `max_sends_per_run` a run. - auto sent = std::size_t{0U}; - while (sent < max_sends_per_run && !stop.stop_requested() && send_next_due(runtime, now_ms)) + // At most one send a run, in its own hold of the runtime lock, and the lock free between runs (ADR-0353). + if (!stop.stop_requested()) { - ++sent; + std::ignore = send_next_due(runtime, now_ms); } auto const guard = RuntimeLock{runtime.homeserver.mutex}; auto& store = runtime.homeserver.database.persistent_store; @@ -697,8 +713,11 @@ auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, return std::chrono::milliseconds{max_job_wait_ms}; } auto const due_ms = database::delayed_event_due_ms(*next); - auto const wait_ms = due_ms > now_ms ? due_ms - now_ms : std::uint64_t{0U}; - return std::chrono::milliseconds{std::min(wait_ms, max_job_wait_ms)}; + if (due_ms <= now_ms) + { + return std::chrono::milliseconds{delayed_events_pace}; + } + return std::chrono::milliseconds{std::clamp(due_ms - now_ms, delayed_events_pace, max_job_wait_ms)}; } } // namespace merovingian::homeserver diff --git a/src/main.cpp b/src/main.cpp index 84f471f3e..c0a05d30b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -932,8 +932,10 @@ struct ListenerBinding final sync_pool.request_stop(); return std::nullopt; } - // Sends the delayed events (MSC4140, ADR-0825) that have fallen due, one send per hold of the runtime lock, and - // ages out finalised ones. It sleeps until the earliest deadline and is woken when an earlier one is scheduled. + // Sends the delayed events (MSC4140, ADR-0825) that have fallen due: at most one send per run, in its own hold of + // the runtime lock, and a run that left another event due waits `delayed_events_pace` (5 ms) with the lock free, + // as the profile propagation job does (ADR-0353). It also ages out finalised rows. Otherwise it sleeps until the + // earliest deadline (60 s at most) and is woken when an earlier one is scheduled. // It only ever touches `runtime`, which outlives this function. Declared after the pools so it is stopped and // joined before them. auto delayed_events_job = merovingian::homeserver::PeriodicJob{ @@ -1255,6 +1257,14 @@ struct ListenerBinding final LOG_INFO("Listeners active; awaiting traffic. Send SIGINT or SIGTERM to stop."); auto runtime = std::move(runtime_result.runtime); + // Once per start-up: delayed-event rows hydration refused to read exactly are left in the database and never + // sent (ADR-0825). The count only; never a row's content, state key or any other column. + if (auto const skipped = runtime.homeserver.database.persistent_store.delayed_event_rows_skipped_at_load; + skipped != 0U) + { + LOG_WARNING("delayed_events.rows_skipped_at_load: " + std::to_string(skipped) + + " delayed event rows were not read and stay in the database"); + } // Wire federation callbacks and start the outbound dispatch worker now that // the runtime is in its final stable location (no further moves). Doing this // eagerly means thread-creation failure surfaces at startup with a clear log From de14cfdbdeedc444649638c8651f0d8349bf1eb7 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:41:04 +0000 Subject: [PATCH 18/29] Read delayed-event rows strictly at hydration, count the skipped ones, and always delete on deactivation Co-Authored-By: Claude Fable 5.2 --- .../database/DelayedEventRowText.hpp | 29 ++++++ .../database/DelayedEventStore.hpp | 5 +- .../database/DelayedEventTable.hpp | 3 +- .../merovingian/database/PersistentStore.hpp | 4 + src/database/AGENTS.md | 9 +- src/database/DelayedEventRowText.cpp | 98 +++++++++++++++++++ src/database/DelayedEventStore.cpp | 6 +- src/database/DelayedEventTable.cpp | 5 +- src/database/PostgresqlStore.cpp | 19 ++-- src/database/SqliteStore.cpp | 23 ++--- src/database/meson.build | 1 + 11 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 include/merovingian/database/DelayedEventRowText.hpp create mode 100644 src/database/DelayedEventRowText.cpp diff --git a/include/merovingian/database/DelayedEventRowText.hpp b/include/merovingian/database/DelayedEventRowText.hpp new file mode 100644 index 000000000..028ccb53f --- /dev/null +++ b/include/merovingian/database/DelayedEventRowText.hpp @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "merovingian/database/PersistentDelayedEvent.hpp" + +#include +#include +#include +#include + +namespace merovingian::database +{ + +// The number of columns of a `delayed_events` row, in the order the loaders select them: `delay_id`, `user_id`, +// `device_id`, `room_id`, `event_type`, `is_state`, `state_key`, `content_json`, `txn_id`, `via_appservice`, +// `delay_ms`, `delayed_since_ms`, `status`, `finalised_at_ms`, `event_id`, `error_json`. +inline constexpr auto delayed_event_row_columns = std::size_t{16U}; + +// The strict reading of one stored `delayed_events` row, shared by both backends' loaders. Nullopt (the row is +// skipped, never repaired or guessed) unless: there are at least `delayed_event_row_columns` columns; `delay_ms`, +// `delayed_since_ms` and `finalised_at_ms` are each one or more ASCII digits that fit 64 bits; the status is one of +// the four; both boolean columns are exactly "true" or "false"; a scheduled row's `delay_ms` is between 1 and +// `max_delayed_event_delay_ms`; and a finalised row's `finalised_at_ms` is not 0. Reading a number as 0 would load a +// scheduled row as due at once, which is early. +[[nodiscard]] auto parse_delayed_event_row(std::span columns) + -> std::optional; + +} // namespace merovingian::database diff --git a/include/merovingian/database/DelayedEventStore.hpp b/include/merovingian/database/DelayedEventStore.hpp index a9625a9c8..7ee5f31f8 100644 --- a/include/merovingian/database/DelayedEventStore.hpp +++ b/include/merovingian/database/DelayedEventStore.hpp @@ -64,8 +64,9 @@ inline constexpr auto max_delayed_event_delay_ms = std::uint64_t{2'678'400'000U} std::string error_json, std::size_t max_finalised_per_user) -> DelayedEventOutcomeResult; -// Deletes every row of `user_id`, scheduled and finalised, in one `DELETE ... WHERE user_id = $1`. True when the user -// has none (nothing is written) or the rows are gone; false when the commit fails, with memory unchanged. +// Deletes every row of `user_id`, scheduled and finalised, in one `DELETE ... WHERE user_id = $1`, which always runs +// (a row hydration skipped is in the database only). True when the rows are gone, then memory is erased; false when the +// commit fails, with memory unchanged. [[nodiscard]] auto delete_delayed_events_of_user(PersistentStore& store, std::string_view user_id) -> bool; // Deletes finalised rows whose `finalised_at_ms` is below `cutoff_ms`, oldest first, at most `max_rows`, in one diff --git a/include/merovingian/database/DelayedEventTable.hpp b/include/merovingian/database/DelayedEventTable.hpp index 49cbb601f..5b68f4daf 100644 --- a/include/merovingian/database/DelayedEventTable.hpp +++ b/include/merovingian/database/DelayedEventTable.hpp @@ -74,7 +74,8 @@ class DelayedEventTable final [[nodiscard]] auto is_being_sent(std::string_view delay_id) const -> bool; // Adds `row`, indexing it by its status. False, and nothing changes, when its delay ID is held, or it has a - // transaction ID and another row already holds the same (user, device, room, event type, transaction ID). + // transaction ID and another row already holds the same (user, device, room, event type, transaction ID), or it is + // `sent` and another sent row already claims its event ID. auto insert(PersistentDelayedEvent row) -> bool; // Sets `delayed_since_ms` of a scheduled row, moving it in the due order. False when the row is absent or not // scheduled. diff --git a/include/merovingian/database/PersistentStore.hpp b/include/merovingian/database/PersistentStore.hpp index 91b8fe475..743b5eddb 100644 --- a/include/merovingian/database/PersistentStore.hpp +++ b/include/merovingian/database/PersistentStore.hpp @@ -579,6 +579,10 @@ struct PersistentStore final // DelayedEventStore.hpp; it has no lock of its own, so in main `runtime.mutex` serialises it. The federation worker // never holds it. DelayedEventTable delayed_events{}; + // How many `delayed_events` rows the last hydration skipped because they were not read exactly (a number that is + // not digits or is out of range, a boolean that is not "true" or "false", an unknown status, a duplicate key). The + // rows stay in the database. The homeserver warns once at start-up when this is not zero. + std::size_t delayed_event_rows_skipped_at_load{0U}; std::vector appservice_txn_cursors{}; // Events queued for appservices, kept sorted by (appservice_id, stream_ordering). See // PersistentAppserviceOutboxEntry. diff --git a/src/database/AGENTS.md b/src/database/AGENTS.md index ae194169c..d4777ef14 100644 --- a/src/database/AGENTS.md +++ b/src/database/AGENTS.md @@ -18,6 +18,7 @@ Manages schema migrations and provides a dual-backend (SQLite / PostgreSQL) pers | `DelayedEventTable.cpp` | `DelayedEventTable`, the in-memory copy of `delayed_events`: by delay ID, per user (scheduled by due time, finalised by finalised time), the due order that skips a being-sent row, by (user, device, room, type, transaction ID), by event ID, and the finalised rows by finalised time (ADR-0824). Also `delayed_event_due_ms` (a saturating sum) | | `DelayedEventStore.cpp` | The delayed-event store functions and their caps (`max_delayed_event_id_bytes` 64, `max_delayed_event_content_bytes` 65,536, `max_delayed_event_error_bytes` 1,024, `max_delayed_event_field_bytes` 255, `max_delayed_event_delay_ms` 31 days): store, restart, cancel, record the outcome of a send, delete a user's rows, prune by age. Finalising evicts the user's oldest finalised rows in the same transaction (ADR-0824, ADR-0825) | | `DelayedEventStatusText.cpp` | The `status` column's text for `DelayedEventStatus` and its strict parse (an unknown text makes hydration skip the row) | +| `DelayedEventRowText.cpp` | `parse_delayed_event_row`, the one strict reading of a stored row both loaders use: the three numeric columns are 1+ ASCII digits that fit 64 bits, both booleans are exactly `true` or `false`, a scheduled row's `delay_ms` is 1 to `max_delayed_event_delay_ms`, a finalised row has a non-zero `finalised_at_ms`. Anything else is skipped, never repaired or guessed | | `OAuthStore.cpp` | The OAuth 2.0 store functions and their caps (`max_oauth_client_metadata_bytes` 16 KiB, `max_oauth_scope_bytes` 2048, `max_oauth_client_id_bytes` 128): insert-or-keep clients, the once-a-day touch, delete a set of clients, sessionless clients oldest first, and the session row's prepare/remember pair | | `EventTable.cpp` | `EventTable`, the in-memory mirror of the `events` table with its indexes by event ID, stream order, room and stream order, room and time, and relation target (ADR-0215, ADR-0300, ADR-0430, ADR-0435) | | `RedactionView.cpp` | Client-facing redaction presentation: adds `redacts` to `m.room.redaction` events and `redacted_because` under `unsigned` of redacted events (CSAZ-11) | @@ -230,8 +231,12 @@ exception: `record_delayed_event_outcome` finalises memory even when its commit row left scheduled would be sent again on every run; the database row still reads `scheduled`, so the next start-up sends it once more, ADR-0825). A new function that finalises a row keeps that bounded eviction in the same transaction as the status update. The "being sent" mark is memory only and `restart` and `cancel` refuse a marked row. **Every path that -deletes a user's account removes the user's rows** with `delete_delayed_events_of_user`; logout and device deletion do -not (a call's delayed leave must outlive its device). `store_delayed_event` does not enforce the per-user cap on +deletes a user's account removes the user's rows** with `delete_delayed_events_of_user`, which always runs its +`DELETE` (a row hydration skipped is in the database only) and then erases memory; logout and device deletion do +not (a call's delayed leave must outlive its device). Hydration counts every row it skips in +`PersistentStore::delayed_event_rows_skipped_at_load` (a duplicate delay ID, transaction key or sent event ID counts), +and `main.cpp` logs it once as `delayed_events.rows_skipped_at_load`; never log a skipped row's columns. +`store_delayed_event` does not enforce the per-user cap on scheduled rows: the caller does, because it computes `Retry-After`. Never log a row's `content_json`. The federation worker never reads the table. diff --git a/src/database/DelayedEventRowText.cpp b/src/database/DelayedEventRowText.cpp new file mode 100644 index 000000000..076b29546 --- /dev/null +++ b/src/database/DelayedEventRowText.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "merovingian/database/DelayedEventRowText.hpp" + +#include "merovingian/database/DelayedEventStatus.hpp" +#include "merovingian/database/DelayedEventStatusText.hpp" +#include "merovingian/database/DelayedEventStore.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace merovingian::database +{ +namespace +{ + + // One or more ASCII digits whose value fits 64 bits; nullopt for anything else (empty, a sign, a space, a letter, + // 21 digits). + [[nodiscard]] auto strict_u64(std::string_view text) noexcept -> std::optional + { + if (text.empty()) + { + return std::nullopt; + } + auto value = std::uint64_t{0U}; + for (auto const character : text) + { + if (character < '0' || character > '9') + { + return std::nullopt; + } + auto const digit = static_cast(character - '0'); + if (value > (std::numeric_limits::max() - digit) / 10U) + { + return std::nullopt; + } + value = (value * 10U) + digit; + } + return value; + } + + // The two texts the store writes for a boolean. + [[nodiscard]] auto strict_bool(std::string_view text) noexcept -> std::optional + { + if (text == "true") + { + return true; + } + if (text == "false") + { + return false; + } + return std::nullopt; + } + +} // namespace + +auto parse_delayed_event_row(std::span columns) -> std::optional +{ + if (columns.size() < delayed_event_row_columns) + { + return std::nullopt; + } + auto const status = parse_delayed_event_status(columns[12]); + auto const is_state = strict_bool(columns[5]); + auto const via_appservice = strict_bool(columns[9]); + auto const delay_ms = strict_u64(columns[10]); + auto const delayed_since_ms = strict_u64(columns[11]); + auto const finalised_at_ms = strict_u64(columns[13]); + if (!status.has_value() || !is_state.has_value() || !via_appservice.has_value() || !delay_ms.has_value() || + !delayed_since_ms.has_value() || !finalised_at_ms.has_value()) + { + return std::nullopt; + } + if (*status == DelayedEventStatus::scheduled) + { + if (*delay_ms < 1U || *delay_ms > max_delayed_event_delay_ms) + { + return std::nullopt; + } + } + else if (*finalised_at_ms == 0U) + { + return std::nullopt; + } + return PersistentDelayedEvent{columns[0], columns[1], columns[2], columns[3], + columns[4], *is_state, columns[6], columns[7], + columns[8], *via_appservice, *delay_ms, *delayed_since_ms, + *status, *finalised_at_ms, columns[14], columns[15]}; +} + +} // namespace merovingian::database diff --git a/src/database/DelayedEventStore.cpp b/src/database/DelayedEventStore.cpp index 971a69c65..7d261ae0e 100644 --- a/src/database/DelayedEventStore.cpp +++ b/src/database/DelayedEventStore.cpp @@ -234,10 +234,8 @@ auto record_delayed_event_outcome(PersistentStore& store, std::string_view delay auto delete_delayed_events_of_user(PersistentStore& store, std::string_view user_id) -> bool { - if (store.delayed_events.scheduled_count(user_id) == 0U && store.delayed_events.finalised_count(user_id) == 0U) - { - return true; - } + // Always the DELETE, even when memory holds no row for the user: a row that hydration skipped is in the database + // and must not survive a deactivation. Deactivation is rare. auto const statement = PreparedStatement{ "delete_delayed_events_of_user", "DELETE FROM delayed_events WHERE user_id = $1", {text(user_id)}}; if (!commit_persistent_transaction(store, {statement})) diff --git a/src/database/DelayedEventTable.cpp b/src/database/DelayedEventTable.cpp index 7973257ec..aa0a0d1ba 100644 --- a/src/database/DelayedEventTable.cpp +++ b/src/database/DelayedEventTable.cpp @@ -218,7 +218,10 @@ auto DelayedEventTable::unindex_row(PersistentDelayedEvent const& row) -> void auto DelayedEventTable::insert(PersistentDelayedEvent row) -> bool { - if (m_rows.contains(row.delay_id) || (!row.txn_id.empty() && m_by_transaction.contains(transaction_key_of(row)))) + // A second `sent` row for an event ID is refused like a duplicate transaction key: the index would name only the + // first, and erasing it would orphan the second. + if (m_rows.contains(row.delay_id) || (!row.txn_id.empty() && m_by_transaction.contains(transaction_key_of(row))) || + (row.status == DelayedEventStatus::sent && m_by_event_id.contains(row.event_id))) { return false; } diff --git a/src/database/PostgresqlStore.cpp b/src/database/PostgresqlStore.cpp index 1eb80fbf0..b78507c8b 100644 --- a/src/database/PostgresqlStore.cpp +++ b/src/database/PostgresqlStore.cpp @@ -8,7 +8,7 @@ #include "merovingian/canonicaljson/Parser.hpp" #include "merovingian/canonicaljson/Serializer.hpp" #include "merovingian/database/ConnectionPool.hpp" -#include "merovingian/database/DelayedEventStatusText.hpp" +#include "merovingian/database/DelayedEventRowText.hpp" #include "merovingian/database/Migration.hpp" #include "merovingian/database/PostgresqlConnection.hpp" #include "merovingian/database/PostgresqlConnectionDeleter.hpp" @@ -1422,7 +1422,8 @@ namespace } // Migration 003 (ADR-0824): delayed events, main only (the federation worker sends no events). A row whose - // status text is not one of the four, or that duplicates a delay ID or a transaction key, is skipped. + // status text is not one of the four, that duplicates a delay ID, a transaction key or a sent event ID, or + // whose numbers or booleans are not exactly what the store writes, is skipped and counted, never repaired. if (table_load_profile_includes("delayed_events", profile)) { auto delayed = query_rows(connection, "postgresql_load_delayed_events", @@ -1435,17 +1436,11 @@ namespace } for (auto const& row : delayed.rows) { - if (row.size() < 16U) + // Strict: a row that is not read exactly is skipped, left in the database and counted. + auto parsed = parse_delayed_event_row(row); + if (!parsed.has_value() || !store.delayed_events.insert(std::move(*parsed))) { - continue; - } - auto const status = parse_delayed_event_status(row[12]); - if (status.has_value()) - { - std::ignore = store.delayed_events.insert( - {row[0], row[1], row[2], row[3], row[4], text_is_true(row[5]), row[6], row[7], row[8], - text_is_true(row[9]), parse_u64(row[10]), parse_u64(row[11]), *status, parse_u64(row[13]), - row[14], row[15]}); + ++store.delayed_event_rows_skipped_at_load; } } } diff --git a/src/database/SqliteStore.cpp b/src/database/SqliteStore.cpp index e69f866b0..94acafbcf 100644 --- a/src/database/SqliteStore.cpp +++ b/src/database/SqliteStore.cpp @@ -7,7 +7,7 @@ #include "merovingian/canonicaljson/Serializer.hpp" #include "merovingian/core/FileDescriptor.hpp" #include "merovingian/database/ConnectionPool.hpp" -#include "merovingian/database/DelayedEventStatusText.hpp" +#include "merovingian/database/DelayedEventRowText.hpp" #include "merovingian/database/ExaminedRows.hpp" #include "merovingian/database/Migration.hpp" #include "merovingian/database/MigrationStep.hpp" @@ -1184,23 +1184,24 @@ namespace parse_u64(column_text(row, 4))}); }) && // Migration 003 (ADR-0824): delayed events. A row whose status text is not one of the four, or that - // duplicates a delay ID or a transaction key, is skipped, never repaired. + // duplicates a delay ID, a transaction key or a sent event ID, or whose numbers + // or booleans are not exactly what the store writes, is skipped and counted, never repaired. load_rows(connection, "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, content_json, " "txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " "error_json FROM delayed_events ORDER BY delay_id", [&store](sqlite3_stmt& row) { - auto const status = parse_delayed_event_status(column_text(row, 12)); - if (!status.has_value()) + auto columns = std::array{}; + for (auto index = std::size_t{0U}; index < columns.size(); ++index) { - return; + columns[index] = column_text(row, static_cast(index)); + } + // Strict: a row that is not read exactly is skipped, left in the file and counted. + auto parsed = parse_delayed_event_row(columns); + if (!parsed.has_value() || !store.delayed_events.insert(std::move(*parsed))) + { + ++store.delayed_event_rows_skipped_at_load; } - std::ignore = store.delayed_events.insert( - {column_text(row, 0), column_text(row, 1), column_text(row, 2), column_text(row, 3), - column_text(row, 4), text_is_true(column_text(row, 5)), column_text(row, 6), - column_text(row, 7), column_text(row, 8), text_is_true(column_text(row, 9)), - parse_u64(column_text(row, 10)), parse_u64(column_text(row, 11)), *status, - parse_u64(column_text(row, 13)), column_text(row, 14), column_text(row, 15)}); }) && // Application service delivery state: transaction cursors, the outbox and the directory mapping. load_rows(connection, diff --git a/src/database/meson.build b/src/database/meson.build index e00f7b85f..538b14505 100644 --- a/src/database/meson.build +++ b/src/database/meson.build @@ -1,5 +1,6 @@ database_sources = files( 'Connection.cpp', + 'DelayedEventRowText.cpp', 'DelayedEventStatusText.cpp', 'DelayedEventStore.cpp', 'DelayedEventTable.cpp', From faad02ad2f5480ba1a1434c4a75183bb2b127576 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:41:04 +0000 Subject: [PATCH 19/29] Test the one-send pacing, the strict hydration, the review's route cases and the always-delete Co-Authored-By: Claude Fable 5.2 --- .../integration/test_delayed_events_flow.cpp | 298 ++++++++++++++-- .../test_delayed_events_routes_flow.cpp | 333 +++++++++++++++++- .../test_postgresql_store_behaviour_flow.cpp | 98 ++++++ tests/unit/test_delayed_event_store.cpp | 193 +++++++++- tests/unit/test_delayed_event_table.cpp | 31 ++ 5 files changed, 918 insertions(+), 35 deletions(-) diff --git a/tests/integration/test_delayed_events_flow.cpp b/tests/integration/test_delayed_events_flow.cpp index aee6f4f2a..5e39e32c3 100644 --- a/tests/integration/test_delayed_events_flow.cpp +++ b/tests/integration/test_delayed_events_flow.cpp @@ -210,6 +210,23 @@ class Fixture final return hs::send_due_delayed_events(fixture.runtime(), now_ms, stop); } +// Runs the job until no scheduled event is due at `now_ms` (the job sends at most one a run), and returns how many runs +// it took. A bound stops a job that makes no progress. +auto drain_job(Fixture& fixture, std::uint64_t now_ms) -> std::size_t +{ + auto runs = std::size_t{0U}; + for (; runs < 1'000U; ++runs) + { + auto const* next = fixture.table().earliest_due(); + if (next == nullptr || db::delayed_event_due_ms(*next) > now_ms) + { + break; + } + std::ignore = run_job(fixture, now_ms); + } + return runs; +} + [[nodiscard]] auto scheduled_id(Fixture& fixture, hs::DelayedEventScheduleRequest const& made) -> std::string { auto const result = schedule(fixture, made); @@ -419,11 +436,11 @@ SCENARIO("A delayed event is never sent early, and is then sent once with its de { auto const wait = run_job(fixture, due - 1U); - THEN("nothing is sent and the job asks to run again in one millisecond") + THEN("nothing is sent and the job asks to run again after the 5 ms pace (the 1 ms left is raised to it)") { CHECK(fixture.event_count() == before); CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); - CHECK(wait == std::chrono::milliseconds{1}); + CHECK(wait == std::chrono::milliseconds{5}); } } @@ -481,7 +498,7 @@ SCENARIO("A delayed state event is sent as a state event and stamped with the ti WHEN("the job runs after the due time") { - std::ignore = run_job(fixture, t0 + 30'000U); + drain_job(fixture, t0 + 30'000U); THEN("the event is a state event whose origin_server_ts is the send time, not the schedule time") { @@ -516,11 +533,11 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o CHECK(restarted.value.empty()); CHECK(*int_member(looked_up(fixture, alice, id), "delayed_since_ts") == static_cast(t0 + 20'000U)); - std::ignore = run_job(fixture, t0 + 60'000U); + drain_job(fixture, t0 + 60'000U); CHECK(fixture.event_count() == before); - std::ignore = run_job(fixture, t0 + 80'000U - 1U); + drain_job(fixture, t0 + 80'000U - 1U); CHECK(fixture.event_count() == before); - std::ignore = run_job(fixture, t0 + 80'000U); + drain_job(fixture, t0 + 80'000U); CHECK(fixture.event_count() == before + 1U); } } @@ -536,7 +553,7 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o auto const finalised = *object_member_as_object(looked_up(fixture, alice, id), "finalised"); CHECK(string_member(finalised, "event_id") == nullptr); CHECK(object_member_as_object(finalised, "error") == nullptr); - std::ignore = run_job(fixture, t0 + 10'000'000U); + drain_job(fixture, t0 + 10'000'000U); CHECK(fixture.event_count() == before); } } @@ -555,7 +572,7 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o fixture.bob_token, "{}") .status == 200U); // `sent` is due first (1 s), then `failed` (bob has left, 2 s); `scheduled` (10 s) is not yet due. - std::ignore = run_job(fixture, t0 + 5'000U); + drain_job(fixture, t0 + 5'000U); REQUIRE(status_of(fixture, alice, sent) == db::DelayedEventStatus::sent); REQUIRE(status_of(fixture, bob, failed) == db::DelayedEventStatus::failed); REQUIRE(fixture.table().mark_being_sent(being_sent)); @@ -666,7 +683,7 @@ SCENARIO("A delayed event is authorised when it is sent, not when it was schedul { set_power(fixture, bob, 0); auto const before = fixture.event_count(); - std::ignore = run_job(fixture, t0 + 10'000U); + drain_job(fixture, t0 + 10'000U); THEN("the state event is not sent and fails with the room's refusal, and is not tried again") { @@ -678,7 +695,7 @@ SCENARIO("A delayed event is authorised when it is sent, not when it was schedul // The message needs no power, so exactly it was sent. CHECK(status_of(fixture, bob, message_id) == db::DelayedEventStatus::sent); CHECK(fixture.event_count() == before + 1U); - std::ignore = run_job(fixture, t0 + 20'000U); + drain_job(fixture, t0 + 20'000U); CHECK(fixture.event_count() == before + 1U); } } @@ -695,9 +712,9 @@ SCENARIO("A delayed event is authorised when it is sent, not when it was schedul WHEN("the due time passes and the job runs twice") { - std::ignore = run_job(fixture, t0 + 10'000U); + drain_job(fixture, t0 + 10'000U); auto const after_first = fixture.event_count(); - std::ignore = run_job(fixture, t0 + 20'000U); + drain_job(fixture, t0 + 20'000U); THEN("nothing is sent, the row is failed with the error in the lookup, and no second attempt is made") { @@ -757,7 +774,7 @@ SCENARIO("A suspended, locked or deactivated sender sends nothing when the event WHEN("the due time passes") { - std::ignore = run_job(fixture, t0 + 10'000U); + drain_job(fixture, t0 + 10'000U); THEN("the event is failed with the account's errcode and nothing is sent") { @@ -790,6 +807,8 @@ SCENARIO("The send-time rate limit fails a due event when the user's per-user po { GIVEN("a per-user policy of one send a minute for the send route, and two due events") { + // The rate-limit engine reads the steady clock itself (`ClientServerClock`, no test seam), so this relies on + // the two sends completing within the policy's 60-second window. auto limits = merovingian::config::ClientRateLimitsConfig{}; limits.per_user["/_matrix/client/v3/rooms/{roomId}/send/"] = merovingian::http::RateLimitPolicy{1U, 60U}; auto fixture = Fixture{memory_config(std::move(limits))}; @@ -798,7 +817,7 @@ SCENARIO("The send-time rate limit fails a due event when the user's per-user po WHEN("the job runs when both are due") { - std::ignore = run_job(fixture, t0 + 5'000U); + drain_job(fixture, t0 + 5'000U); THEN("the first is sent and the second fails with M_LIMIT_EXCEEDED") { @@ -819,7 +838,7 @@ SCENARIO("The send-time rate limit fails a due event when the user's per-user po WHEN("the job runs when both are due") { - std::ignore = run_job(fixture, t0 + 5'000U); + drain_job(fixture, t0 + 5'000U); THEN("both are sent") { @@ -924,7 +943,7 @@ SCENARIO("With the feature disabled nothing new is scheduled, but what was sched { auto const refused = schedule(fixture, message_request(fixture, alice, "c")); auto const cancel = manage(fixture, alice, cancelled, hs::DelayedEventAction::cancel); - std::ignore = run_job(fixture, t0 + 10'000U); + drain_job(fixture, t0 + 10'000U); THEN("the schedule is 403 M_FORBIDDEN, the cancel works and the due event is sent") { @@ -1096,7 +1115,7 @@ SCENARIO("A delayed event survives a restart, and one that fell due while the se CHECK(held->status == db::DelayedEventStatus::sent); CHECK(early_count == before); CHECK(homeserver.database.persistent_store.events.size() == before + 1U); - CHECK(early == std::chrono::milliseconds{1}); + CHECK(early == std::chrono::milliseconds{5}); CHECK(late == std::chrono::milliseconds{60'000}); } } @@ -1118,8 +1137,8 @@ SCENARIO("A failed outcome commit leaves the event sent once and never sent agai WHEN("the job runs twice") { - std::ignore = run_job(fixture, t0 + 10'000U); - std::ignore = run_job(fixture, t0 + 20'000U); + drain_job(fixture, t0 + 10'000U); + drain_job(fixture, t0 + 20'000U); THEN("the event was sent once and memory says it was sent") { @@ -1216,6 +1235,48 @@ SCENARIO("Deactivating an account deletes every delayed event of that user and n } } +SCENARIO("Deactivation succeeds when the delayed-events delete fails, and the job then fails the user's due event", + "[homeserver][delayed-events][deactivation]") +{ + GIVEN("a scheduled event of bob and a delete of delayed events that fails") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, bob, "x", 10'000U)); + auto const before = fixture.event_count(); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "delete_delayed_events_of_user"); + + WHEN("bob's account is deactivated and the job runs at the due time") + { + { + auto const guard = hs::RuntimeLock{fixture.runtime().homeserver.mutex}; + auto const result = hs::deactivate_user_account(fixture.runtime().homeserver, bob, + {std::string{bob}, std::string{bob}, "test"}); + REQUIRE(result.ok); + } + auto const still_held = fixture.table().find(id) != nullptr; + drain_job(fixture, t0 + 10'000U); + + THEN("the account is deactivated, and the event is failed M_USER_DEACTIVATED and not sent") + { + auto deactivated = false; + for (auto const& user : fixture.runtime().homeserver.database.users) + { + deactivated = deactivated || (user.user_id == bob && user.deactivated); + } + CHECK(deactivated); + CHECK(still_held); + CHECK(status_of(fixture, bob, id) == db::DelayedEventStatus::failed); + auto const error = *object_member_as_object( + *object_member_as_object(looked_up(fixture, bob, id), "finalised"), "error"); + CHECK(*string_member(error, "errcode") == "M_USER_DEACTIVATED"); + CHECK(fixture.event_count() == before); + } + } + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.clear(); + } +} + SCENARIO("The job's return value says when it should run next, and it stops when asked", "[homeserver][delayed-events][job][pacing]") { @@ -1252,27 +1313,61 @@ SCENARIO("The job's return value says when it should run next, and it stops when } } - GIVEN("102 events due at once") + GIVEN("102 events due at once, due 1 ms apart") { auto fixture = Fixture{memory_config()}; fixture.runtime().limits.max_scheduled_delayed_events_per_user = 1000U; + auto ids = std::vector{}; for (auto index = 0; index < 102; ++index) { - std::ignore = scheduled_id(fixture, message_request(fixture, alice, std::to_string(index), 1'000U)); + // Scheduled newest-deadline first, so the deadline order is not the order they were stored in. + ids.push_back(scheduled_id(fixture, message_request(fixture, alice, std::to_string(index), + 2'000U - static_cast(index)))); } auto const before = fixture.event_count(); WHEN("the job runs once") { - auto const wait = run_job(fixture, t0 + 1'000U); + auto const wait = run_job(fixture, t0 + 5'000U); - THEN("it sent 100 and asks to run again at once, and the next run sends the other two") + THEN("it sent exactly the earliest one and asks to run again after the pace, never at once") { - CHECK(fixture.event_count() == before + 100U); - CHECK(wait == std::chrono::milliseconds{0}); - auto const second = run_job(fixture, t0 + 1'000U); + CHECK(fixture.event_count() == before + 1U); + CHECK(status_of(fixture, alice, ids[101]) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, ids[100]) == db::DelayedEventStatus::scheduled); + CHECK(wait == std::chrono::milliseconds{5}); + } + } + + WHEN("the job runs three times") + { + auto const first = run_job(fixture, t0 + 5'000U); + auto const second = run_job(fixture, t0 + 5'000U); + auto const third = run_job(fixture, t0 + 5'000U); + + THEN("three events are sent in deadline order, each run returning the pace") + { + CHECK(fixture.event_count() == before + 3U); + CHECK(status_of(fixture, alice, ids[101]) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, ids[100]) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, ids[99]) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, ids[98]) == db::DelayedEventStatus::scheduled); + CHECK(first == std::chrono::milliseconds{5}); + CHECK(second == std::chrono::milliseconds{5}); + CHECK(third == std::chrono::milliseconds{5}); + } + } + + WHEN("the job is run until nothing is due") + { + auto const runs = drain_job(fixture, t0 + 5'000U); + auto const last = run_job(fixture, t0 + 5'000U); + + THEN("it took one run per event, and the run with nothing due asks for 60 seconds") + { + CHECK(runs == 102U); CHECK(fixture.event_count() == before + 102U); - CHECK(second == std::chrono::milliseconds{60'000}); + CHECK(last == std::chrono::milliseconds{60'000}); } } @@ -1280,7 +1375,7 @@ SCENARIO("The job's return value says when it should run next, and it stops when { auto stop = hs::StopFlag{}; stop.request_stop(); - std::ignore = run_job(fixture, t0 + 1'000U, stop); + std::ignore = run_job(fixture, t0 + 5'000U, stop); THEN("nothing is sent") { @@ -1289,6 +1384,153 @@ SCENARIO("The job's return value says when it should run next, and it stops when } } } + + GIVEN("one event due now and one due in 2 ms") + { + auto fixture = Fixture{memory_config()}; + std::ignore = scheduled_id(fixture, message_request(fixture, alice, "a", 1'000U)); + std::ignore = scheduled_id(fixture, message_request(fixture, alice, "b", 1'002U)); + + WHEN("the job runs at the first deadline") + { + auto const wait = run_job(fixture, t0 + 1'000U); + + THEN("the wait to the next deadline is raised to the pace") + { + CHECK(wait == std::chrono::milliseconds{5}); + } + } + } +} + +SCENARIO("An exception while recording the outcome keeps the mark, so the event is not sent again by this process", + "[homeserver][delayed-events][job][exception]") +{ + GIVEN("a due event, and the number of backend commits one send and its record make") + { + auto commits = std::size_t{0U}; + { + auto calibration = Fixture{memory_config()}; + std::ignore = scheduled_id(calibration, message_request(calibration, alice, "boom", 10'000U)); + calibration.runtime().homeserver.database.persistent_store.test_before_backend_commit = + [&commits](std::size_t) { + ++commits; + }; + drain_job(calibration, t0 + 10'000U); + calibration.runtime().homeserver.database.persistent_store.test_before_backend_commit = nullptr; + } + REQUIRE(commits >= 2U); + + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "boom", 10'000U)); + auto const before = fixture.event_count(); + auto seen = std::size_t{0U}; + fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = [&seen, + commits](std::size_t) { + if (++seen == commits) + { + throw std::runtime_error{"simulated outcome commit failure"}; + } + }; + + WHEN("the job runs at the due time and twice afterwards") + { + CHECK_NOTHROW(std::ignore = run_job(fixture, t0 + 10'000U)); + auto const after_first = fixture.event_count(); + CHECK_NOTHROW(std::ignore = run_job(fixture, t0 + 20'000U)); + CHECK_NOTHROW(std::ignore = run_job(fixture, t0 + 30'000U)); + + THEN("the event was sent once, the row is still scheduled and marked, and nothing is sent again") + { + REQUIRE(seen == commits); + CHECK(after_first == before + 1U); + CHECK(fixture.event_count() == before + 1U); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + CHECK(fixture.table().is_being_sent(id)); + } + } + fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = nullptr; + } +} + +SCENARIO("The send action is subject to the user's rate limit for the send route", + "[homeserver][delayed-events][manage][rate-limit]") +{ + GIVEN("a per-user policy of one send a minute for the send route and two scheduled events") + { + // The rate-limit engine reads the steady clock itself (`ClientServerClock`, no test seam), so this relies on + // the two sends completing within the policy's 60-second window. + auto limits = merovingian::config::ClientRateLimitsConfig{}; + limits.per_user["/_matrix/client/v3/rooms/{roomId}/send/"] = merovingian::http::RateLimitPolicy{1U, 60U}; + auto fixture = Fixture{memory_config(std::move(limits))}; + auto const first = scheduled_id(fixture, message_request(fixture, alice, "a", 60'000U)); + auto const second = scheduled_id(fixture, message_request(fixture, alice, "b", 60'000U)); + + WHEN("the first is sent now, and then the second") + { + auto const sent = manage(fixture, alice, first, hs::DelayedEventAction::send); + auto const limited = manage(fixture, alice, second, hs::DelayedEventAction::send); + + THEN("the second is 429 M_LIMIT_EXCEEDED with retry_after_ms and is still scheduled") + { + CHECK(sent.ok); + CHECK_FALSE(limited.ok); + CHECK(limited.status == 429U); + CHECK(limited.errcode == "M_LIMIT_EXCEEDED"); + CHECK(limited.retry_after_ms > 0U); + CHECK(status_of(fixture, alice, first) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, second) == db::DelayedEventStatus::scheduled); + CHECK_FALSE(fixture.table().is_being_sent(second)); + } + } + } +} + +SCENARIO("A restart is persisted: the reopened event's timer starts at the restart and it is sent at the new deadline", + "[homeserver][delayed-events][persistence]") +{ + GIVEN("an event scheduled for 60 s on a SQLite file, restarted 10 s later, and the runtime started again") + { + auto const path = unique_sqlite_path(); + std::filesystem::remove(path); + auto const config = sqlite_config(path); + auto id = std::string{}; + { + auto first = Fixture{config}; + id = scheduled_id(first, message_request(first, alice, "txn-1", 60'000U)); + first.runtime().homeserver.test_delayed_event_clock_ms = t0 + 10'000U; + REQUIRE(manage(first, alice, id, hs::DelayedEventAction::restart).ok); + } + auto restarted = hs::start_client_server(config); + REQUIRE(restarted.started); + auto& homeserver = restarted.runtime.homeserver; + homeserver.test_frozen_event_clock_ms = static_cast(t0); + + WHEN("the job runs at the original deadline and then at the new one") + { + auto const before = homeserver.database.persistent_store.events.size(); + std::ignore = hs::send_due_delayed_events(restarted.runtime, t0 + 60'000U, {}); + auto const at_original = homeserver.database.persistent_store.events.size(); + auto const* held = homeserver.database.persistent_store.delayed_events.find_owned(alice, id); + REQUIRE(held != nullptr); + auto const since = held->delayed_since_ms; + auto const scheduled_at_original = held->status == db::DelayedEventStatus::scheduled; + std::ignore = hs::send_due_delayed_events(restarted.runtime, t0 + 70'000U, {}); + + THEN("the timer started at the restart, nothing was sent at the original deadline, and it is sent at the " + "new") + { + CHECK(since == t0 + 10'000U); + CHECK(at_original == before); + CHECK(scheduled_at_original); + CHECK(homeserver.database.persistent_store.events.size() == before + 1U); + CHECK(homeserver.database.persistent_store.delayed_events.find_owned(alice, id)->status == + db::DelayedEventStatus::sent); + } + } + + std::filesystem::remove(path); + } } SCENARIO("A send that throws in the job is recorded failed with M_UNKNOWN and is not sent again", diff --git a/tests/integration/test_delayed_events_routes_flow.cpp b/tests/integration/test_delayed_events_routes_flow.cpp index d829e064e..42a3ff77d 100644 --- a/tests/integration/test_delayed_events_routes_flow.cpp +++ b/tests/integration/test_delayed_events_routes_flow.cpp @@ -59,7 +59,7 @@ constexpr auto unstable = std::string_view{"/_matrix/client/unstable/org.matrix. constexpr auto delay_param = std::string_view{"org.matrix.msc4140.delay"}; constexpr auto message_content = std::string_view{R"({"body":"later"})"}; -[[nodiscard]] auto memory_config() -> merovingian::config::Config +[[nodiscard]] auto memory_config(merovingian::config::ClientRateLimitsConfig limits = {}) -> merovingian::config::Config { auto security = merovingian::config::SecurityConfig{}; security.secrets.master_key_file = shared_master_key_file(); @@ -69,7 +69,7 @@ constexpr auto message_content = std::string_view{R"({"body":"later"})"}; merovingian::config::ListenersConfig{}, in_memory_database_config(), std::move(security), - merovingian::config::ClientRateLimitsConfig{}, + std::move(limits), merovingian::config::LogModulesConfig{}, }; } @@ -117,6 +117,21 @@ constexpr auto message_content = std::string_view{R"({"body":"later"})"}; return *token; } +// Signs an existing user in on a new device; the access token. +[[nodiscard]] auto log_in(hs::ClientServerRuntime& runtime, std::string const& name, std::string const& device) + -> std::string +{ + auto const login = + request(runtime, "POST", "/_matrix/client/v3/login", {}, + R"({"type":"m.login.password","identifier":{"type":"m.id.user","user":"@)" + name + + R"(:example.org"},"password":")" + std::string{password} + R"(","device_id":")" + device + R"("})"); + REQUIRE(login.status == 200U); + auto const body = parse_object(login.body); + auto const* token = string_member(body, "access_token"); + REQUIRE(token != nullptr); + return *token; +} + // A running homeserver with alice and bob in one public room. Alice created it. class Fixture final { @@ -164,10 +179,19 @@ class Fixture final runtime().homeserver.test_frozen_event_clock_ms = static_cast(now_ms); } - // The job body at `now_ms`. + // The job body at `now_ms`, run until no scheduled event is due then: one run sends at most one event. A bound + // stops a job that makes no progress. auto run_job(std::uint64_t now_ms) -> void { - std::ignore = hs::send_due_delayed_events(runtime(), now_ms, hs::StopFlag{}); + for (auto runs = 0; runs < 1'000; ++runs) + { + auto const* next = table().earliest_due(); + if (next == nullptr || db::delayed_event_due_ms(*next) > now_ms) + { + return; + } + std::ignore = hs::send_due_delayed_events(runtime(), now_ms, hs::StopFlag{}); + } } std::string alice_token{}; @@ -352,6 +376,20 @@ SCENARIO("Every scheduling form answers a delay ID, sends nothing early and send CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); CHECK(timeline_has(fixture, event_type)); } + AND_THEN("a state form produced the state key it named, and no other") + { + fixture.set_time(t0 + 30'000U); + fixture.run_job(t0 + 30'000U); + if (is_state) + { + auto const keyed = state_content(fixture, event_type, "key"); + auto const empty_key = state_content(fixture, event_type); + // Form 3 names the key `key`; forms 4 (`/`) and 5 (no key segment) mean the empty state key. + CHECK((form == 3 ? keyed.status : empty_key.status) == 200U); + CHECK((form == 3 ? empty_key.status : keyed.status) == 404U); + CHECK((form == 3 ? keyed.body : empty_key.body).find("\"v\":1") != std::string::npos); + } + } } } } @@ -597,11 +635,17 @@ SCENARIO("The delay parameter is read with leading zeros and an encoded name, an "org%2Ematrix.msc4140.delay=5000"); auto const not_object = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "z3", std::string{delay_param} + "=5000", "[]"); + auto const state_encoded = put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", + "org%2Ematrix.msc4140.delay=5000"); + auto const state_padded = put_state_delayed(fixture, fixture.alice_token, "m.test.state", "/k", + std::string{delay_param} + "=0007"); THEN("the first two are scheduled, the third is M_BAD_JSON, and nothing is sent") { CHECK(delay_id_of(padded).size() > 0U); CHECK(delay_id_of(encoded).size() > 0U); + CHECK(delay_id_of(state_encoded).size() > 0U); + CHECK(delay_id_of(state_padded).size() > 0U); CHECK(not_object.status == 400U); CHECK(errcode_of(not_object) == "M_BAD_JSON"); CHECK(fixture.event_count() == before); @@ -1238,6 +1282,8 @@ SCENARIO("/versions and /capabilities advertise delayed events with the configur SCENARIO("The built-in per-user limits stop the 121st management request and the 61st scheduling request", "[homeserver][delayed-events][routes][rate-limit]") { + // The rate-limit engine reads the steady clock itself (`ClientServerClock`, no test seam), so this scenario relies + // on its 121 (or 61) in-process requests completing within the policy's 60-second window. GIVEN("alice and bob on one server with the default rate limits") { auto fixture = Fixture{}; @@ -1388,8 +1434,27 @@ SCENARIO("A NUL in the content of a delayed event is refused as a raw byte and k CHECK(row->content_json.find(R"(\u0000)") != std::string::npos); CHECK(get_delayed(fixture, true, id, fixture.alice_token).status == 200U); } + } + AND_THEN("after the job each is sent and the sent event holds the escape, as the client sent it") + { + auto ids = std::vector{}; + for (auto const* response : {&l1, &l2, &s1, &s2}) + { + ids.push_back(delay_id_of(*response)); + } fixture.set_time(t0 + 5'000U); fixture.run_job(t0 + 5'000U); + CHECK(fixture.event_count() == before + 4U); + for (auto const& id : ids) + { + auto const* row = fixture.table().find_owned(alice, id); + REQUIRE(row != nullptr); + CHECK(row->status == db::DelayedEventStatus::sent); + auto const event = request(fixture.runtime(), "GET", + fixture.room_path() + "/event/" + row->event_id, fixture.alice_token); + REQUIRE(event.status == 200U); + CHECK(event.body.find(R"("a":"x\u0000y")") != std::string::npos); + } } } } @@ -1426,3 +1491,263 @@ SCENARIO("Any other route that sends from the caller's body refuses the delay pa } } } + +SCENARIO("The delayed branch comes before the ordinary route's idempotency lookup, and the two namespaces are separate", + "[homeserver][delayed-events][routes][idempotency]") +{ + GIVEN("alice in a room") + { + auto fixture = Fixture{}; + + WHEN("an ordinary send is made, then the same path and transaction ID with the delay parameter") + { + auto const ordinary = request(fixture.runtime(), "PUT", fixture.room_path() + "/send/m.test.delayed/tx1", + fixture.alice_token, std::string{message_content}); + REQUIRE(ordinary.status == 200U); + REQUIRE(string_member(parse_object(ordinary.body), "event_id") != nullptr); + auto const before = fixture.event_count(); + auto const delayed = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "tx1", + std::string{delay_param} + "=5000"); + + THEN("the answer is a delay_id, not the cached event_id, and nothing new is in the timeline") + { + REQUIRE(delayed.status == 200U); + auto const body = parse_object(delayed.body); + CHECK(string_member(body, "delay_id") != nullptr); + CHECK(string_member(body, "event_id") == nullptr); + CHECK(fixture.event_count() == before); + } + } + + WHEN("an event is scheduled with a transaction ID, then sent ordinarily with the same one") + { + auto const delayed = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "tx2", + std::string{delay_param} + "=5000"); + REQUIRE(delayed.status == 200U); + auto const before = fixture.event_count(); + auto const ordinary = request(fixture.runtime(), "PUT", fixture.room_path() + "/send/m.test.delayed/tx2", + fixture.alice_token, std::string{message_content}); + + THEN("the ordinary send sends an ordinary event") + { + REQUIRE(ordinary.status == 200U); + CHECK(string_member(parse_object(ordinary.body), "event_id") != nullptr); + CHECK(fixture.event_count() == before + 1U); + } + } + } +} + +SCENARIO( + "The send action under the user's per-user limit for the send route is 429 with Retry-After and stays scheduled", + "[homeserver][delayed-events][routes][rate-limit]") +{ + GIVEN("a per-user policy of one send a minute for the send route and two scheduled events") + { + // The engine reads the steady clock itself (no test seam): the two sends below rely on completing within the + // policy's 60-second window. + auto limits = merovingian::config::ClientRateLimitsConfig{}; + limits.per_user["/_matrix/client/v3/rooms/{roomId}/send/"] = merovingian::http::RateLimitPolicy{1U, 60U}; + auto fixture = Fixture{memory_config(std::move(limits))}; + auto const first = schedule_message(fixture, "a"); + auto const second = schedule_message(fixture, "b"); + + WHEN("the first is sent now and then the second") + { + auto const sent = post_action(fixture, true, first, "send", fixture.alice_token); + auto const limited = post_action(fixture, true, second, "send", fixture.alice_token); + + THEN("the second is 429 M_LIMIT_EXCEEDED with retry_after_ms and Retry-After, and is still scheduled") + { + CHECK(sent.status == 200U); + CHECK(limited.status == 429U); + CHECK(errcode_of(limited) == "M_LIMIT_EXCEEDED"); + auto const limited_body = parse_object(limited.body); + CHECK(int_member(limited_body, "retry_after_ms") != nullptr); + CHECK_FALSE(header_of(limited, "Retry-After").empty()); + CHECK(status_of(fixture, alice, second) == db::DelayedEventStatus::scheduled); + } + } + } +} + +SCENARIO("Scheduling wakes the job once per scheduled event, and a refusal or a restart does not", + "[homeserver][delayed-events][routes][wake]") +{ + GIVEN("a runtime whose wake hook counts") + { + auto fixture = Fixture{}; + auto wakes = 0; + fixture.runtime().homeserver.delayed_events_wake = [&wakes]() { + ++wakes; + }; + + WHEN("an event is scheduled, a schedule is refused, and the event is restarted") + { + auto const scheduled = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "w1", schedule_body(5'000U)); + auto const after_schedule = wakes; + auto const refused = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "w2", schedule_body(0U)); + auto const after_refusal = wakes; + auto const restart = post_action(fixture, true, delay_id_of(scheduled), "restart", fixture.alice_token); + + THEN("one wake for the schedule, none for the refusal and none for the restart") + { + CHECK(after_schedule == 1); + CHECK(refused.status == 400U); + CHECK(after_refusal == 1); + CHECK(restart.status == 200U); + CHECK(wakes == 1); + } + } + fixture.runtime().homeserver.delayed_events_wake = nullptr; + } +} + +SCENARIO("Ending the scheduling device leaves the delayed event scheduled, and another device manages it", + "[homeserver][delayed-events][routes][device]") +{ + GIVEN("two events scheduled by device D1 of alice, and a second device D3") + { + auto fixture = Fixture{}; + auto const restarted = schedule_message(fixture, "a", 60'000U); + auto const cancelled = schedule_message(fixture, "b", 60'000U); + auto const other_token = log_in(fixture.runtime(), "alice", "D3"); + auto const delete_device = GENERATE(false, true); + auto const before = fixture.event_count(); + + WHEN("D1 is logged out or deleted") + { + if (delete_device) + { + REQUIRE(request(fixture.runtime(), "DELETE", "/_matrix/client/v3/devices/D1", other_token, + R"({"auth":{"type":"m.login.password","password":")" + std::string{password} + R"("}})") + .status == 200U); + } + else + { + REQUIRE( + request(fixture.runtime(), "POST", "/_matrix/client/v3/logout", fixture.alice_token, "{}").status == + 200U); + } + fixture.set_time(t0 + 10'000U); + auto const restart = post_action(fixture, true, restarted, "restart", other_token); + auto const cancel = post_action(fixture, true, cancelled, "cancel", other_token); + + THEN("both rows were still scheduled, D3 restarts one and cancels the other, and the restarted one is sent " + "when due") + { + CHECK(restart.status == 200U); + CHECK(cancel.status == 200U); + CHECK(status_of(fixture, alice, cancelled) == db::DelayedEventStatus::cancelled); + fixture.run_job(t0 + 69'999U); + CHECK(status_of(fixture, alice, restarted) == db::DelayedEventStatus::scheduled); + fixture.set_time(t0 + 70'000U); + fixture.run_job(t0 + 70'000U); + CHECK(status_of(fixture, alice, restarted) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 1U); + } + } + } +} + +SCENARIO( + "Content of exactly 65,536 bytes is accepted and 65,537 is refused 413 on the dedicated route and the query form", + "[homeserver][delayed-events][routes][size]") +{ + GIVEN("alice in a room and content objects of 65,536 and 65,537 serialised bytes") + { + auto fixture = Fixture{}; + auto const exact = R"({"a":")" + std::string(65'528U, 'x') + R"("})"; + auto const over = R"({"a":")" + std::string(65'529U, 'x') + R"("})"; + REQUIRE(exact.size() == 65'536U); + REQUIRE(over.size() == 65'537U); + + WHEN("each is scheduled on the dedicated route and on the query form") + { + auto const dedicated_ok = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "s1", schedule_body(5'000U, exact)); + auto const dedicated_over = + put_dedicated(fixture, true, fixture.alice_token, "m.test.delayed", "s2", schedule_body(5'000U, over)); + auto const query_ok = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "q1", + std::string{delay_param} + "=5000", exact); + auto const query_over = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "q2", + std::string{delay_param} + "=5000", over); + + THEN("65,536 is scheduled and 65,537 is 413 M_TOO_LARGE, on both") + { + CHECK(delay_id_of(dedicated_ok).size() > 0U); + CHECK(delay_id_of(query_ok).size() > 0U); + CHECK(dedicated_over.status == 413U); + CHECK(errcode_of(dedicated_over) == "M_TOO_LARGE"); + CHECK(query_over.status == 413U); + CHECK(errcode_of(query_over) == "M_TOO_LARGE"); + CHECK(fixture.table().scheduled_count(alice) == 2U); + } + } + } +} + +SCENARIO("An application service scheduling for a user is recorded as such, and a user's own request is not", + "[homeserver][delayed-events][routes][appservice]") +{ + GIVEN("an application service whose namespace holds alice") + { + auto bridge = TestAppservice{.id = "bridge", .users_regex = "@.*:example.org"}; + auto fixture = Fixture{appservice_test_config({bridge})}; + auto const token = bridge.as_token(); + auto const as_alice = std::string{"user_id=@alice:example.org"}; + + WHEN("it schedules as alice, and alice schedules directly") + { + auto const by_service = request(fixture.runtime(), "PUT", + "/_matrix/client/unstable/org.matrix.msc4140/rooms/" + fixture.room_id + + "/delayed_event/m.test.delayed/as1?" + as_alice, + token, schedule_body(5'000U)); + auto const by_user = + put_dedicated(fixture, false, fixture.alice_token, "m.test.delayed", "u1", schedule_body(5'000U)); + + THEN("the stored row of the first has via_appservice and the second does not") + { + auto const* service_row = fixture.table().find_owned(alice, delay_id_of(by_service)); + auto const* user_row = fixture.table().find_owned(alice, delay_id_of(by_user)); + REQUIRE(service_row != nullptr); + REQUIRE(user_row != nullptr); + CHECK(service_row->via_appservice); + CHECK_FALSE(user_row->via_appservice); + } + } + + WHEN("the ts parameter is sent with the delay parameter, by the service and by alice") + { + auto const by_service = request(fixture.runtime(), "PUT", + fixture.room_path() + "/send/m.test.delayed/ts1?" + as_alice + "&ts=1&" + + std::string{delay_param} + "=5000", + token, std::string{message_content}); + auto const by_user = put_send_delayed(fixture, fixture.alice_token, "m.test.delayed", "ts2", + "ts=1&" + std::string{delay_param} + "=5000"); + auto const service_id = delay_id_of(by_service); + auto const user_id = delay_id_of(by_user); + fixture.set_time(t0 + 5'000U); + fixture.run_job(t0 + 5'000U); + + THEN("both are scheduled, ts is ignored, and each event's origin_server_ts is the send time") + { + for (auto const& id : {service_id, user_id}) + { + auto const* row = fixture.table().find_owned(alice, id); + REQUIRE(row != nullptr); + REQUIRE(row->status == db::DelayedEventStatus::sent); + auto const event = request(fixture.runtime(), "GET", + fixture.room_path() + "/event/" + row->event_id, fixture.alice_token); + REQUIRE(event.status == 200U); + auto const event_body = parse_object(event.body); + auto const* stamp = int_member(event_body, "origin_server_ts"); + REQUIRE(stamp != nullptr); + CHECK(*stamp == static_cast(t0 + 5'000U)); + } + } + } + } +} diff --git a/tests/integration/test_postgresql_store_behaviour_flow.cpp b/tests/integration/test_postgresql_store_behaviour_flow.cpp index ab9bebc1a..615c5cb4d 100644 --- a/tests/integration/test_postgresql_store_behaviour_flow.cpp +++ b/tests/integration/test_postgresql_store_behaviour_flow.cpp @@ -7,6 +7,7 @@ // MEROVINGIAN_TEST_POSTGRESQL_URI is not set; a skip is not a pass. #include +#include #include #include #include @@ -14,9 +15,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -1789,3 +1792,98 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every } } } + +SCENARIO("PostgreSQL hydration skips delayed event rows that are not read exactly and counts them", + "[database][postgresql][integration][pg-behaviour][delayed-events]") +{ + GIVEN("a valid scheduled row and rows with a damaged number or boolean, written with bound parameters") + { + auto const uri = postgresql_uri(); + if (uri.empty()) + { + SUCCEED("skipped: MEROVINGIAN_TEST_POSTGRESQL_URI is not set"); + return; + } + auto const suffix = unique_suffix(); + auto const user = "@strict-" + suffix + ":example.org"; + auto const baseline = reopen(uri).store.delayed_event_rows_skipped_at_load; + + struct Damaged final + { + std::string_view name; + std::size_t column; + std::string_view value; + }; + auto const cases = std::vector{ + {"not a number", 10U, "abc" }, + {"empty", 10U, "" }, + {"negative", 10U, "-5" }, + {"zero", 10U, "0" }, + {"over the maximum", 10U, "2678400001" }, + {"21 digits", 10U, "123456789012345678901"}, + {"since not a number", 11U, "abc" }, + {"a boolean holding yes", 5U, "yes" }, + }; + auto const insert_raw = [&](std::string const& delay_id, std::size_t damaged_column, + std::string_view damaged_value) { + auto columns = std::vector{delay_id, + user, + "DEVICE", + "!strict-" + suffix + ":example.org", + "m.room.message", + "false", + "", + "{}", + "", + "false", + "60000", + "1000", + "scheduled", + "0", + "", + ""}; + if (damaged_column < columns.size()) + { + columns[damaged_column] = std::string{damaged_value}; + } + auto statement = db::PreparedStatement{ + "insert_raw_delayed_event", + "INSERT INTO delayed_events (delay_id, user_id, device_id, room_id, event_type, is_state, state_key, " + "content_json, txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, " + "event_id, error_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, " + "$16)", + {}}; + for (auto const& column : columns) + { + statement.parameters.push_back(db::BoundValue{column, false}); + } + auto opened = reopen(uri); + return db::commit_persistent_transaction(opened.store, {statement}); + }; + REQUIRE(insert_raw("d-strict-good-" + suffix, 99U, {})); + for (auto const& damaged : cases) + { + REQUIRE(insert_raw("d-strict-" + std::string{damaged.name} + "-" + suffix, damaged.column, damaged.value)); + } + + WHEN("the store is opened") + { + auto reopened = reopen(uri); + + THEN("only the valid row is loaded and each damaged row is counted as skipped") + { + CHECK(reopened.store.delayed_events.find("d-strict-good-" + suffix) != nullptr); + CHECK(reopened.store.delayed_events.scheduled_count(user) == 1U); + CHECK(reopened.store.delayed_event_rows_skipped_at_load == baseline + cases.size()); + } + + AND_THEN("deleting the user's rows removes the skipped rows from the database as well") + { + REQUIRE(db::delete_delayed_events_of_user(reopened.store, user)); + auto again = reopen(uri); + CHECK(again.store.delayed_event_rows_skipped_at_load == baseline); + CHECK(again.store.delayed_events.scheduled_count(user) == 0U); + } + } + } +} diff --git a/tests/unit/test_delayed_event_store.cpp b/tests/unit/test_delayed_event_store.cpp index d5dca9efd..415458c5b 100644 --- a/tests/unit/test_delayed_event_store.cpp +++ b/tests/unit/test_delayed_event_store.cpp @@ -5,7 +5,10 @@ // after a commit (and the one exception), the per-user eviction in the finalising transaction, the age prune, bound // parameters only, and a round trip through a SQLite file. #include "../support/TempDirectory.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/database/BoundValue.hpp" #include "merovingian/database/DelayedEventOutcomeResult.hpp" +#include "merovingian/database/DelayedEventRowText.hpp" #include "merovingian/database/DelayedEventStatus.hpp" #include "merovingian/database/DelayedEventStore.hpp" #include "merovingian/database/DelayedEventTable.hpp" @@ -15,9 +18,11 @@ #include "merovingian/database/PreparedStatement.hpp" #include +#include #include #include +#include #include #include #include @@ -109,8 +114,67 @@ struct SqliteClose final return text; } +using RawRow = std::array; + +// The 16 columns of a valid scheduled row, as the text the store writes, for a test to change one of. +[[nodiscard]] auto raw_row(std::string delay_id) -> RawRow +{ + return {std::move(delay_id), + std::string{alice}, + "DEVICE", + "!room:example.org", + "m.room.message", + "false", + "", + "{}", + "", + "false", + "60000", + "1000", + "scheduled", + "0", + "", + ""}; +} + +// Writes `values` to the store's database as one row of `delayed_events`, without touching memory (a bound-parameter +// insert, so any text goes in), the way a damaged or foreign file would hold it. +[[nodiscard]] auto insert_raw_row(db::PersistentStore& store, RawRow const& values) -> bool +{ + auto statement = db::PreparedStatement{ + "insert_raw_delayed_event", + "INSERT INTO delayed_events (delay_id, user_id, device_id, room_id, event_type, is_state, state_key, " + "content_json, txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " + "error_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)", + {}}; + for (auto const& value : values) + { + statement.parameters.push_back(db::BoundValue{value, false}); + } + return db::commit_persistent_transaction(store, {statement}); +} + +struct RawRowCase final +{ + std::string_view name{}; + std::size_t column{0U}; + std::string_view value{}; +}; + } // namespace +SCENARIO("The delayed-event delay maximum is one number in the config and the database module", + "[database][delayed-events][delayed-event-store]") +{ + GIVEN("the config module's longest delay and the store's") + { + THEN("they are equal, or a configured delay could be accepted by the config and refused by the store") + { + REQUIRE(merovingian::config::max_delayed_event_delay_ms == db::max_delayed_event_delay_ms); + } + } +} + SCENARIO("A delayed event is stored once, within its caps, and refused outside them", "[database][delayed-events][delayed-event-store]") { @@ -705,15 +769,16 @@ SCENARIO("All of a user's delayed events are deleted in one statement", auto const transactions = db::committed_transaction_count(store) - before; auto const none = db::delete_delayed_events_of_user(store, "@nobody:example.org"); - THEN("one DELETE by user removes them all, the other user keeps theirs, and no write is made for none") + THEN("one DELETE by user removes them all, the other user keeps theirs, and one for none is still made") { REQUIRE(deleted); REQUIRE(transactions == 1U); - REQUIRE(store.captured_statements.size() == 1U); REQUIRE(store.captured_statements[0].sql == "DELETE FROM delayed_events WHERE user_id = $1"); REQUIRE(store.captured_statements[0].parameters[0].value == std::string{alice}); REQUIRE(none); - REQUIRE(db::committed_transaction_count(store) - before == 1U); + REQUIRE(db::committed_transaction_count(store) - before == 2U); + REQUIRE(store.captured_statements.size() == 2U); + REQUIRE(store.captured_statements[1].parameters[0].value == "@nobody:example.org"); REQUIRE(store.delayed_events.size() == 1U); REQUIRE(store.delayed_events.find("b-live") != nullptr); } @@ -721,6 +786,35 @@ SCENARIO("All of a user's delayed events are deleted in one statement", } } +SCENARIO("Deactivation deletes a delayed event that exists only in the database", + "[database][delayed-events][delayed-event-store][sqlite]") +{ + GIVEN("a SQLite store with a row in the file that memory does not hold, for a user with no row in memory") + { + auto const path = merovingian::tests::temporary_directory() / + ("merovingian-delayed-events-only-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".sqlite3"); + auto opened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(opened.ok); + REQUIRE(insert_raw_row(opened.store, raw_row("only-in-file"))); + REQUIRE(opened.store.delayed_events.size() == 0U); + REQUIRE(sqlite_first_text(path, "SELECT count(*) FROM delayed_events") == "1"); + + WHEN("the user's delayed events are deleted") + { + auto const deleted = db::delete_delayed_events_of_user(opened.store, alice); + + THEN("the row is gone from the file") + { + REQUIRE(deleted); + REQUIRE(sqlite_first_text(path, "SELECT count(*) FROM delayed_events") == "0"); + } + } + opened = {}; + std::filesystem::remove(path); + } +} + SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up to the row limit", "[database][delayed-events][delayed-event-store]") { @@ -934,3 +1028,96 @@ SCENARIO("Delayed events survive a reopen of a SQLite store with every column an std::filesystem::remove(path); } } + +SCENARIO("Hydration reads a delayed event row strictly and counts the rows it skips", + "[database][delayed-events][delayed-event-store][sqlite]") +{ + GIVEN("a SQLite file holding a valid scheduled row, a sent row and one row that is not read exactly") + { + auto const bad = GENERATE(RawRowCase{"a delay_ms that is not a number", 10U, "abc"}, + RawRowCase{"an empty delay_ms", 10U, ""}, + RawRowCase{"a negative delay_ms", 10U, "-5"}, RawRowCase{"a delay_ms of 0", 10U, "0"}, + RawRowCase{"a delay_ms over the maximum", 10U, "2678400001"}, + RawRowCase{"a delay_ms of 21 digits", 10U, "123456789012345678901"}, + RawRowCase{"a delayed_since_ms that is not a number", 11U, "abc"}, + RawRowCase{"a boolean column holding yes", 5U, "yes"}, + RawRowCase{"a finalised row with no finalised time", 13U, "0"}); + INFO(bad.name); + auto const path = merovingian::tests::temporary_directory() / + ("merovingian-delayed-events-strict-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".sqlite3"); + { + auto opened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(opened.ok); + REQUIRE(db::store_delayed_event(opened.store, row("m-good"))); + REQUIRE(db::store_delayed_event(opened.store, row("a-sent"))); + REQUIRE(db::record_delayed_event_outcome(opened.store, "a-sent", DelayedEventStatus::sent, 2000U, + "$sent:example.org", {}, + 10U) == DelayedEventOutcomeResult::recorded); + auto damaged = raw_row("z-bad"); + damaged[bad.column] = std::string{bad.value}; + if (bad.column == 13U) + { + damaged[12] = "cancelled"; + } + REQUIRE(insert_raw_row(opened.store, damaged)); + } + + WHEN("the store is opened") + { + auto reopened = db::open_sqlite_persistent_store(path.string()); + + THEN("the row is not loaded, stays in the file and is counted, and the valid rows load") + { + REQUIRE(reopened.ok); + REQUIRE(reopened.store.delayed_events.size() == 2U); + REQUIRE(reopened.store.delayed_events.find("z-bad") == nullptr); + REQUIRE(reopened.store.delayed_events.find("m-good") != nullptr); + REQUIRE(reopened.store.delayed_events.find("a-sent") != nullptr); + REQUIRE(reopened.store.delayed_event_rows_skipped_at_load == 1U); + REQUIRE(sqlite_first_text(path, "SELECT count(*) FROM delayed_events WHERE delay_id = 'z-bad'") == "1"); + } + } + std::filesystem::remove(path); + } +} + +SCENARIO("Hydration skips a second sent row that claims an event ID another row holds", + "[database][delayed-events][delayed-event-store][sqlite]") +{ + GIVEN("a SQLite file holding a sent row and a second sent row with the same event ID") + { + auto const path = merovingian::tests::temporary_directory() / + ("merovingian-delayed-events-dup-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".sqlite3"); + { + auto opened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(opened.ok); + REQUIRE(db::store_delayed_event(opened.store, row("a-sent"))); + REQUIRE(db::record_delayed_event_outcome(opened.store, "a-sent", DelayedEventStatus::sent, 2000U, + "$sent:example.org", {}, + 10U) == DelayedEventOutcomeResult::recorded); + auto duplicate = raw_row("z-sent"); + duplicate[12] = "sent"; + duplicate[13] = "2001"; + duplicate[14] = "$sent:example.org"; + REQUIRE(insert_raw_row(opened.store, duplicate)); + } + + WHEN("the store is opened") + { + auto reopened = db::open_sqlite_persistent_store(path.string()); + + THEN("the first row keeps the event ID, the second is skipped and counted, and stays in the file") + { + REQUIRE(reopened.ok); + REQUIRE(reopened.store.delayed_events.size() == 1U); + REQUIRE(reopened.store.delayed_events.find_sent_by_event_id("$sent:example.org") == + reopened.store.delayed_events.find("a-sent")); + REQUIRE(reopened.store.delayed_event_rows_skipped_at_load == 1U); + REQUIRE(sqlite_first_text(path, "SELECT count(*) FROM delayed_events") == "2"); + } + } + std::filesystem::remove(path); + } +} diff --git a/tests/unit/test_delayed_event_table.cpp b/tests/unit/test_delayed_event_table.cpp index b413c131b..1231589c0 100644 --- a/tests/unit/test_delayed_event_table.cpp +++ b/tests/unit/test_delayed_event_table.cpp @@ -469,6 +469,37 @@ SCENARIO("A transaction ID is looked up by user, device, room, event type and ID } } +SCENARIO("A second sent row claiming an event ID another row holds is refused", "[database][delayed-events]") +{ + GIVEN("a sent row") + { + auto table = DelayedEventTable{}; + auto first = scheduled("d-first", "@a:example.org", 0U, 1000U); + first.status = DelayedEventStatus::sent; + first.finalised_at_ms = 5U; + first.event_id = "$event:example.org"; + REQUIRE(table.insert(first)); + + WHEN("another sent row with the same event ID is inserted") + { + auto second = scheduled("d-second", "@a:example.org", 0U, 1000U); + second.status = DelayedEventStatus::sent; + second.finalised_at_ms = 6U; + second.event_id = "$event:example.org"; + auto const accepted = table.insert(second); + + THEN("it is refused, and the first row still owns the event ID and erasing it frees the ID") + { + REQUIRE_FALSE(accepted); + REQUIRE(table.find("d-second") == nullptr); + REQUIRE(table.find_sent_by_event_id("$event:example.org") == table.find("d-first")); + REQUIRE(table.erase("d-first")); + REQUIRE(table.find_sent_by_event_id("$event:example.org") == nullptr); + } + } + } +} + SCENARIO("Erasing a row, or all of a user's rows, removes it from every index", "[database][delayed-events]") { GIVEN("two users with scheduled, sent and cancelled rows") From a88e75d774064d2ce26fef9be2d2f6bd38b0ba21 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 03:42:22 +0000 Subject: [PATCH 20/29] Say which logger writes the skipped-rows start-up warning Co-Authored-By: Claude Fable 5.2 --- docs/log-filtering.md | 2 +- docs/observability-audit.md | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/log-filtering.md b/docs/log-filtering.md index a94c443d2..15d4a6345 100644 --- a/docs/log-filtering.md +++ b/docs/log-filtering.md @@ -133,7 +133,7 @@ used as the first argument to `observability::log_diagnostic` and `log_diagnosti | `stream_token_epoch` | The database's sync token epoch: created at first start, replaced by `--after-restore`, or refused | | `tls` | TLS context set-up and handshake accept or reject | | `oauth` | OAuth 2.0 client registration, eviction at the cap, and stored metadata that no longer validates | -| `delayed_events` | Delayed events: scheduled, restarted, cancelled, sent, failed, a failed outcome write, a failed prune, and rows skipped at load; never event content or a state key | +| `delayed_events` | Delayed events: scheduled, restarted, cancelled, sent, failed, a failed outcome write and a failed prune; never event content or a state key. The count of rows skipped at load is a start-up warning from the process entry point, not this module | Unrecognised module names are accepted and the level is recorded — there is no error. Restart the server to apply. diff --git a/docs/observability-audit.md b/docs/observability-audit.md index 716e861b8..738c07773 100644 --- a/docs/observability-audit.md +++ b/docs/observability-audit.md @@ -634,11 +634,14 @@ The delayed-events handlers and the `delayed-events` job log under the `delayed_ `delay_id`, `user_id`, `room_id` and `event_type`, never the content and never the state key (a call membership's state key holds a device ID): `delayed_event.scheduled` (info, with `delay_ms`), `delayed_event.restarted` (info), `delayed_event.cancelled` (info), `delayed_event.sent` (info, with `event_id`), `delayed_event.failed` (warning, with -`status` and `errcode`; error, with `status` and the exception's `type`, when the job's send threw), `delayed_event.prune_failed` (warning, no row fields: the age prune's commit failed and the rows stay for the next run), -`delayed_events.rows_skipped_at_load` (warning, once per start-up when rows were skipped, with the count only: the -loader skips a row it cannot read and never repairs it, ADR-0824) and `delayed_event.finalise_commit_failed` (error, -with `outcome`, `commit_failed`, `refused` or `exception`: the event was sent but its row could not be recorded, so it -is sent again after a restart). +`status` and `errcode`; error, with `status` and the exception's `type`, when the job's send threw), +`delayed_event.prune_failed` (warning, no row fields: the age prune's commit failed and the rows stay for the next +run) and `delayed_event.finalise_commit_failed` (error, with `outcome`, `commit_failed`, `refused` or `exception`: the +event was sent but its row could not be recorded, so it is sent again after a restart). + +The process entry point, not the `delayed_events` module, writes one start-up warning beginning +`delayed_events.rows_skipped_at_load:` when the loader skipped rows, with the count only: the loader skips a row it +cannot read and never repairs it (ADR-0824). ## Registration token file diagnostics From 236bc3496e113027bf826a7dac188d639598aad6 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 10:30:03 +0000 Subject: [PATCH 21/29] Count the delayed-events job's wait from when its run ends and skip delayed_events for the worker profile on SQLite Co-Authored-By: Claude Fable 5.2 --- docs/database-persistence.md | 4 +- .../merovingian/homeserver/DelayedEvents.hpp | 14 +++ src/database/AGENTS.md | 4 +- src/database/SqliteStore.cpp | 42 ++++---- src/homeserver/AGENTS.md | 6 +- src/homeserver/DelayedEvents.cpp | 17 ++++ src/main.cpp | 5 +- .../integration/test_delayed_events_flow.cpp | 97 +++++++++++++++++++ tests/unit/test_delayed_event_store.cpp | 50 ++++++++++ 9 files changed, 212 insertions(+), 27 deletions(-) diff --git a/docs/database-persistence.md b/docs/database-persistence.md index ad6aeeaeb..0194eaaf3 100644 --- a/docs/database-persistence.md +++ b/docs/database-persistence.md @@ -92,8 +92,8 @@ value. Test fixtures that do not exercise persistence select it through `event_type`, `state_key`, `txn_id` and `event_id`, and `delay_ms` from 1 to 2,678,400,000 (31 days). A row over a cap is refused, never truncated. The store does not enforce the per-user cap on scheduled rows: the caller does, because it must compute `Retry-After` from the user's rows. - - In memory: `PersistentStore::delayed_events` (`DelayedEventTable`), hydrated on both backends, with no lock of - its own (`runtime.mutex` serialises it). It indexes the rows by delay ID, by user (the scheduled rows by due + - In memory: `PersistentStore::delayed_events` (`DelayedEventTable`), hydrated on both backends (neither loads it for the federation worker + profile), with no lock of its own (`runtime.mutex` serialises it). It indexes the rows by delay ID, by user (the scheduled rows by due time, the finalised rows by finalised time), every scheduled row not being sent by due time (`delayed_since_ms + delay_ms`, then delay ID, so the earliest due is O(1)), by (user, device, room, event type, transaction ID) for idempotency, by event ID for sent rows, and every finalised row by finalised time for the age prune. A memory-only diff --git a/include/merovingian/homeserver/DelayedEvents.hpp b/include/merovingian/homeserver/DelayedEvents.hpp index 568c9833f..8ef97104d 100644 --- a/include/merovingian/homeserver/DelayedEvents.hpp +++ b/include/merovingian/homeserver/DelayedEvents.hpp @@ -73,4 +73,18 @@ auto wake_delayed_events(HomeserverRuntime& runtime) -> void; [[nodiscard]] auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, StopFlag const& stop) -> std::chrono::milliseconds; +// The wait left once `elapsed_ms` of `wait` has already passed: `wait` less `elapsed_ms`, never under the job's pace +// (5 ms) and never over its longest wait (60 s). An `elapsed_ms` of 0 returns `wait` unchanged when `wait` is within +// those bounds; an `elapsed_ms` of `wait` or more returns the pace. Pure; cannot throw. +[[nodiscard]] auto delayed_events_wait_after(std::chrono::milliseconds wait, std::uint64_t elapsed_ms) noexcept + -> std::chrono::milliseconds; + +// The "delayed-events" job's whole body (main.cpp calls it; tests too): reads `delayed_event_now_ms`, runs +// `send_due_delayed_events` with it, reads the clock again and returns that function's wait less the time the run took +// (`delayed_events_wait_after`; none if the clock went backwards). `PeriodicJob` counts its sleep from the body's +// return, so the time the body spent, mostly waiting for the runtime lock, must come off the wait or a deadline is +// overslept by that much. Takes `runtime.homeserver.mutex` itself, as `send_due_delayed_events` does. +[[nodiscard]] auto run_delayed_events_job(ClientServerRuntime& runtime, StopFlag const& stop) + -> std::chrono::milliseconds; + } // namespace merovingian::homeserver diff --git a/src/database/AGENTS.md b/src/database/AGENTS.md index d4777ef14..728be821d 100644 --- a/src/database/AGENTS.md +++ b/src/database/AGENTS.md @@ -235,7 +235,9 @@ deletes a user's account removes the user's rows** with `delete_delayed_events_o `DELETE` (a row hydration skipped is in the database only) and then erases memory; logout and device deletion do not (a call's delayed leave must outlive its device). Hydration counts every row it skips in `PersistentStore::delayed_event_rows_skipped_at_load` (a duplicate delay ID, transaction key or sent event ID counts), -and `main.cpp` logs it once as `delayed_events.rows_skipped_at_load`; never log a skipped row's columns. +and `main.cpp` logs it once as `delayed_events.rows_skipped_at_load`; never log a skipped row's columns. Both loaders +read the table only when `table_load_profile_includes("delayed_events", profile)` (the worker profile skips it, counting +no row), so one backend never loads what the other withholds. `store_delayed_event` does not enforce the per-user cap on scheduled rows: the caller does, because it computes `Retry-After`. Never log a row's `content_json`. The federation worker never reads the table. diff --git a/src/database/SqliteStore.cpp b/src/database/SqliteStore.cpp index 94acafbcf..f1465d68a 100644 --- a/src/database/SqliteStore.cpp +++ b/src/database/SqliteStore.cpp @@ -759,7 +759,7 @@ namespace // Events go to `hydrated_events` rather than `store.events`, for the caller to index in one pass // (ADR-0215): the SELECT has no ORDER BY, so adding them one at a time could shift the // stream-order index for every row. - auto load_persistent_rows(sqlite3& connection, PersistentStore& store, + auto load_persistent_rows(sqlite3& connection, PersistentStore& store, TableLoadProfile profile, std::vector& hydrated_events) -> bool { // Tables load in dependency order and the chain short-circuits: the first failed SELECT @@ -1186,23 +1186,27 @@ namespace // Migration 003 (ADR-0824): delayed events. A row whose status text is not one of the four, or that // duplicates a delay ID, a transaction key or a sent event ID, or whose numbers // or booleans are not exactly what the store writes, is skipped and counted, never repaired. - load_rows(connection, - "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, content_json, " - "txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " - "error_json FROM delayed_events ORDER BY delay_id", - [&store](sqlite3_stmt& row) { - auto columns = std::array{}; - for (auto index = std::size_t{0U}; index < columns.size(); ++index) - { - columns[index] = column_text(row, static_cast(index)); - } - // Strict: a row that is not read exactly is skipped, left in the file and counted. - auto parsed = parse_delayed_event_row(columns); - if (!parsed.has_value() || !store.delayed_events.insert(std::move(*parsed))) - { - ++store.delayed_event_rows_skipped_at_load; - } - }) && + // Skipped for a profile that does not include the table, as PostgreSQL does: the federation worker never + // reads a scheduled event's content, and a skipped load counts no row. + (!table_load_profile_includes("delayed_events", profile) || + load_rows( + connection, + "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, content_json, " + "txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " + "error_json FROM delayed_events ORDER BY delay_id", + [&store](sqlite3_stmt& row) { + auto columns = std::array{}; + for (auto index = std::size_t{0U}; index < columns.size(); ++index) + { + columns[index] = column_text(row, static_cast(index)); + } + // Strict: a row that is not read exactly is skipped, left in the file and counted. + auto parsed = parse_delayed_event_row(columns); + if (!parsed.has_value() || !store.delayed_events.insert(std::move(*parsed))) + { + ++store.delayed_event_rows_skipped_at_load; + } + })) && // Application service delivery state: transaction cursors, the outbox and the directory mapping. load_rows(connection, "SELECT appservice_id, next_txn_id, delivered_stream_ordering, pending_txn_id, " @@ -1753,7 +1757,7 @@ auto open_sqlite_persistent_store(std::string const& path, TableLoadProfile prof }); } auto hydrated_events = std::vector{}; - if (!load_persistent_rows(sqlite, store, hydrated_events)) + if (!load_persistent_rows(sqlite, store, profile, hydrated_events)) { log_diagnostic("store.rejected", { diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index ad187e3f0..eb70aa7c7 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -27,7 +27,7 @@ This is the largest and most complex module — read this file carefully before | `AppserviceDelivery.cpp` | Application Service API outbound queue (ADR-0163): `enqueue_appservice_delivery` (request path, no network) and `drain_appservice_queues` (the one drainer thread, run by a `PeriodicJob` from `main.cpp`) | | `ClientTxnPrune.cpp` | `prune_client_transactions`: the sweep the `client-txn-prune` `PeriodicJob` runs every hour, deleting to-device `client_txn_ids` records older than 7 days (F-54, ADR-0381) | | `RetentionSweep.cpp` | `run_retention_sweep`: the sweep the `retention-sweep` `PeriodicJob` runs every hour. Deletes sessions and tokens that can no longer authenticate and `audit_log` rows past `security.audit_log_retention_days`, OAuth 2.0 session rows whose device holds no live access or refresh token (judged again in the hold that deletes them, ADR-0808), and OAuth 2.0 clients with no session unused for 30 days (`oauth_client_retention`, least recently used first; never a client with a session, nor one `oauth_client_has_pending_authorisation` names: a live pending authorisation or an unredeemed code, answered in O(1) from the stores; ADR-0777, ADR-0782), in transactions of at most 500 rows (`retention_batch_rows`), at most 20 batches per kind per sweep. `runtime.mutex` is taken for one batch at a time, never across the whole sweep (ADR-0488, ADR-0489) | -| `DelayedEvents.cpp` | Delayed events (MSC4140, ADR-0823 to ADR-0826): `schedule_delayed_event`, `manage_delayed_event` (send, cancel, restart), `delayed_event_lookup`, the clock `delayed_event_now_ms` (and its seam `test_delayed_event_clock_ms`), `wake_delayed_events`, and `send_due_delayed_events`, the body of the `delayed-events` `PeriodicJob` in `main.cpp`. The caller holds `runtime.mutex` for all but the job body, which takes it for one send a run | +| `DelayedEvents.cpp` | Delayed events (MSC4140, ADR-0823 to ADR-0826): `schedule_delayed_event`, `manage_delayed_event` (send, cancel, restart), `delayed_event_lookup`, the clock `delayed_event_now_ms` (and its seam `test_delayed_event_clock_ms`), `wake_delayed_events`, `send_due_delayed_events`, and `run_delayed_events_job`, the body of the `delayed-events` `PeriodicJob` in `main.cpp`. The caller holds `runtime.mutex` for all but the job body, which takes it for one send a run | | `ClientServerRuntime.cpp` (delayed events) | The MSC4140 routes (ADR-0823): S1 and S2 `PUT .../delayed_event/...`, the `org.matrix.msc4140.delay` branch of `PUT /send` and `/state`, M1 to M3 `POST .../delayed_events/{delayId}[/{action}]` and G1 and G2 `GET .../delayed_events/{delayId}`, with the `/versions` flags and `/capabilities` entries. See "Delayed events" below | | `ClientServerRuntimeInternal.hpp` | Call-throughs from `ClientServerRuntime.cpp`'s anonymous namespace that `DelayedEvents.cpp` shares with the `/send` and `/state` routes: `rate_limit_route_template` (`normalized_target`), `route_event_body` (`event_body_from_content`), `event_txn_part_is_too_long`, `errcode_for_refusal_status`, and the two reasons the `/send` route gives for a too-long type or transaction ID. Not public API | | `OAuthClientRegistration.cpp` | `POST /_merovingian/oauth2/register` (`register_oauth_client`), the client ID (`oauth_client_id_of`, ADR-0779), the cap and its eviction (ADR-0780), and `find_registered_oauth_client` | @@ -1216,7 +1216,9 @@ next layer) only parse a request and call it. Invariants a change must keep: (`DelayedEventTable::mark_being_sent`, memory only) before `send_event_as_resolved_user`, and finalising clears the mark. A marked row is offered by nobody: restart, cancel and send answer 409 "being sent". The job sends AT MOST ONE event a run, in one hold, and returns `delayed_events_pace` (5 ms) when another is already due, so the lock is free between - sends as in the profile propagation job (ADR-0353); it never returns 0, and it checks its `StopFlag` before the send. + sends as in the profile propagation job (ADR-0353); it never returns 0, and it checks its `StopFlag` before the send. `PeriodicJob` counts its sleep from the body's + return, so `main.cpp` runs `run_delayed_events_job`, which takes the time the run took (mostly waiting for the lock) + off the wait with `delayed_events_wait_after`; never put `send_due_delayed_events` straight in the job. A record the store refuses, or that throws (`delayed_event.finalise_commit_failed`, `outcome` `exception`, the type only), leaves the row marked, so it is never sent again by that process (the next start sends it once more); never clear the mark to retry. diff --git a/src/homeserver/DelayedEvents.cpp b/src/homeserver/DelayedEvents.cpp index fb700b7e7..68eaa5796 100644 --- a/src/homeserver/DelayedEvents.cpp +++ b/src/homeserver/DelayedEvents.cpp @@ -720,4 +720,21 @@ auto send_due_delayed_events(ClientServerRuntime& runtime, std::uint64_t now_ms, return std::chrono::milliseconds{std::clamp(due_ms - now_ms, delayed_events_pace, max_job_wait_ms)}; } +auto delayed_events_wait_after(std::chrono::milliseconds wait, std::uint64_t elapsed_ms) noexcept + -> std::chrono::milliseconds +{ + // Unsigned arithmetic on a non-negative wait, compared before subtracting, so a huge `elapsed_ms` cannot wrap. + auto const wait_ms = wait.count() > 0 ? static_cast(wait.count()) : std::uint64_t{0U}; + auto const left = elapsed_ms >= wait_ms ? std::uint64_t{0U} : wait_ms - elapsed_ms; + return std::chrono::milliseconds{std::clamp(left, delayed_events_pace, max_job_wait_ms)}; +} + +auto run_delayed_events_job(ClientServerRuntime& runtime, StopFlag const& stop) -> std::chrono::milliseconds +{ + auto const earlier = delayed_event_now_ms(runtime.homeserver); + auto const wait = send_due_delayed_events(runtime, earlier, stop); + auto const later = delayed_event_now_ms(runtime.homeserver); + return delayed_events_wait_after(wait, later > earlier ? later - earlier : std::uint64_t{0U}); +} + } // namespace merovingian::homeserver diff --git a/src/main.cpp b/src/main.cpp index c0a05d30b..fffef2e22 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -935,13 +935,12 @@ struct ListenerBinding final // Sends the delayed events (MSC4140, ADR-0825) that have fallen due: at most one send per run, in its own hold of // the runtime lock, and a run that left another event due waits `delayed_events_pace` (5 ms) with the lock free, // as the profile propagation job does (ADR-0353). It also ages out finalised rows. Otherwise it sleeps until the - // earliest deadline (60 s at most) and is woken when an earlier one is scheduled. + // earliest deadline (60 s at most, less the time the run took) and is woken when an earlier one is scheduled. // It only ever touches `runtime`, which outlives this function. Declared after the pools so it is stopped and // joined before them. auto delayed_events_job = merovingian::homeserver::PeriodicJob{ "delayed-events", [&runtime](merovingian::homeserver::StopFlag const& stop) { - return merovingian::homeserver::send_due_delayed_events( - runtime, merovingian::homeserver::delayed_event_now_ms(runtime.homeserver), stop); + return merovingian::homeserver::run_delayed_events_job(runtime, stop); }}; if (!delayed_events_job.start()) { diff --git a/tests/integration/test_delayed_events_flow.cpp b/tests/integration/test_delayed_events_flow.cpp index 5e39e32c3..17e198020 100644 --- a/tests/integration/test_delayed_events_flow.cpp +++ b/tests/integration/test_delayed_events_flow.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -1605,3 +1606,99 @@ SCENARIO("A send action that throws is answered 500 by the backstop and leaves t fixture.runtime().homeserver.database.persistent_store.test_before_backend_commit = nullptr; } } + +SCENARIO("The job's next wait has the time the run took taken off it, within the pace and the longest wait", + "[homeserver][delayed-events][job][pacing]") +{ + using std::chrono::milliseconds; + + GIVEN("a wait of 10 seconds") + { + auto const wait = milliseconds{10'000}; + + WHEN("no time, less, exactly and more than the wait has passed") + { + auto const none = hs::delayed_events_wait_after(wait, 0U); + auto const less = hs::delayed_events_wait_after(wait, 3'000U); + auto const equal = hs::delayed_events_wait_after(wait, 10'000U); + auto const more = hs::delayed_events_wait_after(wait, 10'001U); + + THEN("the wait shrinks by what passed and never falls under the 5 ms pace") + { + CHECK(none == milliseconds{10'000}); + CHECK(less == milliseconds{7'000}); + CHECK(equal == milliseconds{5}); + CHECK(more == milliseconds{5}); + } + } + + WHEN("all but 2 ms of it has passed, and a huge time has passed") + { + auto const under_pace = hs::delayed_events_wait_after(wait, 9'998U); + auto const huge = hs::delayed_events_wait_after(wait, std::numeric_limits::max()); + + THEN("the result is the pace") + { + CHECK(under_pace == milliseconds{5}); + CHECK(huge == milliseconds{5}); + } + } + } + + GIVEN("a wait at the 60 second cap") + { + WHEN("no time and 1 second have passed") + { + auto const none = hs::delayed_events_wait_after(milliseconds{60'000}, 0U); + auto const one = hs::delayed_events_wait_after(milliseconds{60'000}, 1'000U); + + THEN("the wait is 60 seconds and 59 seconds") + { + CHECK(none == milliseconds{60'000}); + CHECK(one == milliseconds{59'000}); + } + } + } +} + +SCENARIO("The job body returns what the send function returns for the clock's time, and sends a due event", + "[homeserver][delayed-events][job][pacing]") +{ + GIVEN("a runtime with one event due in 30 seconds and the clock moved to its deadline less 10 seconds") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "a", 30'000U)); + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 20'000U; + auto const expected = hs::send_due_delayed_events(fixture.runtime(), t0 + 20'000U, hs::StopFlag{}); + + WHEN("the job body runs") + { + auto const wait = hs::run_delayed_events_job(fixture.runtime(), hs::StopFlag{}); + + THEN("it returns the send function's wait and sends nothing early") + { + CHECK(wait == expected); + CHECK(wait == std::chrono::milliseconds{10'000}); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + } + + GIVEN("a runtime with an event that is due at the clock's time") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "a", 1'000U)); + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 1'000U; + + WHEN("the job body runs") + { + auto const wait = hs::run_delayed_events_job(fixture.runtime(), hs::StopFlag{}); + + THEN("the event is sent and nothing else is scheduled, so the wait is 60 seconds") + { + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(wait == std::chrono::milliseconds{60'000}); + } + } + } +} diff --git a/tests/unit/test_delayed_event_store.cpp b/tests/unit/test_delayed_event_store.cpp index 415458c5b..ed07d9d31 100644 --- a/tests/unit/test_delayed_event_store.cpp +++ b/tests/unit/test_delayed_event_store.cpp @@ -16,6 +16,7 @@ #include "merovingian/database/PersistentStore.hpp" #include "merovingian/database/PersistentStoreOpenResult.hpp" #include "merovingian/database/PreparedStatement.hpp" +#include "merovingian/database/TableLoadProfile.hpp" #include #include @@ -1121,3 +1122,52 @@ SCENARIO("Hydration skips a second sent row that claims an event ID another row std::filesystem::remove(path); } } + +SCENARIO("A store opened with the federation worker profile does not load delayed events and leaves them in the file", + "[database][delayed-events][delayed-event-store][sqlite]") +{ + GIVEN("a SQLite store holding two scheduled events") + { + auto const path = merovingian::tests::temporary_directory() / + ("merovingian-delayed-events-profile-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".sqlite3"); + { + auto opened = db::open_sqlite_persistent_store(path.string()); + REQUIRE(opened.ok); + REQUIRE(db::store_delayed_event(opened.store, row("d-one"))); + REQUIRE(db::store_delayed_event(opened.store, row("d-two", bob))); + } + + WHEN("it is opened with the full profile and with the federation worker profile") + { + auto const full = db::open_sqlite_persistent_store(path.string(), db::TableLoadProfile::full); + auto const worker = + db::open_sqlite_persistent_store(path.string(), db::TableLoadProfile::federation_worker); + + THEN("the full profile loads both, the worker profile loads none and counts no skipped row") + { + REQUIRE(full.ok); + REQUIRE(full.store.delayed_events.size() == 2U); + REQUIRE(worker.ok); + REQUIRE(worker.store.delayed_events.size() == 0U); + REQUIRE(worker.store.delayed_event_rows_skipped_at_load == 0U); + } + } + + WHEN("it is opened with the worker profile and then with the full profile") + { + auto const worker = + db::open_sqlite_persistent_store(path.string(), db::TableLoadProfile::federation_worker); + auto const again = db::open_sqlite_persistent_store(path.string(), db::TableLoadProfile::full); + + THEN("the rows are still in the file") + { + REQUIRE(worker.ok); + REQUIRE(again.ok); + REQUIRE(again.store.delayed_events.size() == 2U); + REQUIRE(sqlite_first_text(path, "SELECT count(*) FROM delayed_events") == "2"); + } + } + std::filesystem::remove(path); + } +} From 58255ddc8ce74b00b39d4863f4109bc74e36c5fd Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 10:30:03 +0000 Subject: [PATCH 22/29] Put the parse in WHEN in the delayed-events config tests Co-Authored-By: Claude Fable 5.2 --- tests/unit/test_config_delayed_events.cpp | 233 ++++++++++++++-------- 1 file changed, 155 insertions(+), 78 deletions(-) diff --git a/tests/unit/test_config_delayed_events.cpp b/tests/unit/test_config_delayed_events.cpp index 331603b5b..36bedbd87 100644 --- a/tests/unit/test_config_delayed_events.cpp +++ b/tests/unit/test_config_delayed_events.cpp @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -70,17 +71,20 @@ SCENARIO("server.delayed_events.* default to on with a one-day delay and 100 eve { GIVEN("a configuration that does not mention the section") { - auto const parsed = merovingian::config::parse_key_value_config(""); - - THEN("delayed events are enabled with the documented limits and nothing is reported") + WHEN("it is parsed") { - auto const& delayed = parsed.config.server().delayed_events; - CHECK(delayed.enabled); - CHECK(delayed.max_delay_ms == std::uint64_t{86'400'000U}); - CHECK(delayed.max_scheduled_per_user == 100U); - CHECK(delayed.max_finalised_per_user == 100U); - CHECK(parsed.findings.empty()); - CHECK(merovingian::config::is_valid(parsed.config)); + auto const parsed = merovingian::config::parse_key_value_config(""); + + THEN("delayed events are enabled with the documented limits and nothing is reported") + { + auto const& delayed = parsed.config.server().delayed_events; + CHECK(delayed.enabled); + CHECK(delayed.max_delay_ms == std::uint64_t{86'400'000U}); + CHECK(delayed.max_scheduled_per_user == 100U); + CHECK(delayed.max_finalised_per_user == 100U); + CHECK(parsed.findings.empty()); + CHECK(merovingian::config::is_valid(parsed.config)); + } } } } @@ -89,28 +93,34 @@ SCENARIO("each server.delayed_events.* key is parsed", "[config][delayed-events] { GIVEN("a configuration that sets every key to a non-default value") { - auto const parsed = merovingian::config::parse_key_value_config(all_non_default); - - THEN("the values are held and the configuration is valid") + WHEN("it is parsed") { - auto const& delayed = parsed.config.server().delayed_events; - CHECK_FALSE(delayed.enabled); - CHECK(delayed.max_delay_ms == std::uint64_t{3'600'000U}); - CHECK(delayed.max_scheduled_per_user == 7U); - CHECK(delayed.max_finalised_per_user == 9U); - CHECK(parsed.findings.empty()); - CHECK(merovingian::config::is_valid(parsed.config)); + auto const parsed = merovingian::config::parse_key_value_config(all_non_default); + + THEN("the values are held and the configuration is valid") + { + auto const& delayed = parsed.config.server().delayed_events; + CHECK_FALSE(delayed.enabled); + CHECK(delayed.max_delay_ms == std::uint64_t{3'600'000U}); + CHECK(delayed.max_scheduled_per_user == 7U); + CHECK(delayed.max_finalised_per_user == 9U); + CHECK(parsed.findings.empty()); + CHECK(merovingian::config::is_valid(parsed.config)); + } } } GIVEN("enabled written as true") { - auto const parsed = parse_key(enabled_key, "true"); - - THEN("it is on") + WHEN("it is parsed") { - CHECK(parsed.config.server().delayed_events.enabled); - CHECK(parsed.findings.empty()); + auto const parsed = parse_key(enabled_key, "true"); + + THEN("it is on") + { + CHECK(parsed.config.server().delayed_events.enabled); + CHECK(parsed.findings.empty()); + } } } } @@ -119,13 +129,23 @@ SCENARIO("server.delayed_events.enabled refuses anything that is not a boolean", { GIVEN("values that are not true or false") { - THEN("each is refused with a finding naming the key") + auto const values = std::vector{"", "yes", "1", "0", "TRUE", "True", "on", "2"}; + + WHEN("each is parsed") { - for (auto const* bad : {"", "yes", "1", "0", "TRUE", "True", "on", "2"}) + auto parsed = std::vector{}; + for (auto const& bad : values) { - INFO(bad); - auto const parsed = parse_key(enabled_key, bad); - REQUIRE(has_finding(parsed.findings, enabled_key)); + parsed.push_back(parse_key(enabled_key, bad)); + } + + THEN("each is refused with a finding naming the key") + { + for (auto index = std::size_t{0U}; index < values.size(); ++index) + { + INFO(values[index]); + REQUIRE(has_finding(parsed[index].findings, enabled_key)); + } } } } @@ -135,31 +155,51 @@ SCENARIO("server.delayed_events.max_delay_ms accepts 1000 to 2678400000 millisec { GIVEN("values at and inside the boundaries") { - THEN("each is accepted and held exactly") + auto const values = + std::vector{1'000U, 1'001U, 3'600'000U, 86'400'000U, 2'678'399'999U, 2'678'400'000U}; + + WHEN("each is parsed") { - for (auto const value : - {std::uint64_t{1'000U}, std::uint64_t{1'001U}, std::uint64_t{3'600'000U}, std::uint64_t{86'400'000U}, - std::uint64_t{2'678'399'999U}, std::uint64_t{2'678'400'000U}}) + auto parsed = std::vector{}; + for (auto const value : values) { - INFO(value); - auto const parsed = parse_key(delay_key, std::to_string(value)); - REQUIRE_FALSE(has_finding(parsed.findings, delay_key)); - REQUIRE(parsed.config.server().delayed_events.max_delay_ms == value); - REQUIRE(merovingian::config::is_valid(parsed.config)); + parsed.push_back(parse_key(delay_key, std::to_string(value))); + } + + THEN("each is accepted and held exactly") + { + for (auto index = std::size_t{0U}; index < values.size(); ++index) + { + INFO(values[index]); + REQUIRE_FALSE(has_finding(parsed[index].findings, delay_key)); + REQUIRE(parsed[index].config.server().delayed_events.max_delay_ms == values[index]); + REQUIRE(merovingian::config::is_valid(parsed[index].config)); + } } } } GIVEN("values outside the range, or not an unsigned integer") { - THEN("each is refused with a finding naming the key") + auto const values = std::vector{ + "0", "1", "999", "2678400001", "4294967296", "99999999999999999999", "-1", "-86400000", "", "abc", + "1.5", "1e6", "+1000", "0x3e8", "1h"}; + + WHEN("each is parsed") { - for (auto const* bad : {"0", "1", "999", "2678400001", "4294967296", "99999999999999999999", "-1", - "-86400000", "", "abc", "1.5", "1e6", "+1000", "0x3e8", "1h"}) + auto parsed = std::vector{}; + for (auto const& bad : values) { - INFO(bad); - auto const parsed = parse_key(delay_key, bad); - REQUIRE(has_finding(parsed.findings, delay_key)); + parsed.push_back(parse_key(delay_key, bad)); + } + + THEN("each is refused with a finding naming the key") + { + for (auto index = std::size_t{0U}; index < values.size(); ++index) + { + INFO(values[index]); + REQUIRE(has_finding(parsed[index].findings, delay_key)); + } } } } @@ -169,38 +209,61 @@ SCENARIO("the per-user delayed event counts accept 1 to 1000", "[config][delayed { GIVEN("values at and inside the boundaries") { - THEN("each is accepted and held exactly for both counts") + auto const values = std::vector{1U, 2U, 100U, 999U, 1000U}; + + WHEN("each is parsed for both counts") { - for (auto const value : {1U, 2U, 100U, 999U, 1000U}) + auto scheduled = std::vector{}; + auto finalised = std::vector{}; + for (auto const value : values) { - INFO(value); - auto const scheduled = parse_key(scheduled_key, std::to_string(value)); - REQUIRE_FALSE(has_finding(scheduled.findings, scheduled_key)); - REQUIRE(scheduled.config.server().delayed_events.max_scheduled_per_user == value); - REQUIRE(merovingian::config::is_valid(scheduled.config)); - - auto const finalised = parse_key(finalised_key, std::to_string(value)); - REQUIRE_FALSE(has_finding(finalised.findings, finalised_key)); - REQUIRE(finalised.config.server().delayed_events.max_finalised_per_user == value); - REQUIRE(merovingian::config::is_valid(finalised.config)); + scheduled.push_back(parse_key(scheduled_key, std::to_string(value))); + finalised.push_back(parse_key(finalised_key, std::to_string(value))); + } + + THEN("each is accepted and held exactly for both counts") + { + for (auto index = std::size_t{0U}; index < values.size(); ++index) + { + INFO(values[index]); + REQUIRE_FALSE(has_finding(scheduled[index].findings, scheduled_key)); + REQUIRE(scheduled[index].config.server().delayed_events.max_scheduled_per_user == values[index]); + REQUIRE(merovingian::config::is_valid(scheduled[index].config)); + + REQUIRE_FALSE(has_finding(finalised[index].findings, finalised_key)); + REQUIRE(finalised[index].config.server().delayed_events.max_finalised_per_user == values[index]); + REQUIRE(merovingian::config::is_valid(finalised[index].config)); + } } } } GIVEN("values outside the range, or not an unsigned integer") { - THEN("each is refused with a finding naming its own key") + auto const values = + std::vector{"0", "1001", "1000000", "4294967296", "-1", "-100", "", "abc", "1.5"}; + + WHEN("each is parsed for both counts") { - for (auto const* bad : {"0", "1001", "1000000", "4294967296", "-1", "-100", "", "abc", "1.5"}) + auto scheduled = std::vector{}; + auto finalised = std::vector{}; + for (auto const& bad : values) { - INFO(bad); - auto const scheduled = parse_key(scheduled_key, bad); - REQUIRE(has_finding(scheduled.findings, scheduled_key)); - REQUIRE_FALSE(has_finding(scheduled.findings, finalised_key)); - - auto const finalised = parse_key(finalised_key, bad); - REQUIRE(has_finding(finalised.findings, finalised_key)); - REQUIRE_FALSE(has_finding(finalised.findings, scheduled_key)); + scheduled.push_back(parse_key(scheduled_key, bad)); + finalised.push_back(parse_key(finalised_key, bad)); + } + + THEN("each is refused with a finding naming its own key") + { + for (auto index = std::size_t{0U}; index < values.size(); ++index) + { + INFO(values[index]); + REQUIRE(has_finding(scheduled[index].findings, scheduled_key)); + REQUIRE_FALSE(has_finding(scheduled[index].findings, finalised_key)); + + REQUIRE(has_finding(finalised[index].findings, finalised_key)); + REQUIRE_FALSE(has_finding(finalised[index].findings, scheduled_key)); + } } } } @@ -212,10 +275,16 @@ SCENARIO("server.delayed_events.* changes are diffed by the reload plan and need { auto const before = merovingian::config::parse_key_value_config(""); auto const after = merovingian::config::parse_key_value_config(all_non_default); + auto const keys = std::vector{enabled_key, delay_key, scheduled_key, finalised_key}; - WHEN("a reload is planned") + WHEN("a reload is planned and each key is classified") { auto const plan = merovingian::config::build_reload_plan(before.config, after.config); + auto policies = std::vector{}; + for (auto const key : keys) + { + policies.push_back(merovingian::config::reload_policy_for_key(key)); + } THEN("each key is named and the plan needs a restart") { @@ -228,11 +297,10 @@ SCENARIO("server.delayed_events.* changes are diffed by the reload plan and need THEN("the policy classifies each key restart-required") { - for (auto const key : {enabled_key, delay_key, scheduled_key, finalised_key}) + for (auto index = std::size_t{0U}; index < keys.size(); ++index) { - INFO(key); - CHECK(merovingian::config::reload_policy_for_key(key) == - merovingian::config::ReloadPolicy::restart_required); + INFO(keys[index]); + CHECK(policies[index] == merovingian::config::ReloadPolicy::restart_required); } } } @@ -241,16 +309,25 @@ SCENARIO("server.delayed_events.* changes are diffed by the reload plan and need GIVEN("two configurations that differ in one key only") { auto const before = merovingian::config::parse_key_value_config(""); + auto const keys = std::vector{enabled_key, delay_key, scheduled_key, finalised_key}; - THEN("the plan names exactly that key") + WHEN("a reload is planned for each key") { - for (auto const key : {enabled_key, delay_key, scheduled_key, finalised_key}) + auto plans = std::vector{}; + for (auto const key : keys) { - INFO(key); auto const after = parse_key(key, key == enabled_key ? "false" : "5000"); - auto const plan = merovingian::config::build_reload_plan(before.config, after.config); - REQUIRE(plan.changes().size() == 1U); - REQUIRE(reported_in_plan(plan, key)); + plans.push_back(merovingian::config::build_reload_plan(before.config, after.config)); + } + + THEN("the plan names exactly that key") + { + for (auto index = std::size_t{0U}; index < keys.size(); ++index) + { + INFO(keys[index]); + REQUIRE(plans[index].changes().size() == 1U); + REQUIRE(reported_in_plan(plans[index], keys[index])); + } } } } From 45b9d5ed7d286d179e01c03fd67fd18fb4376942 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 10:30:03 +0000 Subject: [PATCH 23/29] Say what disabling delayed events leaves running and that the prune bound is for finalised rows Co-Authored-By: Claude Fable 5.2 --- .../merovingian-review/references/module-invariants.md | 4 ++-- CHANGELOG.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.agent/skills/merovingian-review/references/module-invariants.md b/.agent/skills/merovingian-review/references/module-invariants.md index 60ea15792..32e82193c 100644 --- a/.agent/skills/merovingian-review/references/module-invariants.md +++ b/.agent/skills/merovingian-review/references/module-invariants.md @@ -112,8 +112,8 @@ look at every time. Read the `AGENTS.md` itself before judging a change to that an OAuth client that has a session (ADR-0776, ADR-0777). Whatever replaces a device's credentials replaces its `oauth_sessions` row in the same transaction; a grant's audit row is in its transaction (ADR-0792). `src/database/AGENTS.md#OAuth 2.0 tables (migration 002, ADR-0776, ADR-0777)` -- A finalised-row bound and the 7-day prune apply to every `delayed_events` row; only `DelayedEventTable` touches the - rows, and the in-memory mirror changes only after the commit (one exception, ADR-0825). A path that deletes an +- The finalised-row bound and the 7-day prune apply to finalised `delayed_events` rows only; a scheduled row is bounded + by the per-user scheduled cap, never pruned by age. Only `DelayedEventTable` touches the rows, and the in-memory mirror changes only after the commit (one exception, ADR-0825). A path that deletes an account's data deletes these rows too (ADR-0824). ## src/email diff --git a/CHANGELOG.md b/CHANGELOG.md index 6316aaaa6..25c75ee6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Release notes Delayed events for MatrixRTC, on top of 1.1.1. Upgrading runs migration 003, which adds one table, and a downgrade -drops it with whatever is scheduled. The feature is on by default and `server.delayed_events.enabled = false` turns it -off. Operators running Element Call keep `max_delay_ms` at 3600000 or more. +drops it with whatever is scheduled. The feature is on by default; setting `server.delayed_events.enabled = false` and +restarting stops new scheduling, while events already scheduled are still sent and can still be managed. Operators +running Element Call keep `max_delay_ms` at 3600000 or more. # Version 1.1.1 From 631f247e8c7d4303e78ab214fa115acb1ccc2401 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 11:54:59 +0000 Subject: [PATCH 24/29] Test the delayed-events failure paths no scenario ran Co-Authored-By: Claude Fable 5.2 --- .../integration/test_delayed_events_flow.cpp | 265 ++++++++++++++++++ .../test_delayed_events_routes_flow.cpp | 29 ++ 2 files changed, 294 insertions(+) diff --git a/tests/integration/test_delayed_events_flow.cpp b/tests/integration/test_delayed_events_flow.cpp index 17e198020..aad01d536 100644 --- a/tests/integration/test_delayed_events_flow.cpp +++ b/tests/integration/test_delayed_events_flow.cpp @@ -22,9 +22,12 @@ #include "merovingian/config/LogModulesConfig.hpp" #include "merovingian/config/SecurityConfig.hpp" #include "merovingian/config/ServerConfig.hpp" +#include "merovingian/database/BoundValue.hpp" +#include "merovingian/database/DelayedEventRowText.hpp" #include "merovingian/database/DelayedEventStatus.hpp" #include "merovingian/database/DelayedEventTable.hpp" #include "merovingian/database/PersistentStore.hpp" +#include "merovingian/database/PreparedStatement.hpp" #include "merovingian/homeserver/AuthService.hpp" #include "merovingian/homeserver/ClientServerRuntime.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" @@ -40,6 +43,7 @@ #include #include +#include #include #include #include @@ -1702,3 +1706,264 @@ SCENARIO("The job body returns what the send function returns for the clock's ti } } } + +SCENARIO("A failed store write on schedule is a 500, stores nothing and does not wake the job", + "[homeserver][delayed-events][schedule][store-failure]") +{ + GIVEN("a runtime whose wake hook counts and whose insert of a delayed event fails once") + { + auto fixture = Fixture{memory_config()}; + auto wakes = 0; + fixture.runtime().homeserver.delayed_events_wake = [&wakes]() { + ++wakes; + }; + auto const scheduled_before = fixture.table().scheduled_count(alice); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "insert_delayed_event"); + + WHEN("alice schedules an event, and then the same request again") + { + auto const failed = schedule(fixture, message_request(fixture)); + auto const scheduled_after_failure = fixture.table().scheduled_count(alice); + auto const wakes_after_failure = wakes; + auto const retried = schedule(fixture, message_request(fixture)); + + THEN("the first is 500 M_UNKNOWN with nothing stored and no wake, and the retry succeeds") + { + CHECK_FALSE(failed.ok); + CHECK(failed.status == 500U); + CHECK(failed.errcode == "M_UNKNOWN"); + CHECK(scheduled_after_failure == scheduled_before); + CHECK(wakes_after_failure == 0); + CHECK(retried.ok); + CHECK(fixture.table().scheduled_count(alice) == scheduled_before + 1U); + CHECK(wakes == 1); + } + } + fixture.runtime().homeserver.delayed_events_wake = nullptr; + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.clear(); + } +} + +SCENARIO("A failed store write on restart is a 500 and leaves the timer alone", + "[homeserver][delayed-events][manage][store-failure]") +{ + GIVEN("an event scheduled at t0 for 60 s and an update of its timer that fails once") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "txn-1", 60'000U)); + auto const before = fixture.event_count(); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "restart_delayed_event"); + + WHEN("alice restarts it halfway, and the job runs at the original deadline") + { + fixture.runtime().homeserver.test_delayed_event_clock_ms = t0 + 30'000U; + auto const restart = manage(fixture, alice, id, hs::DelayedEventAction::restart); + auto const shown = looked_up(fixture, alice, id); + drain_job(fixture, t0 + 60'000U); + + THEN("the restart is 500 M_UNKNOWN, the timer still starts at t0 and the event is sent at t0 + 60 s") + { + CHECK_FALSE(restart.ok); + CHECK(restart.status == 500U); + CHECK(restart.errcode == "M_UNKNOWN"); + CHECK(*int_member(shown, "delayed_since_ts") == static_cast(t0)); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 1U); + } + } + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.clear(); + } +} + +SCENARIO("A failed store write on cancel is a 500 and leaves the event scheduled", + "[homeserver][delayed-events][manage][store-failure]") +{ + GIVEN("a scheduled event and a finalising write that fails once") + { + auto fixture = Fixture{memory_config()}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "txn-1", 10'000U)); + auto const before = fixture.event_count(); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "finalise_delayed_event"); + + WHEN("alice cancels it, and the job runs at the deadline") + { + auto const cancel = manage(fixture, alice, id, hs::DelayedEventAction::cancel); + auto const status_after_cancel = status_of(fixture, alice, id); + drain_job(fixture, t0 + 10'000U); + + THEN("the cancel is 500 M_UNKNOWN, the event stayed scheduled and the job then sent it") + { + CHECK_FALSE(cancel.ok); + CHECK(cancel.status == 500U); + CHECK(cancel.errcode == "M_UNKNOWN"); + CHECK(status_after_cancel == db::DelayedEventStatus::scheduled); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 1U); + } + } + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.clear(); + } +} + +SCENARIO("A failed age prune is tolerated and the row goes on a later run", + "[homeserver][delayed-events][job][retention][store-failure]") +{ + GIVEN("a row finalised eight days before the run and a prune DELETE that fails once") + { + auto fixture = Fixture{memory_config()}; + auto constexpr day = std::uint64_t{24U * 60U * 60U * 1000U}; + auto const id = scheduled_id(fixture, message_request(fixture, alice, "old")); + REQUIRE(manage(fixture, alice, id, hs::DelayedEventAction::cancel).ok); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "delete_delayed_events"); + + WHEN("the job runs at t0 + 8 days, and again") + { + auto failed_wait = std::chrono::milliseconds{0}; + CHECK_NOTHROW(failed_wait = run_job(fixture, t0 + 8U * day)); + auto const held_after_failure = fixture.table().find_owned(alice, id) != nullptr; + std::ignore = run_job(fixture, t0 + 8U * day); + + THEN("the first run returns its normal wait with the row held, and the second removes the row") + { + CHECK(failed_wait == std::chrono::milliseconds{60'000}); + CHECK(held_after_failure); + CHECK(fixture.table().find_owned(alice, id) == nullptr); + } + } + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.clear(); + } +} + +SCENARIO("With no rate-limit engine a due event is still sent, and so is the send action", + "[homeserver][delayed-events][job][manage][rate-limit]") +{ + GIVEN("a runtime with no rate-limit engine, a due event and a scheduled one") + { + auto fixture = Fixture{memory_config()}; + REQUIRE(fixture.runtime().rate_limit_engine != nullptr); + fixture.runtime().rate_limit_engine.reset(); + auto const due = scheduled_id(fixture, message_request(fixture, alice, "a", 10'000U)); + auto const later = scheduled_id(fixture, message_request(fixture, alice, "b", 60'000U)); + auto const before = fixture.event_count(); + + WHEN("the job runs at the due time and alice sends the other now") + { + drain_job(fixture, t0 + 10'000U); + auto const sent = manage(fixture, alice, later, hs::DelayedEventAction::send); + + THEN("both events were sent") + { + CHECK(sent.ok); + CHECK(status_of(fixture, alice, due) == db::DelayedEventStatus::sent); + CHECK(status_of(fixture, alice, later) == db::DelayedEventStatus::sent); + CHECK(fixture.event_count() == before + 2U); + } + } + } +} + +SCENARIO("The real clock reads a time after 2026-01-01", "[homeserver][delayed-events][clock]") +{ + GIVEN("a runtime with the test clock unset") + { + auto fixture = Fixture{memory_config()}; + fixture.runtime().homeserver.test_delayed_event_clock_ms.reset(); + + WHEN("the delayed-event clock is read") + { + auto const now = hs::delayed_event_now_ms(fixture.runtime().homeserver); + + THEN("it is at least 2026-01-01T00:00:00Z in milliseconds") + { + CHECK(now >= std::uint64_t{1'767'225'600'000U}); + } + } + } +} + +SCENARIO("A stored row whose content is not a JSON object fails when due and cannot be shown", + "[homeserver][delayed-events][persistence][job][lookup]") +{ + GIVEN("a SQLite file holding two scheduled rows of alice, one with content [] and one with content not json") + { + auto const path = unique_sqlite_path(); + std::filesystem::remove(path); + auto const config = sqlite_config(path); + auto const array_id = std::string{"stored-array-content"}; + auto const text_id = std::string{"stored-text-content"}; + { + auto first = Fixture{config}; + auto const insert = [&first](std::string const& delay_id, std::string const& content, + std::string const& delay_ms) { + auto const values = std::array{delay_id, + std::string{alice}, + "D1", + first.room_id, + "m.test.delayed", + "false", + "", + content, + "", + "false", + delay_ms, + "1700000000000", + "scheduled", + "0", + "", + ""}; + auto statement = db::PreparedStatement{ + "insert_raw_delayed_event", + "INSERT INTO delayed_events (delay_id, user_id, device_id, room_id, event_type, is_state, " + "state_key, content_json, txn_id, via_appservice, delay_ms, delayed_since_ms, status, " + "finalised_at_ms, event_id, error_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, " + "$12, $13, $14, $15, $16)", + {}}; + for (auto const& value : values) + { + statement.parameters.push_back(db::BoundValue{value, false}); + } + return db::commit_persistent_transaction(first.runtime().homeserver.database.persistent_store, + {statement}); + }; + REQUIRE(insert(array_id, "[]", "1000")); + REQUIRE(insert(text_id, "not json", "5000")); + } + auto restarted = hs::start_client_server(config); + REQUIRE(restarted.started); + auto& homeserver = restarted.runtime.homeserver; + homeserver.test_frozen_event_clock_ms = static_cast(t0); + REQUIRE(homeserver.database.persistent_store.delayed_events.find_owned(alice, array_id) != nullptr); + REQUIRE(homeserver.database.persistent_store.delayed_events.find_owned(alice, text_id) != nullptr); + + WHEN("the unreadable one is looked up, and the job runs when only the array one is due") + { + auto const before = homeserver.database.persistent_store.events.size(); + auto const shown = [&restarted, &text_id]() { + auto const guard = hs::RuntimeLock{restarted.runtime.homeserver.mutex}; + return hs::delayed_event_lookup(restarted.runtime, alice, text_id); + }(); + std::ignore = hs::send_due_delayed_events(restarted.runtime, t0 + 1'000U, {}); + auto const* row = homeserver.database.persistent_store.delayed_events.find_owned(alice, array_id); + REQUIRE(row != nullptr); + auto const failed = row->status == db::DelayedEventStatus::failed; + auto const error_text = row->error_json; + auto const sent_count = homeserver.database.persistent_store.events.size() - before; + + THEN("the lookup is 500 M_UNKNOWN, and the array row is failed M_BAD_JSON with nothing sent") + { + CHECK_FALSE(shown.ok); + CHECK(shown.status == 500U); + CHECK(shown.errcode == "M_UNKNOWN"); + CHECK(failed); + CHECK(*string_member(parse_object(error_text), "errcode") == "M_BAD_JSON"); + CHECK(sent_count == 0U); + } + } + + std::filesystem::remove(path); + } +} diff --git a/tests/integration/test_delayed_events_routes_flow.cpp b/tests/integration/test_delayed_events_routes_flow.cpp index 42a3ff77d..cb0074dd6 100644 --- a/tests/integration/test_delayed_events_routes_flow.cpp +++ b/tests/integration/test_delayed_events_routes_flow.cpp @@ -1751,3 +1751,32 @@ SCENARIO("An application service scheduling for a user is recorded as such, and } } } + +SCENARIO("A failed store write on restart is answered 500 M_UNKNOWN and leaves the event scheduled with its timer", + "[homeserver][delayed-events][routes]") +{ + GIVEN("an event scheduled at t0 and an update of its timer that fails once") + { + auto fixture = Fixture{}; + auto const id = schedule_message(fixture, "r1", 60'000U); + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.insert( + "restart_delayed_event"); + + WHEN("alice posts the restart halfway through the delay, and then reads the event") + { + fixture.set_time(t0 + 30'000U); + auto const restart = post_action(fixture, true, id, "restart", fixture.alice_token); + auto const shown = get_delayed(fixture, true, id, fixture.alice_token); + + THEN("the restart is HTTP 500 M_UNKNOWN and the event is still scheduled since t0") + { + CHECK(restart.status == 500U); + CHECK(errcode_of(restart) == "M_UNKNOWN"); + REQUIRE(shown.status == 200U); + CHECK(*int_member(parse_object(shown.body), "delayed_since_ts") == static_cast(t0)); + CHECK(status_of(fixture, alice, id) == db::DelayedEventStatus::scheduled); + } + } + fixture.runtime().homeserver.database.persistent_store.force_failure_statement_names.clear(); + } +} From 5a8b5db10898f9266e21473860ff570b0d3ed17d Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 12:07:17 +0000 Subject: [PATCH 25/29] Say in the 1.1.2 release notes that an older server refuses the migrated database Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25c75ee6c..57f9ea3ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,10 +28,11 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Release notes -Delayed events for MatrixRTC, on top of 1.1.1. Upgrading runs migration 003, which adds one table, and a downgrade -drops it with whatever is scheduled. The feature is on by default; setting `server.delayed_events.enabled = false` and -restarting stops new scheduling, while events already scheduled are still sent and can still be managed. Operators -running Element Call keep `max_delay_ms` at 3600000 or more. +Delayed events for MatrixRTC, on top of 1.1.1. 1.1.2 migrates the database: on first start it applies migration 003, +which adds one empty table and changes no existing one. A 1.1.1 or older server refuses the migrated database, so +back it up before upgrading if you may need to go back. The feature is on by default; setting +`server.delayed_events.enabled = false` and restarting stops new scheduling, while events already scheduled are still +sent and can still be managed. Operators running Element Call keep `max_delay_ms` at 3600000 or more. # Version 1.1.1 From 1642111b4550f1188d2027b38bf7eef3e09dd1bb Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 14:33:29 +0000 Subject: [PATCH 26/29] Load the delayed-events rows through a named helper instead of a short-circuit Co-Authored-By: Claude Fable 5.2 --- src/database/PostgresqlStore.cpp | 7 +++- src/database/SqliteStore.cpp | 62 +++++++++++++++++++------------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/database/PostgresqlStore.cpp b/src/database/PostgresqlStore.cpp index b78507c8b..9268d0dbd 100644 --- a/src/database/PostgresqlStore.cpp +++ b/src/database/PostgresqlStore.cpp @@ -1438,7 +1438,12 @@ namespace { // Strict: a row that is not read exactly is skipped, left in the database and counted. auto parsed = parse_delayed_event_row(row); - if (!parsed.has_value() || !store.delayed_events.insert(std::move(*parsed))) + if (!parsed.has_value()) + { + ++store.delayed_event_rows_skipped_at_load; + continue; + } + if (!store.delayed_events.insert(std::move(*parsed))) { ++store.delayed_event_rows_skipped_at_load; } diff --git a/src/database/SqliteStore.cpp b/src/database/SqliteStore.cpp index f1465d68a..307f45964 100644 --- a/src/database/SqliteStore.cpp +++ b/src/database/SqliteStore.cpp @@ -756,6 +756,42 @@ namespace return step == SQLITE_DONE; } + // Migration 003 (ADR-0824): delayed events. A row whose status text is not one of the four, or that + // duplicates a delay ID, a transaction key or a sent event ID, or whose numbers + // or booleans are not exactly what the store writes, is skipped and counted, never repaired. + // Skipped for a profile that does not include the table, as PostgreSQL does: the federation worker never + // reads a scheduled event's content, and a skipped load counts no row. + [[nodiscard]] auto load_delayed_event_rows(sqlite3& connection, PersistentStore& store, TableLoadProfile profile) + -> bool + { + if (!table_load_profile_includes("delayed_events", profile)) + { + return true; + } + return load_rows(connection, + "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, content_json, " + "txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " + "error_json FROM delayed_events ORDER BY delay_id", + [&store](sqlite3_stmt& row) { + auto columns = std::array{}; + for (auto index = std::size_t{0U}; index < columns.size(); ++index) + { + columns[index] = column_text(row, static_cast(index)); + } + // Strict: a row that is not read exactly is skipped, left in the file and counted. + auto parsed = parse_delayed_event_row(columns); + if (!parsed.has_value()) + { + ++store.delayed_event_rows_skipped_at_load; + return; + } + if (!store.delayed_events.insert(std::move(*parsed))) + { + ++store.delayed_event_rows_skipped_at_load; + } + }); + } + // Events go to `hydrated_events` rather than `store.events`, for the caller to index in one pass // (ADR-0215): the SELECT has no ORDER BY, so adding them one at a time could shift the // stream-order index for every row. @@ -1183,30 +1219,8 @@ namespace column_text(row, 2), column_text(row, 3), parse_u64(column_text(row, 4))}); }) && - // Migration 003 (ADR-0824): delayed events. A row whose status text is not one of the four, or that - // duplicates a delay ID, a transaction key or a sent event ID, or whose numbers - // or booleans are not exactly what the store writes, is skipped and counted, never repaired. - // Skipped for a profile that does not include the table, as PostgreSQL does: the federation worker never - // reads a scheduled event's content, and a skipped load counts no row. - (!table_load_profile_includes("delayed_events", profile) || - load_rows( - connection, - "SELECT delay_id, user_id, device_id, room_id, event_type, is_state, state_key, content_json, " - "txn_id, via_appservice, delay_ms, delayed_since_ms, status, finalised_at_ms, event_id, " - "error_json FROM delayed_events ORDER BY delay_id", - [&store](sqlite3_stmt& row) { - auto columns = std::array{}; - for (auto index = std::size_t{0U}; index < columns.size(); ++index) - { - columns[index] = column_text(row, static_cast(index)); - } - // Strict: a row that is not read exactly is skipped, left in the file and counted. - auto parsed = parse_delayed_event_row(columns); - if (!parsed.has_value() || !store.delayed_events.insert(std::move(*parsed))) - { - ++store.delayed_event_rows_skipped_at_load; - } - })) && + // Migration 003 (ADR-0824): delayed events, loaded by the helper above. + load_delayed_event_rows(connection, store, profile) && // Application service delivery state: transaction cursors, the outbox and the directory mapping. load_rows(connection, "SELECT appservice_id, next_txn_id, delivered_stream_ordering, pending_txn_id, " From f601d2446ac9e82d1bbe2bd0bb574908582b0605 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 17:20:45 +0000 Subject: [PATCH 27/29] Explain what the long delayed-events scenarios prove and why their steps exist Co-Authored-By: Claude Fable 5.2 --- .../integration/test_delayed_events_flow.cpp | 68 ++++++++++++++++ .../test_postgresql_store_behaviour_flow.cpp | 27 +++++++ tests/unit/test_delayed_event_store.cpp | 78 +++++++++++++++++++ tests/unit/test_delayed_event_table.cpp | 13 ++++ 4 files changed, 186 insertions(+) diff --git a/tests/integration/test_delayed_events_flow.cpp b/tests/integration/test_delayed_events_flow.cpp index aad01d536..5761cff2d 100644 --- a/tests/integration/test_delayed_events_flow.cpp +++ b/tests/integration/test_delayed_events_flow.cpp @@ -518,6 +518,11 @@ SCENARIO("A delayed state event is sent as a state event and stamped with the ti } } +// The spec answers a repeated management action with 200 or 409 by the state the event is in (ADR-0824, ADR-0825), and +// a restart moves the deadline. The first GIVEN proves the restart and the cancel on one event. The second holds one +// event in each state of its life (scheduled, sent, cancelled, failed and being sent) and asks every action of every +// state, so a cell of the outcome table that nobody thought of is still answered. The cells share one GIVEN because +// building the five states is the expensive part, and a THEN loop with INFO names the cell that fails. SCENARIO("Restarting a delayed event moves its deadline, and every cell of the outcome table is answered", "[homeserver][delayed-events][manage]") { @@ -538,6 +543,10 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o CHECK(restarted.value.empty()); CHECK(*int_member(looked_up(fixture, alice, id), "delayed_since_ts") == static_cast(t0 + 20'000U)); + // t0 + 60 s is the due time the event had before the restart. The restart moved the deadline to the + // restart time plus the delay, so this run must send nothing, and one millisecond before the new + // deadline must send nothing either. Only the exact new deadline sends, which is the boundary of never + // early (ADR-0825). drain_job(fixture, t0 + 60'000U); CHECK(fixture.event_count() == before); drain_job(fixture, t0 + 80'000U - 1U); @@ -547,6 +556,8 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o } } + // A cancelled row stays in the table so that a lookup still answers (ADR-0824). The clock is moved far past the + // due time, so a cancelled row that the job still offered as due would be sent. WHEN("it is cancelled") { auto const cancelled = manage(fixture, alice, id, hs::DelayedEventAction::cancel); @@ -564,6 +575,8 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o } } + // Each state is reached the way it is reached in production, through the job and the management calls, and not by + // writing the status, so the table holds what the runtime would hold. GIVEN("one delayed event in each state of its life") { auto fixture = Fixture{memory_config()}; @@ -573,6 +586,8 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o auto const failed = scheduled_id(fixture, message_request(fixture, bob, "d", 2'000U)); auto const being_sent = scheduled_id(fixture, message_request(fixture, alice, "e")); REQUIRE(manage(fixture, alice, cancelled, hs::DelayedEventAction::cancel).ok); + // Bob leaves the room, so his event fails when it falls due: the sender is authorised against the room as it is + // then, not as it was when the event was scheduled (ADR-0825). That is how the failed state is reached. REQUIRE(request(fixture.runtime(), "POST", "/_matrix/client/v3/rooms/" + fixture.room_id + "/leave", fixture.bob_token, "{}") .status == 200U); @@ -580,6 +595,8 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o drain_job(fixture, t0 + 5'000U); REQUIRE(status_of(fixture, alice, sent) == db::DelayedEventStatus::sent); REQUIRE(status_of(fixture, bob, failed) == db::DelayedEventStatus::failed); + // The being-sent mark is held only while the job is inside the send, and a test cannot stop there, so the mark + // is set through the table. It is in memory only, as in production, and it makes every action answer 409. REQUIRE(fixture.table().mark_being_sent(being_sent)); using Action = hs::DelayedEventAction; @@ -593,6 +610,7 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o std::uint16_t status; std::string_view reason; }; + // The reasons are the exact texts a client reads, so a reworded refusal is a failure. auto const sent_reason = std::string_view{"The delayed event has already been sent."}; auto const cancelled_reason = std::string_view{"The delayed event has already been cancelled."}; auto const being_reason = std::string_view{"The delayed event is being sent."}; @@ -600,6 +618,10 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o THEN("the outcome table of the design holds cell by cell") { auto const cells = std::vector{ + // A sent event answers `send` with 200 and the other two with 409. A cancelled event and a failed one + // are both cancelled for the client: the same reason text, and `cancel` is the repeat that answers 200. + // A being-sent event answers 409 to all three, since a second send, restart or cancel must not race the + // send. {"sent restart", &sent, alice, Action::restart, false, 409U, sent_reason }, {"sent cancel", &sent, alice, Action::cancel, false, 409U, sent_reason }, {"sent send", &sent, alice, Action::send, true, 200U, {} }, @@ -625,6 +647,8 @@ SCENARIO("Restarting a delayed event moves its deadline, and every cell of the o CHECK(result.reason == cell.reason); } } + // The scheduled event was never asked about, and a being-sent row refused every action, so neither has + // changed. CHECK(status_of(fixture, alice, scheduled) == db::DelayedEventStatus::scheduled); CHECK(status_of(fixture, alice, being_sent) == db::DelayedEventStatus::scheduled); } @@ -996,12 +1020,18 @@ SCENARIO("Scheduling the same transaction again returns the same delay ID and st } } +// Scheduling checks what it can check cheaply at once: that the sender is in the room and, for a state event, may send +// it. The real authorisation is at send time (ADR-0825). This scenario proves that a refusal at scheduling is the +// refusal the ordinary route gives for the same event, status for status, so a client cannot tell a delayed send from +// an immediate one by how it is refused, and that nothing is stored when it is refused. The second WHEN proves the size +// and shape limits (ADR-0826) at their boundary, again with nothing stored. SCENARIO("A delayed event the sender could not send now is refused when it is scheduled, as the ordinary routes refuse", "[homeserver][delayed-events][schedule][refusal]") { GIVEN("a room, a user who is not in it and a user without the power to send state") { auto fixture = Fixture{memory_config()}; + // Carol is signed in and is not in the room, so the only reason to refuse her is that she is not a member. auto const carol_token = sign_in(fixture.runtime(), "carol", "D3"); auto const unknown_room = std::string{"!nowhere:example.org"}; @@ -1013,10 +1043,14 @@ SCENARIO("A delayed event the sender could not send now is refused when it is sc auto const unknown_ordinary = request( fixture.runtime(), "PUT", "/_matrix/client/v3/rooms/" + unknown_room + "/send/m.test.delayed/t1", fixture.alice_token, R"({"body":"x"})"); + // A user who is not in the room, and a room that does not exist, are refused by the ordinary route too. The + // same event is sent through it so the expected status is read from the real route and not written here. auto const stranger_result = schedule(fixture, message_request(fixture, "@carol:example.org")); auto const stranger_ordinary = request( fixture.runtime(), "PUT", "/_matrix/client/v3/rooms/" + fixture.room_id + "/send/m.test.delayed/t1", carol_token, R"({"body":"x"})"); + // Bob is in the room, so membership cannot be what refuses him. A state event needs a power level that he + // does not have, which is the `may_send_state_event` check. auto const weak_result = schedule(fixture, state_request(fixture, bob)); auto const weak_ordinary = request(fixture.runtime(), "PUT", "/_matrix/client/v3/rooms/" + fixture.room_id + "/state/m.test.state", @@ -1038,6 +1072,10 @@ SCENARIO("A delayed event the sender could not send now is refused when it is sc WHEN("an oversized type, transaction ID or content, or a NUL byte, is scheduled") { + // Each value is one over a limit, or a shape the ordinary route refuses: 256 bytes is one over the 255 byte + // limit of a type, a transaction ID and a state key, and the content is over the 65,536 byte event limit. A + // NUL byte is refused because the stored text cannot hold one, a delay of 0 because it would be due at + // once, and content that is not an object because an event's content is an object. auto long_type = message_request(fixture); long_type.event_type = std::string(256U, 't'); auto long_txn = message_request(fixture, alice, std::string(256U, 'x')); @@ -1056,6 +1094,8 @@ SCENARIO("A delayed event the sender could not send now is refused when it is sc THEN("each gets the ordinary answer and nothing is stored") { + // The oversized type and content are 413, and the transaction ID is 400, as the ordinary send route + // answers them. auto const type_result = schedule(fixture, long_type); CHECK(type_result.status == 413U); CHECK(type_result.errcode == "M_TOO_LARGE"); @@ -1282,6 +1322,11 @@ SCENARIO("Deactivation succeeds when the delayed-events delete fails, and the jo } } +// The job body returns how long its caller should sleep (ADR-0825). Too short a wait wastes a thread, too long a wait +// sends late, and 0 would spin, so the return value is part of the contract. This scenario proves the exact wait to the +// next deadline, the cap of 60 seconds, the pace of 5 ms while another event is due so that a backlog cannot starve +// requests of the lock, one send per run in deadline order, and that a set stop flag sends nothing. It never sleeps: +// the job is called with a chosen `now_ms`. SCENARIO("The job's return value says when it should run next, and it stops when asked", "[homeserver][delayed-events][job][pacing]") { @@ -1299,6 +1344,10 @@ SCENARIO("The job's return value says when it should run next, and it stops when } } + // The three runs are made before the THEN and at three different clocks. At t0 + 500 ms the first event is + // 12,000 ms away. At t0 - 1 ms, a clock before the schedule, it is 12,501 ms away, which shows the wait follows + // the clock and not a stored value. After the first event is cancelled the next is 120 seconds away, so the + // wait is capped. WHEN("an event due in 12.5 s and one due in two minutes are scheduled") { std::ignore = scheduled_id(fixture, message_request(fixture, alice, "a", 12'500U)); @@ -1321,6 +1370,8 @@ SCENARIO("The job's return value says when it should run next, and it stops when GIVEN("102 events due at once, due 1 ms apart") { auto fixture = Fixture{memory_config()}; + // 102 is more than the default of 100 scheduled events per user, so the cap is raised to the largest value, + // 1,000, to schedule them all. fixture.runtime().limits.max_scheduled_delayed_events_per_user = 1000U; auto ids = std::vector{}; for (auto index = 0; index < 102; ++index) @@ -1331,6 +1382,9 @@ SCENARIO("The job's return value says when it should run next, and it stops when } auto const before = fixture.event_count(); + // Every event is already due, so one run that sent them all would hold the lock for the whole backlog. The run + // must send only the earliest, which is the last one scheduled, and ask to run again after 5 ms and not at + // once. WHEN("the job runs once") { auto const wait = run_job(fixture, t0 + 5'000U); @@ -1376,6 +1430,8 @@ SCENARIO("The job's return value says when it should run next, and it stops when } } + // The stop flag is checked before a send, so a server that is shutting down sends nothing it has not already + // begun. WHEN("the job runs with its stop flag already set") { auto stop = hs::StopFlag{}; @@ -1390,6 +1446,8 @@ SCENARIO("The job's return value says when it should run next, and it stops when } } + // The next deadline is 2 ms away, less than the pace, so the pace is the lower bound of the wait and the wait is + // never shorter than 5 ms. GIVEN("one event due now and one due in 2 ms") { auto fixture = Fixture{memory_config()}; @@ -1885,6 +1943,11 @@ SCENARIO("The real clock reads a time after 2026-01-01", "[homeserver][delayed-e } } +// The loader does not parse a row's content, and the scheduling path refuses content that is not a JSON object, so a +// stored row with such content can only come from a damaged or foreign database. The job must fail it as a send that +// cannot be made, with the standard error stored, and send nothing. A lookup of such a row must not show content it +// cannot read. The rows are written with raw SQL on purpose, because the store functions and the scheduling path would +// refuse them, and the server is restarted so that the loader is what reads them. SCENARIO("A stored row whose content is not a JSON object fails when due and cannot be shown", "[homeserver][delayed-events][persistence][job][lookup]") { @@ -1929,13 +1992,18 @@ SCENARIO("A stored row whose content is not a JSON object fails when due and can return db::commit_persistent_transaction(first.runtime().homeserver.database.persistent_store, {statement}); }; + // The array row is due 1,000 ms after the stored start and the text row 5,000 ms after it, so a run at t0 + + // 1,000 ms touches only the array row, and the text row is the one a lookup is asked about. REQUIRE(insert(array_id, "[]", "1000")); REQUIRE(insert(text_id, "not json", "5000")); } auto restarted = hs::start_client_server(config); REQUIRE(restarted.started); auto& homeserver = restarted.runtime.homeserver; + // The clock is frozen at t0, which is the start time stored in the rows, so the due times are exact. homeserver.test_frozen_event_clock_ms = static_cast(t0); + // Both rows must have loaded: the loader accepts any non-empty content, so the failure being proved is the + // job's and not the loader's. REQUIRE(homeserver.database.persistent_store.delayed_events.find_owned(alice, array_id) != nullptr); REQUIRE(homeserver.database.persistent_store.delayed_events.find_owned(alice, text_id) != nullptr); diff --git a/tests/integration/test_postgresql_store_behaviour_flow.cpp b/tests/integration/test_postgresql_store_behaviour_flow.cpp index 615c5cb4d..5116f203f 100644 --- a/tests/integration/test_postgresql_store_behaviour_flow.cpp +++ b/tests/integration/test_postgresql_store_behaviour_flow.cpp @@ -1687,6 +1687,11 @@ SCENARIO("PostgreSQL OAuth clients and sessions survive a reopen and a session g } } +// A delayed event must survive a restart, or a call's scheduled leave is lost (ADR-0824). This is the PostgreSQL twin +// of the SQLite reopen scenario in `test_delayed_event_store.cpp`: PostgreSQL has its own loader, and its own text +// columns and bound parameters, so the same round trip must be proved on it. One row of each status is written through +// the store functions, the store is closed and opened again, and every column and index is read back. It needs a +// database, so it is skipped, not failed, when `MEROVINGIAN_TEST_POSTGRESQL_URI` is not set. SCENARIO("PostgreSQL delayed events of every status survive a reopen with every column and index", "[database][postgresql][integration][pg-behaviour][delayed-events]") { @@ -1699,6 +1704,8 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every return; } auto opened = reopen(uri); + // The database is shared by every test and every run, so each ID carries a suffix that no other run uses. A row + // of another test can then never collide with these, and the counts below are the counts of these users alone. auto const suffix = unique_suffix(); auto const alice = "@delayed-a-" + suffix + ":example.org"; auto const bob = "@delayed-b-" + suffix + ":example.org"; @@ -1719,6 +1726,8 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every state.is_state = true; state.state_key = alice; state.via_appservice = true; + // The largest delay and a large `delayed_since_ms` check that the numbers come back as the same values, and the + // state key and appservice flag fill the columns that a message leaves empty. state.delay_ms = db::max_delayed_event_delay_ms; state.delayed_since_ms = 123'456'789U; auto message = make("message", alice); @@ -1736,6 +1745,8 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every 2002U, {}, R"({"errcode":"M_FORBIDDEN","error":"nope"})", 10U) == db::DelayedEventOutcomeResult::recorded); + // The first store is discarded before the second is opened, so every value below was read from the database and + // none from the memory that wrote it. WHEN("the store is reopened") { opened = {}; @@ -1780,6 +1791,7 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every REQUIRE(table.find_sent_by_event_id("$sent-" + suffix) == table.find("d-sent-" + suffix)); } + // A third open proves the DELETE reached the database, not just the mirror in `reopened`. AND_THEN("deleting both users' rows removes them from the database as well") { REQUIRE(db::delete_delayed_events_of_user(reopened.store, alice)); @@ -1793,6 +1805,11 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every } } +// The loader reads a row strictly and skips one it cannot read exactly, never repairing it or guessing, because a delay +// read as 0 would make a scheduled row due at once, which is sending early (ADR-0824). The damaged rows cannot be +// written through the store functions, which refuse them, so they are inserted with raw SQL, as a damaged or foreign +// database would hold them. Each case damages one column of an otherwise valid row, so the skipped count names the +// case. The valid row proves the loader keeps going after a skip. SCENARIO("PostgreSQL hydration skips delayed event rows that are not read exactly and counts them", "[database][postgresql][integration][pg-behaviour][delayed-events]") { @@ -1806,6 +1823,8 @@ SCENARIO("PostgreSQL hydration skips delayed event rows that are not read exactl } auto const suffix = unique_suffix(); auto const user = "@strict-" + suffix + ":example.org"; + // The shared database may already hold skipped rows from other tests, so the count is judged as a difference + // from what an open reports before these rows exist. auto const baseline = reopen(uri).store.delayed_event_rows_skipped_at_load; struct Damaged final @@ -1815,6 +1834,9 @@ SCENARIO("PostgreSQL hydration skips delayed event rows that are not read exactl std::string_view value; }; auto const cases = std::vector{ + // Column 10 is `delay_ms`, column 11 is `delayed_since_ms` and column 5 is `is_state`. A number must be 1 + // or more ASCII digits that fit 64 bits, a scheduled row's delay must be 1 to the 31 day maximum + // (2,678,400,000 ms), and a boolean must be exactly `true` or `false`. {"not a number", 10U, "abc" }, {"empty", 10U, "" }, {"negative", 10U, "-5" }, @@ -1857,9 +1879,12 @@ SCENARIO("PostgreSQL hydration skips delayed event rows that are not read exactl { statement.parameters.push_back(db::BoundValue{column, false}); } + // Each row is committed through its own short-lived store, which is dropped at the end of the lambda, so + // the store under test has not seen any of them. auto opened = reopen(uri); return db::commit_persistent_transaction(opened.store, {statement}); }; + // A column index past the end damages nothing, so this inserts the one valid row. REQUIRE(insert_raw("d-strict-good-" + suffix, 99U, {})); for (auto const& damaged : cases) { @@ -1877,6 +1902,8 @@ SCENARIO("PostgreSQL hydration skips delayed event rows that are not read exactl CHECK(reopened.store.delayed_event_rows_skipped_at_load == baseline + cases.size()); } + // A skipped row is in the database and not in memory, and deactivation must still remove it (ADR-0824). The + // skipped count falling back to the baseline on the next open shows the DELETE reached the skipped rows. AND_THEN("deleting the user's rows removes the skipped rows from the database as well") { REQUIRE(db::delete_delayed_events_of_user(reopened.store, user)); diff --git a/tests/unit/test_delayed_event_store.cpp b/tests/unit/test_delayed_event_store.cpp index ed07d9d31..9612bacfd 100644 --- a/tests/unit/test_delayed_event_store.cpp +++ b/tests/unit/test_delayed_event_store.cpp @@ -176,6 +176,11 @@ SCENARIO("The delayed-event delay maximum is one number in the config and the da } } +// `store_delayed_event` is the one place a new row enters the table, and it judges the row before it builds a statement +// (ADR-0824). This scenario proves what it refuses and what it accepts: every length cap is tested at the cap and one +// byte over, a refusal writes nothing, and a repeated delay ID or transaction is refused without a write. The caps are +// cases of one table, not scenarios, because each case is the same call on a fresh store and only the changed field +// differs. The per-user cap on scheduled rows is not enforced here on purpose, and the last WHEN pins that. SCENARIO("A delayed event is stored once, within its caps, and refused outside them", "[database][delayed-events][delayed-event-store]") { @@ -192,6 +197,9 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t std::function change; bool stored; }; + // Each limit is a pair, the largest accepted value and one over it, so a comparison written as < instead of + // <= moves the boundary by one and fails a case. The lambda changes one field of an otherwise valid row, so + // a refusal can only be that field's. auto const cases = std::vector{ {"baseline", [](PersistentDelayedEvent&) { @@ -260,6 +268,9 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t [](PersistentDelayedEvent& r) { r.device_id = std::string(db::max_delayed_event_field_bytes, 'v'); }, true }, + // A NUL byte in any text column is refused, because PostgreSQL TEXT cannot hold one and the table's + // transaction key joins its parts with one. Every column gets its own case so that a check missing from + // one column is a failure. {"delay id with a NUL byte", [](PersistentDelayedEvent& r) { r.delay_id += std::string{'\0'} + "x"; @@ -316,6 +327,9 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t [](PersistentDelayedEvent& r) { r.delay_ms = 1U; }, true }, + // A delay of 0 is refused. Read back as a scheduled row it would be due at once, which is sending early + // (ADR-0824). The largest delay is the 31 day maximum that the config module must agree with (the first + // scenario of this file). {"delay of zero", [](PersistentDelayedEvent& r) { r.delay_ms = 0U; @@ -328,6 +342,9 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t [](PersistentDelayedEvent& r) { r.delay_ms = db::max_delayed_event_delay_ms + 1U; }, false}, + // A new row is always scheduled with no outcome. A row reaches sent, cancelled or failed only by + // finalising a scheduled one, so a row that already claims an outcome, a finalised time, an event ID or + // an error is refused. {"status sent", [](PersistentDelayedEvent& r) { r.status = DelayedEventStatus::sent; @@ -353,12 +370,16 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t r.error_json = "{}"; }, false}, }; + // The outcomes and the transaction counts are collected first and judged in the THEN, so the loop only acts + // and the THEN holds every assertion about it. auto outcomes = std::vector>{}; auto transactions_by_refusal = std::vector{}; for (auto const& each : cases) { auto candidate = row("d-case"); each.change(candidate); + // A refused row must leave the database untouched, so the transaction count is read before the call and + // the difference is recorded for every refused case. // A fresh store per case: the baseline's ID would otherwise collide with the next case. auto fresh = opened_store(); auto const before = db::committed_transaction_count(fresh.store); @@ -384,6 +405,9 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t } } + // The delay ID is the primary key and the (user, device, room, type, transaction ID) key is the idempotency + // key, so a second row for either is refused without a commit. The same transaction ID on another device is a + // different key. The repeat of `d-1` is by another user, to show that the existing row is not overwritten. WHEN("a delay ID is stored twice, and the same transaction is stored under a second delay ID") { auto first = row("d-1"); @@ -412,6 +436,8 @@ SCENARIO("A delayed event is stored once, within its caps, and refused outside t } } + // 150 is above the default cap of 100 per user. The caller needs the count of scheduled rows to compute + // `Retry-After` (ADR-0826), so the cap is the caller's check and the store accepts all of them. WHEN("rows are stored for a user beyond any plausible per-user cap") { auto all = true; @@ -552,6 +578,11 @@ SCENARIO("A failed commit leaves the delayed-events mirror unchanged", } } +// `record_delayed_event_outcome` finalises a row after the send was attempted (ADR-0825). It is the one store function +// whose memory changes even when its commit fails, because the event is already sent and a row left scheduled would be +// sent again on every run of the job. The first WHEN proves that exception and the second proves that an outcome of the +// wrong shape is refused and writes nothing. They share one GIVEN because both start from scheduled rows that the job +// has marked as being sent. SCENARIO("The outcome of a send is recorded, and a failed commit still finalises memory", "[database][delayed-events][delayed-event-store]") { @@ -566,6 +597,8 @@ SCENARIO("The outcome of a send is recorded, and a failed commit still finalises REQUIRE(store.delayed_events.mark_being_sent("d-bad")); REQUIRE(store.delayed_events.mark_being_sent("d-lost")); + // `force_next_persist_failures` makes the next commit fail with no write. The transaction count after equals + // the count before, which shows the database changed nothing while memory still finalised the row. WHEN("a success, a failure and a success whose commit fails are recorded") { auto const sent = @@ -597,6 +630,9 @@ SCENARIO("The outcome of a send is recorded, and a failed commit still finalises } } + // A sent outcome needs an event ID and no error, a failed outcome needs an error and no event ID, and no other + // status is an outcome, so each wrong shape is a separate call on the same scheduled row. The row is then + // finalised once, and a second outcome for it is refused, because an outcome is recorded once. WHEN("an outcome is recorded with the wrong shape, for an absent row and for a finalised row") { auto const no_event = @@ -621,6 +657,7 @@ SCENARIO("The outcome of a send is recorded, and a failed commit still finalises DelayedEventOutcomeResult::recorded); auto const twice = db::record_delayed_event_outcome(store, "d-ok", DelayedEventStatus::sent, 2U, "$f", {}, 10U); + // The error exactly at its cap (1,024 bytes) is the one accepted case, against `long_error` one byte over. auto const at_cap = db::record_delayed_event_outcome(store, "d-bad", DelayedEventStatus::failed, 1U, {}, std::string(db::max_delayed_event_error_bytes, 'e'), 10U); @@ -643,6 +680,12 @@ SCENARIO("The outcome of a send is recorded, and a failed commit still finalises } } +// A finalised row is kept for the spec's idempotent answers, bounded per user by `max_finalised_per_user` (ADR-0824). +// The row being finalised is still scheduled when the bound is judged, so a user at the bound loses the oldest +// finalised row to make room for it. The eviction must be in the same transaction as the status update, so a crash +// cannot leave the user over the bound or a row finalised without its eviction. The captured statements and the +// transaction counts prove that. The GIVENs differ in size: a small one for the exact arithmetic and one with 300 rows +// for the batching of the DELETE. SCENARIO("Finalising keeps at most the bound of a user's finalised rows, evicting the oldest in the same transaction", "[database][delayed-events][delayed-event-store]") { @@ -656,6 +699,8 @@ SCENARIO("Finalising keeps at most the bound of a user's finalised rows, evictin REQUIRE(db::store_delayed_event(store, row("live-2"))); db::enable_statement_capture(store, 16U); + // Three finalised rows and a bound of 3: finalising a fourth evicts exactly the oldest (`old-0`, finalised + // first), so the user ends at 3. The other user's row proves the eviction is per user. WHEN("a row is cancelled with a bound of 3") { auto const before = db::committed_transaction_count(store); @@ -711,6 +756,8 @@ SCENARIO("Finalising keeps at most the bound of a user's finalised rows, evictin } } + // The mirror changes only after the commit (ADR-0824), so a failed commit must leave both the status and the + // rows that would have been evicted as they were. WHEN("the evicting commit fails") { db::force_next_persist_failures(store, 1U); @@ -733,6 +780,8 @@ SCENARIO("Finalising keeps at most the bound of a user's finalised rows, evictin REQUIRE(db::store_delayed_event(store, row("live"))); db::enable_statement_capture(store, 16U); + // 300 held, 1 more, bound 100: 201 rows go and 99 old ones stay. A DELETE names at most 128 delay IDs, so 201 + // rows are two DELETE statements, and with the UPDATE that is three statements in one transaction. WHEN("one more row is cancelled with a bound of 100") { auto const before = db::committed_transaction_count(store); @@ -751,6 +800,9 @@ SCENARIO("Finalising keeps at most the bound of a user's finalised rows, evictin } } +// Deactivation deletes every row of the user, scheduled and finalised, in one statement (ADR-0824). The second call is +// for a user with no rows, which still runs its DELETE, because a row that hydration skipped is in the database and not +// in memory (the next scenario). SCENARIO("All of a user's delayed events are deleted in one statement", "[database][delayed-events][delayed-event-store]") { @@ -787,6 +839,9 @@ SCENARIO("All of a user's delayed events are deleted in one statement", } } +// A row the loader skipped is in the file and not in memory. If deletion asked memory first it would find nothing and +// skip the DELETE, and the row would outlive its account (ADR-0824). The row is written with raw SQL on purpose, +// because the store functions would also put it in memory. SCENARIO("Deactivation deletes a delayed event that exists only in the database", "[database][delayed-events][delayed-event-store][sqlite]") { @@ -816,6 +871,9 @@ SCENARIO("Deactivation deletes a delayed event that exists only in the database" } } +// The delayed-events job prunes finalised rows past the age bound, a bounded batch per run, oldest first (ADR-0824). +// This scenario proves the cutoff is exclusive, the batch limit is honoured, a scheduled row is never pruned, and a +// prune of nothing writes nothing. The rows are finalised at 100 to 109 so the cutoffs name exact rows. SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up to the row limit", "[database][delayed-events][delayed-event-store]") { @@ -825,10 +883,13 @@ SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up auto& store = opened.store; fill_finalised(store, "r", alice, 10U); fill_finalised(store, "early", bob, 1U); + // `early-0` was already cancelled by `fill_finalised`, so a second cancel is refused. It is finalised at 100, + // the same time as `r-0`, so a prune must treat equal times alike and take both in delay ID order. REQUIRE(db::cancel_delayed_event(store, "early-0", 1U, 100U) == false); REQUIRE(db::store_delayed_event(store, row("live"))); db::enable_statement_capture(store, 16U); + // Six rows are below 105 (`r-0` to `r-4` and `early-0`), and a limit of 3 takes the three oldest of them. WHEN("rows below 105 are pruned with a limit of 3") { auto const before = db::committed_transaction_count(store); @@ -860,6 +921,8 @@ SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up } } + // Both calls find nothing. The cutoff is exclusive, so a cutoff equal to the oldest finalised time deletes + // nothing, and neither call may open a transaction. WHEN("the limit is zero, or the cutoff is at the oldest row") { auto const before = db::committed_transaction_count(store); @@ -876,6 +939,7 @@ SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up } } + // 300 rows exceed the 128 IDs one DELETE may name, so the batching is exercised: 128, 128 and 44. GIVEN("300 finalised rows") { auto opened = opened_store(); @@ -899,6 +963,10 @@ SCENARIO("Pruning by age deletes the oldest finalised rows below the cutoff, up } } +// A delayed event must survive a restart, because a lost call leave leaves a vanished participant listed in the call +// (ADR-0824). This scenario writes rows of every status through the store functions into a file, opens the file again +// and compares every column and every index, so a column the writer or the loader forgets is a failure. The later WHENs +// prove the loader skips a row it cannot read exactly and that deletions reached the file, not just memory. SCENARIO("Delayed events survive a reopen of a SQLite store with every column and index", "[database][delayed-events][delayed-event-store][sqlite]") { @@ -917,10 +985,14 @@ SCENARIO("Delayed events survive a reopen of a SQLite store with every column an first.device_id = ""; first.via_appservice = true; first.txn_id = ""; + // The largest delay, an empty device ID, a state key and the appservice flag are set so that this row + // exercises the columns a plain message leaves at their defaults. first.delay_ms = db::max_delayed_event_delay_ms; first.delayed_since_ms = 123'456'789U; auto message = row("d-message"); message.txn_id = "txn-1"; + // Content with an escaped newline, escaped quotes and a non-ASCII letter proves the text comes back byte + // for byte through the column. message.content_json = R"({"body":"line\nbreak \"quoted\" é"})"; REQUIRE(db::store_delayed_event(opened.store, first)); REQUIRE(db::store_delayed_event(opened.store, message)); @@ -974,6 +1046,8 @@ SCENARIO("Delayed events survive a reopen of a SQLite store with every column an REQUIRE(table.scheduled_count(alice) == 2U); REQUIRE(table.finalised_count(alice) == 2U); REQUIRE(table.finalised_count(bob) == 1U); + // `d-message` is due at 61,000. `d-state` is due much later because its delay is the maximum, so the + // due order index must rebuild with `d-message` first. REQUIRE(table.earliest_due() == table.find("d-message")); REQUIRE(table.earliest_due_ms_of_user(alice) == std::optional{61'000U}); REQUIRE(table.find_by_transaction(alice, "DEVICE", "!room:example.org", "m.room.message", "txn-1") == @@ -983,6 +1057,8 @@ SCENARIO("Delayed events survive a reopen of a SQLite store with every column an } } + // 'done' is not a status the store writes. The loader skips the row and never repairs it (ADR-0824), and the + // file still holds it afterwards, since a repair would be a guess. WHEN("the file holds a row whose status text is not one of the four, and the store is opened") { { @@ -1006,6 +1082,8 @@ SCENARIO("Delayed events survive a reopen of a SQLite store with every column an } } + // A prune below 2001 takes `d-sent` (finalised at 2000) and not `d-cancelled` (2001), and the user deletion + // takes `d-failed`. Reopening shows that the file, and not only memory, lost them. WHEN("the rows are deleted by user and by prune and the store is opened again") { { diff --git a/tests/unit/test_delayed_event_table.cpp b/tests/unit/test_delayed_event_table.cpp index 1231589c0..cb7657473 100644 --- a/tests/unit/test_delayed_event_table.cpp +++ b/tests/unit/test_delayed_event_table.cpp @@ -500,6 +500,11 @@ SCENARIO("A second sent row claiming an event ID another row holds is refused", } } +// `DelayedEventTable` owns its rows and keeps every index right by construction (ADR-0824), so an erase that leaves a +// stale entry in one index is the failure to catch: a later lookup would return a pointer to a row that is gone. Each +// WHEN erases by a different route (one row, a user, the being-sent row, everything) and the THEN asks every index that +// could still name the row. The GIVEN holds scheduled, sent and cancelled rows for two users so that each index has an +// entry for the erased rows and for rows that must stay. SCENARIO("Erasing a row, or all of a user's rows, removes it from every index", "[database][delayed-events]") { GIVEN("two users with scheduled, sent and cancelled rows") @@ -513,8 +518,12 @@ SCENARIO("Erasing a row, or all of a user's rows, removes it from every index", REQUIRE(table.finalise("a-2", DelayedEventStatus::sent, 10U, "$sent-a", {})); REQUIRE(table.finalise("a-3", DelayedEventStatus::cancelled, 11U, {}, {})); REQUIRE(table.finalise("b-2", DelayedEventStatus::sent, 12U, "$sent-b", {})); + // The being-sent mark lives in a set of its own beside the row indexes, so the erase of a marked row is its own + // case. REQUIRE(table.mark_being_sent("b-1")); + // `a-2` is a sent row, so it sits in the user's finalised list and in the event ID index as well as the row + // map. WHEN("one row is erased") { auto const erased = table.erase("a-2"); @@ -532,6 +541,8 @@ SCENARIO("Erasing a row, or all of a user's rows, removes it from every index", } } + // The user holds a scheduled row with a transaction ID, a sent row and a cancelled row, one for each index that + // keys on something other than the delay ID. The other user's rows prove the erase is scoped to the user. WHEN("all of the first user's rows are erased") { auto const erased = table.erase_user("@a:example.org"); @@ -553,6 +564,8 @@ SCENARIO("Erasing a row, or all of a user's rows, removes it from every index", } } + // A mark left behind would hide a later row with the same delay ID from the due order, so the mark must go with + // the row. `a-1` is then the earliest due row because `b-1` is gone. WHEN("the row marked as being sent is erased") { REQUIRE(table.erase("b-1")); From c72562a015213a991f7b510ac9b2f4d7098ab8ec Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 19:19:53 +0000 Subject: [PATCH 28/29] Explain the skip rule and the row builder of the PostgreSQL reopen scenario Co-Authored-By: Claude Fable 5.2 --- tests/integration/test_postgresql_store_behaviour_flow.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_postgresql_store_behaviour_flow.cpp b/tests/integration/test_postgresql_store_behaviour_flow.cpp index 5116f203f..f16173cf0 100644 --- a/tests/integration/test_postgresql_store_behaviour_flow.cpp +++ b/tests/integration/test_postgresql_store_behaviour_flow.cpp @@ -1697,6 +1697,9 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every { GIVEN("a scheduled state event, a sent row, a cancelled row and a failed row of two users") { + // This scenario needs a live PostgreSQL server. With no URI configured it ends here with a message. That is one + // passing assertion and no test of the store, so only a run that sets the URI, as the PostgreSQL CI job does, + // proves anything about the loader. auto const uri = postgresql_uri(); if (uri.empty()) { @@ -1710,6 +1713,8 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every auto const alice = "@delayed-a-" + suffix + ":example.org"; auto const bob = "@delayed-b-" + suffix + ":example.org"; auto const room_id = "!delayed-" + suffix + ":example.org"; + // Every row here differs only in its ID and its owner, so one builder fills the other columns. A case that + // cares about a column sets it on the row the builder returns. auto const make = [&](std::string id, std::string const& user) { auto row = db::PersistentDelayedEvent{}; row.delay_id = "d-" + id + "-" + suffix; From adf9d8efe5fd87825f09fbdf09630fb034534dfd Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 19:39:35 +0000 Subject: [PATCH 29/29] Load delayed events outside the loader's chain and shorten the PostgreSQL reopen scenario The SQLite loader calls load_delayed_event_rows as a statement of its own before the chain of table loads, so this change adds no link to that chain (SonarQube cpp:S912). The PostgreSQL reopen scenario builds and writes its rows through two file-local helpers, which takes it under the 100 lines CodeQL's poorly-documented-function rule applies to. Co-Authored-By: Claude Fable 5.2 --- src/database/SqliteStore.cpp | 9 +- .../test_postgresql_store_behaviour_flow.cpp | 90 +++++++++++-------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/database/SqliteStore.cpp b/src/database/SqliteStore.cpp index 307f45964..031b3eceb 100644 --- a/src/database/SqliteStore.cpp +++ b/src/database/SqliteStore.cpp @@ -798,6 +798,13 @@ namespace auto load_persistent_rows(sqlite3& connection, PersistentStore& store, TableLoadProfile profile, std::vector& hydrated_events) -> bool { + // Migration 003 (ADR-0824): delayed events, loaded by the helper above as a statement of its own. The table + // depends on no other table and none depends on it, so it need not be a link of the chain below. A failed + // SELECT fails the whole load, as a failed link does. + if (!load_delayed_event_rows(connection, store, profile)) + { + return false; + } // Tables load in dependency order and the chain short-circuits: the first failed SELECT // fails the whole load, so a partially read store is never served. return load_rows(connection, "SELECT user_id, password_hash, locked, suspended, admin, deactivated FROM users", @@ -1219,8 +1226,6 @@ namespace column_text(row, 2), column_text(row, 3), parse_u64(column_text(row, 4))}); }) && - // Migration 003 (ADR-0824): delayed events, loaded by the helper above. - load_delayed_event_rows(connection, store, profile) && // Application service delivery state: transaction cursors, the outbox and the directory mapping. load_rows(connection, "SELECT appservice_id, next_txn_id, delivered_stream_ordering, pending_txn_id, " diff --git a/tests/integration/test_postgresql_store_behaviour_flow.cpp b/tests/integration/test_postgresql_store_behaviour_flow.cpp index f16173cf0..46dab422a 100644 --- a/tests/integration/test_postgresql_store_behaviour_flow.cpp +++ b/tests/integration/test_postgresql_store_behaviour_flow.cpp @@ -1687,6 +1687,57 @@ SCENARIO("PostgreSQL OAuth clients and sessions survive a reopen and a session g } } +namespace +{ + +// A scheduled message row with the ID "d--" for `user`. Every row of the reopen scenario below starts as +// this row, and a case that cares about another column sets it on the row returned. +[[nodiscard]] auto delayed_event_row(std::string const& suffix, std::string const& room_id, std::string const& id, + std::string const& user) -> db::PersistentDelayedEvent +{ + auto row = db::PersistentDelayedEvent{}; + row.delay_id = "d-" + id + "-" + suffix; + row.user_id = user; + row.device_id = "DEVICE"; + row.room_id = room_id; + row.event_type = "m.room.message"; + row.content_json = R"({"body":"two words \"quoted\""})"; + row.delay_ms = 60'000U; + row.delayed_since_ms = 1'000U; + return row; +} + +// Writes one row of each status through the store functions: a scheduled state event and a scheduled message with a +// transaction ID for `alice`, a sent and a cancelled row for `alice`, and a failed row for `bob`. The state event takes +// the largest delay and a large `delayed_since_ms`, so the numbers are proved to come back as the same values, and its +// state key and appservice flag fill the columns that a message leaves empty. +auto store_delayed_events_of_every_status(db::PersistentStore& store, std::string const& suffix, + std::string const& alice, std::string const& bob, + std::string const& room_id) -> void +{ + auto state = delayed_event_row(suffix, room_id, "state", alice); + state.is_state = true; + state.state_key = alice; + state.via_appservice = true; + state.delay_ms = db::max_delayed_event_delay_ms; + state.delayed_since_ms = 123'456'789U; + auto message = delayed_event_row(suffix, room_id, "message", alice); + message.txn_id = "txn-" + suffix; + REQUIRE(db::store_delayed_event(store, state)); + REQUIRE(db::store_delayed_event(store, message)); + REQUIRE(db::store_delayed_event(store, delayed_event_row(suffix, room_id, "sent", alice))); + REQUIRE(db::store_delayed_event(store, delayed_event_row(suffix, room_id, "cancelled", alice))); + REQUIRE(db::store_delayed_event(store, delayed_event_row(suffix, room_id, "failed", bob))); + REQUIRE(db::record_delayed_event_outcome(store, "d-sent-" + suffix, db::DelayedEventStatus::sent, 2000U, + "$sent-" + suffix, {}, 10U) == db::DelayedEventOutcomeResult::recorded); + REQUIRE(db::cancel_delayed_event(store, "d-cancelled-" + suffix, 2001U, 10U)); + REQUIRE(db::record_delayed_event_outcome(store, "d-failed-" + suffix, db::DelayedEventStatus::failed, 2002U, {}, + R"({"errcode":"M_FORBIDDEN","error":"nope"})", + 10U) == db::DelayedEventOutcomeResult::recorded); +} + +} // namespace + // A delayed event must survive a restart, or a call's scheduled leave is lost (ADR-0824). This is the PostgreSQL twin // of the SQLite reopen scenario in `test_delayed_event_store.cpp`: PostgreSQL has its own loader, and its own text // columns and bound parameters, so the same round trip must be proved on it. One row of each status is written through @@ -1713,42 +1764,9 @@ SCENARIO("PostgreSQL delayed events of every status survive a reopen with every auto const alice = "@delayed-a-" + suffix + ":example.org"; auto const bob = "@delayed-b-" + suffix + ":example.org"; auto const room_id = "!delayed-" + suffix + ":example.org"; - // Every row here differs only in its ID and its owner, so one builder fills the other columns. A case that - // cares about a column sets it on the row the builder returns. - auto const make = [&](std::string id, std::string const& user) { - auto row = db::PersistentDelayedEvent{}; - row.delay_id = "d-" + id + "-" + suffix; - row.user_id = user; - row.device_id = "DEVICE"; - row.room_id = room_id; - row.event_type = "m.room.message"; - row.content_json = R"({"body":"two words \"quoted\""})"; - row.delay_ms = 60'000U; - row.delayed_since_ms = 1'000U; - return row; - }; - auto state = make("state", alice); - state.is_state = true; - state.state_key = alice; - state.via_appservice = true; - // The largest delay and a large `delayed_since_ms` check that the numbers come back as the same values, and the - // state key and appservice flag fill the columns that a message leaves empty. - state.delay_ms = db::max_delayed_event_delay_ms; - state.delayed_since_ms = 123'456'789U; - auto message = make("message", alice); - message.txn_id = "txn-" + suffix; - REQUIRE(db::store_delayed_event(opened.store, state)); - REQUIRE(db::store_delayed_event(opened.store, message)); - REQUIRE(db::store_delayed_event(opened.store, make("sent", alice))); - REQUIRE(db::store_delayed_event(opened.store, make("cancelled", alice))); - REQUIRE(db::store_delayed_event(opened.store, make("failed", bob))); - REQUIRE(db::record_delayed_event_outcome(opened.store, "d-sent-" + suffix, db::DelayedEventStatus::sent, 2000U, - "$sent-" + suffix, {}, - 10U) == db::DelayedEventOutcomeResult::recorded); - REQUIRE(db::cancel_delayed_event(opened.store, "d-cancelled-" + suffix, 2001U, 10U)); - REQUIRE(db::record_delayed_event_outcome(opened.store, "d-failed-" + suffix, db::DelayedEventStatus::failed, - 2002U, {}, R"({"errcode":"M_FORBIDDEN","error":"nope"})", - 10U) == db::DelayedEventOutcomeResult::recorded); + // The rows are written by a helper so that this scenario reads as what it proves: the write, the reopen, then + // every column and index read back. + store_delayed_events_of_every_status(opened.store, suffix, alice, bob, room_id); // The first store is discarded before the second is opened, so every value below was read from the database and // none from the memory that wrote it.