From 1bd502b7f04b755f650a6e9148a8b372528927dd Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 19:24:36 +0000 Subject: [PATCH 01/34] Move this branch to 1.1.1, the build after the OAuth 2.0 API's 1.1.0 The OAuth 2.0 branch already uses 1.1.0, so this branch takes the next build number in every location of docs/versioning.md and opens a 1.1.1 changelog entry above it. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 19 +++++++++++++++++++ meson.build | 2 +- packaging/freebsd/+MANIFEST | 2 +- packaging/netbsd/Makefile | 2 +- packaging/opensuse/merovingian.spec | 5 ++++- packaging/rhel/merovingian.spec | 5 ++++- packaging/rpm/merovingian.spec | 5 ++++- scripts/build-deb.sh | 4 ++-- scripts/build-freebsd-pkg.sh | 4 ++-- scripts/build-netbsd-pkg.sh | 4 ++-- scripts/build-openbsd-pkg.sh | 4 ++-- scripts/build-opensuse-rpm.sh | 4 ++-- scripts/build-rhel-rpm.sh | 4 ++-- scripts/build-rpm.sh | 4 ++-- scripts/build-static-linux.sh | 2 +- src/db_migrate.cpp | 2 +- src/main.cpp | 2 +- 17 files changed, 51 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fda24d24..d09f152dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ Changes from 1.0.0, the first production release, onwards; the format is set out Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md), which is no longer added to ([ADR-0740](docs/adr/0740-the-changelog-starts-at-1-0-0-and-the-pre-release-entries-are-a-frozen-record.md)). +# Version 1.1.1 + +## Features + +* `GET /_matrix/client/v3/sync` honours `use_state_after` (Matrix v1.16): with it true each joined and left room + carries `state_after` and no `state` (ADR-0763, ADR-0764, ADR-0765). + +## Security Fixes + +* None. + +## Bug fixes + +* None. + +## Release notes + +1.1.1 adds `use_state_after` to `/sync`. No configuration change is needed. + # Version 1.1.0 ## Features diff --git a/meson.build b/meson.build index 7dc278ba2..90070cb6a 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'merovingian', 'cpp', 'c', - version: '1.1.0', + version: '1.1.1', license: 'GPL-3.0-or-later', meson_version: '>=1.1.0', default_options: [ diff --git a/packaging/freebsd/+MANIFEST b/packaging/freebsd/+MANIFEST index 869abd079..2ba83013d 100644 --- a/packaging/freebsd/+MANIFEST +++ b/packaging/freebsd/+MANIFEST @@ -1,5 +1,5 @@ name: "merovingian" -version: "1.1.0" +version: "1.1.1" origin: net/merovingian comment: Secure Matrix Protocol homeserver desc: "Merovingian is a Matrix Protocol homeserver." diff --git a/packaging/netbsd/Makefile b/packaging/netbsd/Makefile index d981516b7..ccdfd53dc 100644 --- a/packaging/netbsd/Makefile +++ b/packaging/netbsd/Makefile @@ -1,4 +1,4 @@ -DISTNAME= merovingian-1.1.0 +DISTNAME= merovingian-1.1.1 CATEGORIES= net MASTER_SITES= ${MASTER_SITE_GITHUB:=OMG-Software/} GITHUB_PROJECT= Merovingian diff --git a/packaging/opensuse/merovingian.spec b/packaging/opensuse/merovingian.spec index 73cd3136b..9cbf6bb40 100644 --- a/packaging/opensuse/merovingian.spec +++ b/packaging/opensuse/merovingian.spec @@ -1,5 +1,5 @@ Name: merovingian -Version: 1.1.0 +Version: 1.1.1 Release: 1%{?dist} Summary: Secure Matrix Protocol homeserver @@ -115,6 +115,9 @@ fi %{_sysconfdir}/merovingian/merovingian.conf.example %changelog +* Mon Nov 16 2026 James Chapman - 1.1.1-1 +- Merovingian 1.1.1 + * Sun Nov 15 2026 James Chapman - 1.1.0-1 - Merovingian 1.1.0 diff --git a/packaging/rhel/merovingian.spec b/packaging/rhel/merovingian.spec index 00f68058b..dcad8c654 100644 --- a/packaging/rhel/merovingian.spec +++ b/packaging/rhel/merovingian.spec @@ -1,5 +1,5 @@ Name: merovingian -Version: 1.1.0 +Version: 1.1.1 Release: 1%{?dist} Summary: Secure Matrix Protocol homeserver @@ -100,6 +100,9 @@ fi %{_sysconfdir}/merovingian/merovingian.conf.example %changelog +* Mon Nov 16 2026 James Chapman - 1.1.1-1 +- Merovingian 1.1.1 + * Sun Nov 15 2026 James Chapman - 1.1.0-1 - Merovingian 1.1.0 diff --git a/packaging/rpm/merovingian.spec b/packaging/rpm/merovingian.spec index cf30b3889..861d50dfe 100644 --- a/packaging/rpm/merovingian.spec +++ b/packaging/rpm/merovingian.spec @@ -1,5 +1,5 @@ Name: merovingian -Version: 1.1.0 +Version: 1.1.1 Release: 1%{?dist} Summary: Secure Matrix Protocol homeserver @@ -104,6 +104,9 @@ fi %{_sysconfdir}/merovingian/merovingian.conf.example %changelog +* Mon Nov 16 2026 James Chapman - 1.1.1-1 +- Merovingian 1.1.1 + * Sun Nov 15 2026 James Chapman - 1.1.0-1 - Merovingian 1.1.0 diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh index 2dfa97fb2..222b53984 100755 --- a/scripts/build-deb.sh +++ b/scripts/build-deb.sh @@ -1,9 +1,9 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.0. +# Build for merovingian 1.1.1. set -e -VERSION="1.1.0" +VERSION="1.1.1" PKG_NAME="merovingian" STAGING="staging-deb" diff --git a/scripts/build-freebsd-pkg.sh b/scripts/build-freebsd-pkg.sh index 4dce63ce2..0c4e6fd97 100755 --- a/scripts/build-freebsd-pkg.sh +++ b/scripts/build-freebsd-pkg.sh @@ -1,9 +1,9 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.0. +# Build for merovingian 1.1.1. set -e -VERSION="1.1.0" +VERSION="1.1.1" STAGING="staging-fbsd" # Clean any state (staged files, build dir) from cached FreeBSD VM runs. diff --git a/scripts/build-netbsd-pkg.sh b/scripts/build-netbsd-pkg.sh index 74a8fddc6..736a85ab9 100644 --- a/scripts/build-netbsd-pkg.sh +++ b/scripts/build-netbsd-pkg.sh @@ -1,6 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.0. +# Build for merovingian 1.1.1. # # The checked-in packaging/netbsd/Makefile is the pkgsrc recipe kept for # downstream porters. This script generates a framework-free package so CI @@ -11,7 +11,7 @@ # a staged tree using tar, which produces a package that pkg_add can install. set -e -VERSION="1.1.0" +VERSION="1.1.1" STAGE="staging-netbsd" PREFIX=/usr/pkg diff --git a/scripts/build-openbsd-pkg.sh b/scripts/build-openbsd-pkg.sh index 9a90fd6ba..4779ebb99 100644 --- a/scripts/build-openbsd-pkg.sh +++ b/scripts/build-openbsd-pkg.sh @@ -1,6 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build for merovingian 1.1.0. +# Build for merovingian 1.1.1. # # Standalone pkg_create(1), no ports tree. The checked-in packaging/openbsd/PLIST # is the ports-framework packing list kept for downstream porters; this script @@ -8,7 +8,7 @@ # produce and validate an installable package on every run. set -e -VERSION="1.1.0" +VERSION="1.1.1" STAGE="staging-openbsd" PREFIX=/usr/local diff --git a/scripts/build-opensuse-rpm.sh b/scripts/build-opensuse-rpm.sh index 2f31ed13a..c58981dbb 100644 --- a/scripts/build-opensuse-rpm.sh +++ b/scripts/build-opensuse-rpm.sh @@ -1,6 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build an RPM package for merovingian 1.1.0 on OpenSUSE Tumbleweed or Leap. +# Build an RPM package for merovingian 1.1.1 on OpenSUSE Tumbleweed or Leap. # # Uses packaging/opensuse/merovingian.spec which adapts BuildRequires to # OpenSUSE package names (libopenssl-devel, postgresql-devel, libpng16-devel, @@ -12,7 +12,7 @@ # (.leap16, ADR-0650). set -e -VERSION="1.1.0" +VERSION="1.1.1" DIST="${MEROVINGIAN_RPM_DIST:-.tumbleweed}" case "${DIST}" in .[a-z]*) ;; diff --git a/scripts/build-rhel-rpm.sh b/scripts/build-rhel-rpm.sh index 816e839cd..b9fac8782 100644 --- a/scripts/build-rhel-rpm.sh +++ b/scripts/build-rhel-rpm.sh @@ -1,13 +1,13 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build an RPM package for merovingian 1.1.0 targeting RHEL 10 / AlmaLinux 10. +# Build an RPM package for merovingian 1.1.1 targeting RHEL 10 / AlmaLinux 10. # # Differs from build-rpm.sh only in the spec file used: packaging/rhel/merovingian.spec # omits catch-devel from BuildRequires (Catch2 is not reliably available in EPEL 10 # and is not needed when -Dbuild_tests=false). set -e -VERSION="1.1.0" +VERSION="1.1.1" mkdir -p "${HOME}/rpmbuild/BUILD" mkdir -p "${HOME}/rpmbuild/BUILDROOT" diff --git a/scripts/build-rpm.sh b/scripts/build-rpm.sh index 263ea16bb..5fd3f46ea 100755 --- a/scripts/build-rpm.sh +++ b/scripts/build-rpm.sh @@ -1,9 +1,9 @@ #!/bin/sh # SPDX-License-Identifier: GPL-3.0-or-later -# Build an RPM package for merovingian 1.1.0 using rpmbuild. +# Build an RPM package for merovingian 1.1.1 using rpmbuild. set -e -VERSION="1.1.0" +VERSION="1.1.1" # 1. Create rpmbuild directory tree mkdir -p "${HOME}/rpmbuild/BUILD" diff --git a/scripts/build-static-linux.sh b/scripts/build-static-linux.sh index 14141f61e..bacfb77c5 100755 --- a/scripts/build-static-linux.sh +++ b/scripts/build-static-linux.sh @@ -3,7 +3,7 @@ # Build a portable Linux fallback tarball with musl-linked static PIE binaries. set -eu -VERSION="${MEROVINGIAN_VERSION:-1.1.0}" +VERSION="${MEROVINGIAN_VERSION:-1.1.1}" BUILD_DIR="${BUILD_DIR:-build-static-linux}" STAGING="staging-static-linux" PACKAGE_ROOT="merovingian-${VERSION}-linux-static-x86_64" diff --git a/src/db_migrate.cpp b/src/db_migrate.cpp index d1d9cfcd7..2cb7fb5d9 100644 --- a/src/db_migrate.cpp +++ b/src/db_migrate.cpp @@ -23,7 +23,7 @@ namespace { -constexpr auto version = std::string_view{"1.1.0"}; +constexpr auto version = std::string_view{"1.1.1"}; auto print_help() -> void { diff --git a/src/main.cpp b/src/main.cpp index 6cd78e790..da3e7cfd6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -81,7 +81,7 @@ namespace { -constexpr auto version = std::string_view{"1.1.0"}; +constexpr auto version = std::string_view{"1.1.1"}; struct BootstrapConfigResult final { From 85cbe8a0f6bba1e741c330302e58c6d95d30a397 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 15:29:27 +0000 Subject: [PATCH 02/34] Test use_state_after and state_after on /sync before implementing them Ported from the unmerged feature/sync-use-state-after. The conformance file fails until the next commit; the use_state_after field and its query parsing come with it so the QueryParams scenario compiles. Co-Authored-By: Claude Sonnet 5.5 --- include/merovingian/core/SyncRequest.hpp | 2 + src/core/QueryParams.cpp | 4 + .../test_sync_state_after_conformance.cpp | 1052 +++++++++++++++++ tests/meson.build | 1 + tests/unit/test_query_params.cpp | 23 + 5 files changed, 1082 insertions(+) create mode 100644 tests/conformance/test_sync_state_after_conformance.cpp diff --git a/include/merovingian/core/SyncRequest.hpp b/include/merovingian/core/SyncRequest.hpp index 4817785b2..e6e35fe6c 100644 --- a/include/merovingian/core/SyncRequest.hpp +++ b/include/merovingian/core/SyncRequest.hpp @@ -17,6 +17,8 @@ struct SyncRequest final std::optional filter{}; // GET /sync `set_presence`, as sent (validated by the caller: it is client input). std::optional set_presence{}; + // GET /sync `use_state_after`: whether a room reports `state_after` rather than `state`. + std::optional use_state_after{}; }; } // namespace merovingian::core diff --git a/src/core/QueryParams.cpp b/src/core/QueryParams.cpp index f85b9d82a..178fb92bc 100644 --- a/src/core/QueryParams.cpp +++ b/src/core/QueryParams.cpp @@ -241,6 +241,10 @@ auto parse_query_params(std::string_view target) -> SyncRequest { request.full_state = (value == "true"); } + else if (key == "use_state_after") + { + request.use_state_after = (value == "true"); + } else if (key == "filter") { request.filter = std::string{value}; diff --git a/tests/conformance/test_sync_state_after_conformance.cpp b/tests/conformance/test_sync_state_after_conformance.cpp new file mode 100644 index 000000000..86c8a5b07 --- /dev/null +++ b/tests/conformance/test_sync_state_after_conformance.cpp @@ -0,0 +1,1052 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// GET /_matrix/client/v3/sync `use_state_after` and the `state_after` section of joined and left rooms +// (client-server-api.md, GET /sync, "Joined Room", "Left Room"; ADR-0763 to ADR-0765). +#include "../support/InMemoryDatabaseConfig.hpp" +#include "../support/JsonTestSupport.hpp" +#include "../support/MasterKey.hpp" +#include "../support/RegistrationToken.hpp" +#include "merovingian/config/ClientRateLimitsConfig.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ListenersConfig.hpp" +#include "merovingian/config/LogModulesConfig.hpp" +#include "merovingian/config/SecurityConfig.hpp" +#include "merovingian/config/ServerConfig.hpp" +#include "merovingian/core/QueryParams.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerStartResult.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +using namespace merovingian::tests; +using merovingian::canonicaljson::Object; +using merovingian::homeserver::ClientServerRuntime; + +constexpr auto k_state_after = std::string_view{"&use_state_after=true"}; +constexpr auto k_state_before = std::string_view{"&use_state_after=false"}; +constexpr auto k_no_filter = std::string_view{"{}"}; +constexpr auto k_one_event = std::string_view{R"({"room":{"timeline":{"limit":1}}})"}; +constexpr auto k_lazy = std::string_view{R"({"room":{"state":{"lazy_load_members":true}}})"}; +constexpr auto k_lazy_one_event = + std::string_view{R"({"room":{"timeline":{"limit":1},"state":{"lazy_load_members":true}}})"}; +constexpr auto k_include_leave = std::string_view{R"({"room":{"include_leave":true}})"}; + +[[nodiscard]] auto state_after_config() -> merovingian::config::Config +{ + auto security = merovingian::config::SecurityConfig{}; + security.secrets.master_key_file = merovingian::tests::shared_master_key_file(); + merovingian::tests::enable_token_registration(security); + return { + merovingian::config::ServerConfig{}, + merovingian::config::ListenersConfig{}, + merovingian::tests::in_memory_database_config(), + security, + merovingian::config::ClientRateLimitsConfig{}, + merovingian::config::LogModulesConfig{}, + }; +} + +struct Reply final +{ + std::uint16_t status{0U}; + std::string body{}; +}; + +[[nodiscard]] auto call(ClientServerRuntime& rt, std::string method, std::string target, std::string const& token, + std::string body = {}) -> Reply +{ + auto const result = merovingian::homeserver::handle_client_server_request( + rt, {std::move(method), std::move(target), token, std::move(body)}, /*can_wait=*/false); + return {static_cast(result.response.status), result.response.body}; +} + +[[nodiscard]] auto user_id(std::string_view localpart) -> std::string +{ + return "@" + std::string{localpart} + ":example.org"; +} + +[[nodiscard]] auto register_user(ClientServerRuntime& rt, std::string const& localpart) -> std::string +{ + auto const registered = + call(rt, "POST", "/_matrix/client/v3/register", {}, registration_json(localpart, "CorrectHorse7!")); + REQUIRE(registered.status == 200U); + return *string_member(parse_object(registered.body), "access_token"); +} + +[[nodiscard]] auto create_room(ClientServerRuntime& rt, std::string const& token, std::string const& preset) + -> std::string +{ + auto const created = call(rt, "POST", "/_matrix/client/v3/createRoom", token, R"({"preset":")" + preset + R"("})"); + REQUIRE(created.status == 200U); + return *string_member(parse_object(created.body), "room_id"); +} + +auto room_post(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& action, + std::string body = "{}") -> void +{ + REQUIRE(call(rt, "POST", "/_matrix/client/v3/rooms/" + room_id + "/" + action, token, std::move(body)).status == + 200U); +} + +auto say(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& text) -> void +{ + static auto counter = std::uint64_t{0U}; + auto const sent = + call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/send/m.room.message/sa" + std::to_string(++counter), + token, R"({"msgtype":"m.text","body":")" + text + R"("})"); + REQUIRE(sent.status == 200U); +} + +auto set_topic(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& topic) + -> std::string +{ + auto const sent = call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/state/m.room.topic/", token, + R"({"topic":")" + topic + R"("})"); + REQUIRE(sent.status == 200U); + return *string_member(parse_object(sent.body), "event_id"); +} + +auto set_display_name(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, + std::string const& localpart, std::string const& name) -> void +{ + auto const sent = + call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/state/m.room.member/" + user_id(localpart), token, + R"({"membership":"join","displayname":")" + name + R"("})"); + REQUIRE(sent.status == 200U); +} + +struct Synced final +{ + Object body{}; + std::string next_batch{}; +}; + +[[nodiscard]] auto sync(ClientServerRuntime& rt, std::string const& token, std::string_view filter, + std::string const& since = {}, std::string_view extra = {}) -> Synced +{ + auto target = + std::string{"/_matrix/client/v3/sync?filter="} + merovingian::core::percent_encode_path_component(filter); + if (!since.empty()) + { + target += "&since=" + merovingian::core::percent_encode_path_component(since); + } + target += extra; + auto const reply = call(rt, "GET", target, token); + REQUIRE(reply.status == 200U); + auto body = parse_object(reply.body); + auto next = *string_member(body, "next_batch"); + return {std::move(body), std::move(next)}; +} + +// rooms.
., or nullptr. +[[nodiscard]] auto room_section(Synced const& synced, std::string_view section, std::string const& room_id) + -> Object const* +{ + auto const* rooms = object_member_as_object(synced.body, "rooms"); + auto const* group = rooms == nullptr ? nullptr : object_member_as_object(*rooms, section); + return group == nullptr ? nullptr : object_member_as_object(*group, room_id); +} + +[[nodiscard]] auto required_room(Synced const& synced, std::string_view section, std::string const& room_id) + -> Object const& +{ + auto const* room = room_section(synced, section, room_id); + REQUIRE(room != nullptr); + return *room; +} + +// Whether `room` has the member `part`, whatever it holds. +[[nodiscard]] auto has_part(Object const& room, std::string_view part) -> bool +{ + return object_member_as_object(room, part) != nullptr; +} + +// The events of `room.`, which must be present. +[[nodiscard]] auto events_of(Object const& room, std::string_view part) -> std::vector +{ + auto const* section = object_member_as_object(room, part); + REQUIRE(section != nullptr); + auto const* events = object_member_as_array(*section, "events"); + REQUIRE(events != nullptr); + auto out = std::vector{}; + for (auto const& entry : *events) + { + out.push_back(std::get(entry.storage())); + } + return out; +} + +using Keys = std::multiset; + +[[nodiscard]] auto event_types(std::vector const& events) -> Keys +{ + auto types = Keys{}; + for (auto const& event : events) + { + types.insert(*string_member(event, "type")); + } + return types; +} + +// The state_key of every m.room.member event. +[[nodiscard]] auto member_keys(std::vector const& events) -> Keys +{ + auto keys = Keys{}; + for (auto const& event : events) + { + if (*string_member(event, "type") == "m.room.member") + { + keys.insert(*string_member(event, "state_key")); + } + } + return keys; +} + +// `content.` of the event of (type, state_key), or nullopt when `events` has none. +[[nodiscard]] auto content_of(std::vector const& events, std::string_view type, std::string_view state_key, + std::string_view field) -> std::optional +{ + for (auto const& event : events) + { + auto const* key = string_member(event, "state_key"); + if (*string_member(event, "type") == type && key != nullptr && *key == state_key) + { + auto const* content = object_member_as_object(event, "content"); + auto const* value = content == nullptr ? nullptr : string_member(*content, field); + return value == nullptr ? std::optional{} : std::optional{*value}; + } + } + return std::nullopt; +} + +// The `content.topic` of every m.room.topic event in `events`. +[[nodiscard]] auto topics_in(std::vector const& events) -> std::vector +{ + auto topics = std::vector{}; + for (auto const& event : events) + { + if (*string_member(event, "type") == "m.room.topic") + { + auto const* content = object_member_as_object(event, "content"); + topics.push_back(*string_member(*content, "topic")); + } + } + return topics; +} + +auto require_state_after_only(Object const& room) -> void +{ + REQUIRE(has_part(room, "state_after")); + REQUIRE_FALSE(has_part(room, "state")); +} + +auto require_state_only(Object const& room) -> void +{ + REQUIRE(has_part(room, "state")); + REQUIRE_FALSE(has_part(room, "state_after")); +} + +} // namespace + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state`, +// `state_after` URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// "If this is set to true, servers MUST respond with the state between the previous sync and the end of the timeline in +// state_after and MUST omit state. If false, servers MUST respond with the state between the previous sync and the +// start of the timeline in state and MUST omit state_after." "By default, this is false." `state_after`: "all state up +// to the end of the timeline, if since is not given, or full_state is true". +SCENARIO("An initial sync answers state_after or state as use_state_after asks", + "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room whose topic changed inside the last two events") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + say(rt, alice, room, "a"); + set_topic(rt, alice, room, "second"); + say(rt, alice, room, "b"); + constexpr auto two_events = std::string_view{R"({"room":{"timeline":{"limit":2}}})"}; + + WHEN("alice syncs with use_state_after=true") + { + auto const synced = sync(rt, alice, two_events, {}, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after is the whole state at the end of the timeline, and state is omitted") + { + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(topics_in(after) == std::vector{"second"}); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(member_keys(after) == Keys{user_id("alice")}); + } + } + WHEN("alice syncs with use_state_after=false") + { + auto const synced = sync(rt, alice, two_events, {}, k_state_before); + auto const& joined = required_room(synced, "join", room); + + THEN("state is the state at the start of the timeline, and state_after is omitted") + { + require_state_only(joined); + REQUIRE(topics_in(events_of(joined, "state")) == std::vector{"first"}); + } + } + WHEN("alice syncs without the parameter") + { + auto const synced = sync(rt, alice, two_events); + auto const& joined = required_room(synced, "join", room); + + THEN("it is as if it were false") + { + require_state_only(joined); + REQUIRE(topics_in(events_of(joined, "state")) == std::vector{"first"}); + } + } + } +} + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state_after` +// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// `state_after`: "Updates to the state, between the time indicated by the since parameter, and the end of the +// timeline". "If this field is set, even if it is empty, clients MUST only update their local state with events in this +// list, and MUST NOT update their local state with events in timeline." "Required if use_state_after was set to true in +// the request, even if it is empty." +SCENARIO("An incremental sync's state_after holds what changed up to the end of the timeline", + "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room alice has synced") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + auto const initial = sync(rt, alice, k_no_filter, {}, k_state_after); + + WHEN("only a message arrived") + { + say(rt, alice, room, "hello"); + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after is present and empty, and state is omitted") + { + require_state_after_only(joined); + REQUIRE(events_of(joined, "state_after").empty()); + } + } + WHEN("only a message arrived and the parameter is false or absent") + { + say(rt, alice, room, "hello"); + auto const off = sync(rt, alice, k_no_filter, initial.next_batch, k_state_before); + auto const absent = sync(rt, alice, k_no_filter, initial.next_batch); + + THEN("state is present and state_after is omitted") + { + require_state_only(required_room(off, "join", room)); + require_state_only(required_room(absent, "join", room)); + } + } + WHEN("the timeline carries a topic change") + { + say(rt, alice, room, "before"); + set_topic(rt, alice, room, "second"); + say(rt, alice, room, "after"); + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after repeats the change, as the client takes no state from the timeline, and nothing else") + { + require_state_after_only(joined); + REQUIRE(topics_in(events_of(joined, "timeline")) == std::vector{"second"}); + auto const after = events_of(joined, "state_after"); + REQUIRE(event_types(after) == Keys{"m.room.topic"}); + REQUIRE(topics_in(after) == std::vector{"second"}); + } + } + WHEN("the topic changed twice since the last sync") + { + set_topic(rt, alice, room, "second"); + set_topic(rt, alice, room, "third"); + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + + THEN("state_after holds the value at the end of the timeline, once") + { + REQUIRE(topics_in(events_of(required_room(synced, "join", room), "state_after")) == + std::vector{"third"}); + } + } + WHEN("a topic change falls in the gap of a limited sync") + { + set_topic(rt, alice, room, "second"); + say(rt, alice, room, "one"); + say(rt, alice, room, "two"); + auto const synced = sync(rt, alice, k_one_event, initial.next_batch, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after holds the change and not the state that did not change") + { + auto const* timeline = object_member_as_object(joined, "timeline"); + REQUIRE(timeline != nullptr); + REQUIRE(events_of(joined, "timeline").size() == 1U); + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(event_types(after) == Keys{"m.room.topic"}); + REQUIRE(topics_in(after) == std::vector{"second"}); + } + } + WHEN("the timeline filter hides a topic change") + { + constexpr auto hide_topic = std::string_view{R"({"room":{"timeline":{"not_types":["m.room.topic"]}}})"}; + set_topic(rt, alice, room, "hidden"); + auto const synced = sync(rt, alice, hide_topic, initial.next_batch, k_state_after); + + THEN("state_after still reports it, because the next sync starts after it") + { + auto const& joined = required_room(synced, "join", room); + REQUIRE(events_of(joined, "timeline").empty()); + REQUIRE(topics_in(events_of(joined, "state_after")) == std::vector{"hidden"}); + auto const next = sync(rt, alice, hide_topic, synced.next_batch, k_state_after); + auto const* again = room_section(next, "join", room); + REQUIRE((again == nullptr || events_of(*again, "state_after").empty())); + } + } + WHEN("the state filter excludes the type that changed") + { + constexpr auto no_topic = std::string_view{R"({"room":{"state":{"not_types":["m.room.topic"]}}})"}; + set_topic(rt, alice, room, "second"); + auto const synced = sync(rt, alice, no_topic, initial.next_batch, k_state_after); + + THEN("state_after is present and does not hold it") + { + auto const& joined = required_room(synced, "join", room); + require_state_after_only(joined); + REQUIRE(events_of(joined, "state_after").empty()); + } + } + WHEN("nothing happened") + { + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + + THEN("the room is not reported") + { + REQUIRE(room_section(synced, "join", room) == nullptr); + } + } + WHEN("more state changes than one delta chain can hold happened") + { + for (auto index = 0; index < 120; ++index) + { + set_topic(rt, alice, room, "t" + std::to_string(index)); + } + auto const synced = sync(rt, alice, k_one_event, initial.next_batch, k_state_after); + + THEN("state_after still holds only the key that changed, with its last value") + { + auto const after = events_of(required_room(synced, "join", room), "state_after"); + REQUIRE(event_types(after) == Keys{"m.room.topic"}); + REQUIRE(topics_in(after) == std::vector{"t119"}); + } + } + } +} + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state_after` +// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// `use_state_after`: "the state between the previous sync and the end of the timeline". State resolution can move a key +// back to an older event: no event of the window is about that key, and only comparing the two states shows the change. +SCENARIO("state_after reports a key that state resolution moved back to an older event", + "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room with a forked graph whose resolved topic went back to an earlier event") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + auto const old_topic = set_topic(rt, alice, room, "old"); + set_topic(rt, alice, room, "new"); + auto const initial = sync(rt, alice, k_no_filter, {}, k_state_after); + say(rt, alice, room, "the event that arrived with the fork"); + + auto& store = rt.homeserver.database.persistent_store; + store.forward_extremities.push_back({room, old_topic}); + for (auto& row : store.state) + { + if (row.room_id == room && row.event_type == "m.room.topic") + { + row.event_id = old_topic; + } + } + merovingian::database::rebuild_state_index(store); + + WHEN("alice syncs with use_state_after=true") + { + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + + THEN("state_after holds the older event, and the unchanged state stays out") + { + auto const after = events_of(required_room(synced, "join", room), "state_after"); + REQUIRE(topics_in(after) == std::vector{"old"}); + REQUIRE_FALSE(event_types(after).contains("m.room.create")); + } + } + } +} + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state_after` +// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// `full_state`: "If this is set to true, then all state events will be returned, even if since is non-empty." +// `state_after`: "all state up to the end of the timeline, if since is not given, or full_state is true". +SCENARIO("full_state=true returns the whole state in the section use_state_after names", + "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room alice has synced, whose topic changed since") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + auto const initial = sync(rt, alice, k_no_filter); + set_topic(rt, alice, room, "second"); + + WHEN("alice asks for the full state with use_state_after=true") + { + auto const extra = std::string{"&full_state=true"} + std::string{k_state_after}; + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, extra); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after holds all state, the timeline's change included, and state is omitted") + { + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(topics_in(after) == std::vector{"second"}); + } + } + WHEN("alice asks for the full state with use_state_after=false") + { + auto const extra = std::string{"&full_state=true"} + std::string{k_state_before}; + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, extra); + auto const& joined = required_room(synced, "join", room); + + THEN("state holds all state up to the start of the timeline, and state_after is omitted") + { + require_state_only(joined); + auto const state = events_of(joined, "state"); + REQUIRE(event_types(state).contains("m.room.create")); + REQUIRE(topics_in(state) == std::vector{"first"}); + } + } + WHEN("nothing changed and alice asks for the full state with use_state_after=true") + { + auto const caught_up = sync(rt, alice, k_no_filter, initial.next_batch); + auto const extra = std::string{"&full_state=true"} + std::string{k_state_after}; + auto const synced = sync(rt, alice, k_no_filter, caught_up.next_batch, extra); + + THEN("the room is still reported, with all state") + { + auto const& joined = required_room(synced, "join", room); + require_state_after_only(joined); + REQUIRE(event_types(events_of(joined, "state_after")).contains("m.room.create")); + } + } + } +} + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state_after` +// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// A room the client had no state for: it joined inside the window, so a delta would tell it nothing. +SCENARIO("state_after of a room joined in the window is the room's whole state", + "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room bob joins after his first sync") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const bob = register_user(rt, "bob"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + auto const initial = sync(rt, bob, k_no_filter, {}, k_state_after); + room_post(rt, bob, room, "join"); + + WHEN("bob syncs with use_state_after=true") + { + auto const synced = sync(rt, bob, k_no_filter, initial.next_batch, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after holds all state, his own join included, and state is omitted") + { + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(topics_in(after) == std::vector{"first"}); + REQUIRE(content_of(after, "m.room.member", user_id("bob"), "membership") == "join"); + } + } + } +} + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, Left Room +// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// Left Room `state`: "The state updates for the room up to the start of the timeline. MUST be omitted if +// use_state_after was set to true in the request." `state_after`: "The state updates for the room up to the end of the +// timeline. If this field is set, even if it is empty, clients MUST only update their local state with events in this +// list, and MUST NOT update their local state with events in timeline." Room History Visibility: "After a user has left +// a room, they may see any events which they were allowed to see before they left the room, but no events received +// after they left." +SCENARIO("A left room answers state_after or state as use_state_after asks", + "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room dave joined and synced, and whose topic changed before he left") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + room_post(rt, dave, room, "join"); + auto const synced_before = sync(rt, dave, k_no_filter, {}, k_state_after); + set_topic(rt, alice, room, "changed before the leave"); + room_post(rt, dave, room, "leave"); + set_topic(rt, alice, room, "changed after the leave"); + + WHEN("dave makes an initial sync with include_leave and use_state_after=true") + { + auto const synced = sync(rt, dave, k_include_leave, {}, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after is the state up to his leave, the leave included, and state is omitted") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + REQUIRE(topics_in(after) == std::vector{"changed before the leave"}); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(member_keys(after).count(user_id("dave")) == 1U); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + } + } + WHEN("dave makes an initial sync with include_leave, without the parameter and with it false") + { + auto const absent = sync(rt, dave, k_include_leave); + auto const off = sync(rt, dave, k_include_leave, {}, k_state_before); + + THEN("state is present and state_after is omitted") + { + require_state_only(required_room(absent, "leave", room)); + require_state_only(required_room(off, "leave", room)); + } + } + WHEN("dave makes an incremental sync with use_state_after=true") + { + auto const synced = sync(rt, dave, k_no_filter, synced_before.next_batch, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after holds what changed up to his leave and the leave, though the timeline carries it") + { + require_state_after_only(left); + REQUIRE(content_of(events_of(left, "timeline"), "m.room.member", user_id("dave"), "membership") == + "leave"); + auto const after = events_of(left, "state_after"); + REQUIRE(topics_in(after) == std::vector{"changed before the leave"}); + REQUIRE(event_types(after) == Keys{"m.room.member", "m.room.topic"}); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + } + } + WHEN("dave makes an incremental sync without the parameter") + { + auto const synced = sync(rt, dave, k_no_filter, synced_before.next_batch); + + THEN("state is present and state_after is omitted") + { + require_state_only(required_room(synced, "leave", room)); + } + } + WHEN("dave makes an incremental sync with full_state=true and use_state_after=true") + { + auto const extra = std::string{"&full_state=true"} + std::string{k_state_after}; + auto const synced = sync(rt, dave, k_no_filter, synced_before.next_batch, extra); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after is present and state is omitted; he is no longer a member, so it stays a delta") + { + require_state_after_only(left); + REQUIRE_FALSE(event_types(events_of(left, "state_after")).contains("m.room.create")); + } + } + WHEN("dave makes an incremental sync with full_state=true and use_state_after=false") + { + auto const extra = std::string{"&full_state=true"} + std::string{k_state_before}; + auto const synced = sync(rt, dave, k_no_filter, synced_before.next_batch, extra); + + THEN("state is present and state_after is omitted") + { + require_state_only(required_room(synced, "leave", room)); + } + } + } + GIVEN("a room dave left and synced, then was banned from after its topic changed") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, dave, room, "join"); + room_post(rt, dave, room, "leave"); + auto const after_leave = sync(rt, dave, k_no_filter, {}, k_state_after); + set_topic(rt, alice, room, "secret"); + room_post(rt, alice, room, "ban", R"({"user_id":")" + user_id("dave") + R"("})"); + + WHEN("dave syncs with use_state_after=true") + { + auto const synced = sync(rt, dave, k_no_filter, after_leave.next_batch, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after holds his ban and nothing of what happened after he left") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + REQUIRE(topics_in(after).empty()); + REQUIRE(event_types(after) == Keys{"m.room.member"}); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "ban"); + } + } + WHEN("dave makes an initial sync with include_leave and use_state_after=true") + { + auto const synced = sync(rt, dave, k_include_leave, {}, k_state_after); + auto const after = events_of(required_room(synced, "leave", room), "state_after"); + + THEN("state_after is the room as of his leave, with his own membership the ban") + { + REQUIRE(topics_in(after).empty()); + REQUIRE(member_keys(after).count(user_id("dave")) == 1U); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "ban"); + } + } + } + GIVEN("a private room erin was invited to and declined, never having joined") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const erin = register_user(rt, "erin"); + auto const room = create_room(rt, alice, "private_chat"); + set_topic(rt, alice, room, "secret"); + room_post(rt, alice, room, "invite", R"({"user_id":")" + user_id("erin") + R"("})"); + room_post(rt, erin, room, "leave"); + + WHEN("erin makes an initial sync with include_leave and use_state_after=true") + { + auto const synced = sync(rt, erin, k_include_leave, {}, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after holds only her own leave, as she never could read the room") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + REQUIRE(event_types(after) == Keys{"m.room.member"}); + REQUIRE(content_of(after, "m.room.member", user_id("erin"), "membership") == "leave"); + } + } + } +} + +// Spec: Matrix Client-Server API v1.19 +// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state_after` +// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync +// +// `state_after`: "State updates for m.room.member events will be incomplete if lazy_load_members is enabled in the +// /sync filter, and only return the member events required to display the senders of the timeline events in this +// response." Lazy-loading room members: "servers MUST include the syncing user's own membership event ... when the full +// state of rooms is requested". Redundancy follows ADR-0450; a timeline event is not state here, so ADR-0764 forces +// every change. +SCENARIO("Lazy-loading selects the members of state_after", "[conformance][client-server][sync][state-after]") +{ + GIVEN("a room of alice, bob and carol, in which bob and carol have spoken") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const bob = register_user(rt, "bob"); + auto const carol = register_user(rt, "carol"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, bob, room, "join"); + room_post(rt, carol, room, "join"); + set_topic(rt, alice, room, "first"); + say(rt, carol, room, "carol"); + say(rt, bob, room, "bob"); + + WHEN("alice makes an initial sync with lazy-loading and use_state_after=true") + { + auto const synced = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after keeps other state and only the sender's and her own membership") + { + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(topics_in(after) == std::vector{"first"}); + REQUIRE(member_keys(after) == Keys{user_id("alice"), user_id("bob")}); + } + } + WHEN("alice makes an initial sync with lazy-loading and use_state_after=false") + { + auto const synced = sync(rt, alice, k_lazy_one_event, {}, k_state_before); + auto const& joined = required_room(synced, "join", room); + + THEN("state holds those members, and state_after is omitted") + { + require_state_only(joined); + REQUIRE(member_keys(events_of(joined, "state")) == Keys{user_id("alice"), user_id("bob")}); + } + } + WHEN("carol speaks in two incremental syncs, each acknowledging the last") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + say(rt, carol, room, "carol again"); + auto const second = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + say(rt, carol, room, "carol once more"); + auto const third = sync(rt, alice, k_lazy, second.next_batch, k_state_after); + + THEN("carol's membership comes once, then is redundant") + { + REQUIRE(member_keys(events_of(required_room(second, "join", room), "state_after")) == + Keys{user_id("carol")}); + auto const after = events_of(required_room(third, "join", room), "state_after"); + require_state_after_only(required_room(third, "join", room)); + REQUIRE(member_keys(after).empty()); + } + } + WHEN("the response that carried carol's membership is lost and the client retries") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + say(rt, carol, room, "carol again"); + auto const lost = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + REQUIRE(member_keys(events_of(required_room(lost, "join", room), "state_after")) == Keys{user_id("carol")}); + auto const retried = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + + THEN("it is sent again") + { + REQUIRE(member_keys(events_of(required_room(retried, "join", room), "state_after")) == + Keys{user_id("carol")}); + } + } + WHEN("bob speaks and then changes his display name in the same window") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + say(rt, bob, room, "still bob"); + set_display_name(rt, bob, room, "bob", "Bobby"); + auto const synced = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "join", room), "state_after"); + + THEN("state_after holds his membership as of the end of the timeline, once") + { + REQUIRE(member_keys(after) == Keys{user_id("bob")}); + REQUIRE(content_of(after, "m.room.member", user_id("bob"), "displayname") == "Bobby"); + } + } + WHEN("the timeline carries a membership change about someone who did not speak") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + room_post(rt, alice, room, "kick", R"({"user_id":")" + user_id("carol") + R"("})"); + auto const synced = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "join", room), "state_after"); + + THEN("state_after holds it, because the client takes no state from the timeline") + { + REQUIRE(content_of(after, "m.room.member", user_id("carol"), "membership") == "leave"); + } + } + WHEN("a membership change falls in the gap of a limited sync") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + room_post(rt, carol, room, "leave"); + say(rt, bob, room, "one"); + say(rt, bob, room, "two"); + auto const synced = sync(rt, alice, k_lazy_one_event, first.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "join", room), "state_after"); + + THEN("state_after holds it, and the state that did not change stays out") + { + REQUIRE(content_of(after, "m.room.member", user_id("carol"), "membership") == "leave"); + REQUIRE(topics_in(after).empty()); + } + } + WHEN("alice asks for the full state with lazy-loading and use_state_after=true") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + auto const acknowledged = sync(rt, alice, k_lazy_one_event, first.next_batch, k_state_after); + auto const extra = std::string{"&full_state=true"} + std::string{k_state_after}; + auto const synced = sync(rt, alice, k_lazy_one_event, acknowledged.next_batch, extra); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after holds her own membership and all other state") + { + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(member_keys(after).contains(user_id("alice"))); + REQUIRE_FALSE(member_keys(after).contains(user_id("carol"))); + } + } + WHEN("the client acknowledged bob, and bob leaves and rejoins") + { + auto const bob_first = sync(rt, bob, k_lazy, {}, k_state_after); + say(rt, alice, room, "alice"); + auto const bob_second = sync(rt, bob, k_lazy, bob_first.next_batch, k_state_after); + auto const bob_third = sync(rt, bob, k_lazy, bob_second.next_batch, k_state_after); + room_post(rt, bob, room, "leave"); + auto const bob_left = sync(rt, bob, k_lazy, bob_third.next_batch, k_state_after); + room_post(rt, bob, room, "join"); + say(rt, alice, room, "welcome back"); + auto const rejoined = sync(rt, bob, k_lazy, bob_left.next_batch, k_state_after); + auto const after = events_of(required_room(rejoined, "join", room), "state_after"); + + THEN("the tracker is not consulted, and his own join is in state_after though the timeline carries it") + { + REQUIRE(member_keys(after).contains(user_id("alice"))); + REQUIRE(content_of(after, "m.room.member", user_id("bob"), "membership") == "join"); + REQUIRE(event_types(after).contains("m.room.create")); + } + } + } + GIVEN("a room of alice and dave that dave leaves after bob spoke") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const bob = register_user(rt, "bob"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, bob, room, "join"); + room_post(rt, dave, room, "join"); + auto const before = sync(rt, dave, k_lazy_one_event, {}, k_state_after); + say(rt, bob, room, "bob"); + room_post(rt, dave, room, "leave"); + + WHEN("dave makes an incremental sync with lazy-loading and use_state_after=true") + { + auto const synced = sync(rt, dave, k_lazy, before.next_batch, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after holds his leave and the sender's membership") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + REQUIRE(member_keys(after) == Keys{user_id("bob"), user_id("dave")}); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + } + } + WHEN("dave makes an initial sync with include_leave, lazy-loading and use_state_after=true") + { + constexpr auto lazy = std::string_view{ + R"({"room":{"include_leave":true,"timeline":{"limit":1},"state":{"lazy_load_members":true}}})"}; + auto const synced = sync(rt, dave, lazy, {}, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after keeps other state and holds only his own membership among members") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(member_keys(after) == Keys{user_id("dave")}); + } + } + } + GIVEN("a room dave joins, speaks in and leaves, all after his last sync") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + auto const before = sync(rt, dave, k_lazy, {}, k_state_after); + room_post(rt, dave, room, "join"); + say(rt, dave, room, "hello and goodbye"); + room_post(rt, dave, room, "leave"); + + WHEN("dave syncs with lazy-loading and use_state_after=true") + { + auto const synced = sync(rt, dave, k_lazy, before.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "leave", room), "state_after"); + + THEN("his own membership is there once, as the leave, so his earlier join cannot override it") + { + REQUIRE(member_keys(after).count(user_id("dave")) == 1U); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + } + } + } + GIVEN("a room dave left and synced, which erin then joined, and from which dave was then banned") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const erin = register_user(rt, "erin"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, dave, room, "join"); + room_post(rt, dave, room, "leave"); + auto const after_leave = sync(rt, dave, k_lazy_one_event, {}, k_state_after); + room_post(rt, erin, room, "join"); + say(rt, erin, room, "dave cannot know I am here"); + room_post(rt, alice, room, "ban", R"({"user_id":")" + user_id("dave") + R"("})"); + + WHEN("dave makes a limited incremental sync with lazy-loading and use_state_after=true") + { + auto const synced = sync(rt, dave, k_lazy_one_event, after_leave.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "leave", room), "state_after"); + + THEN("state_after holds his ban and does not show who joined after he left") + { + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "ban"); + REQUIRE_FALSE(member_keys(after).contains(user_id("erin"))); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index 90aa1fca2..32c6bfa11 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -709,6 +709,7 @@ shell = find_program('sh') 'conformance/test_sync_archive_conformance.cpp', 'conformance/test_sync_state_conformance.cpp', 'conformance/test_lazy_load_members_conformance.cpp', + 'conformance/test_sync_state_after_conformance.cpp', 'conformance/test_identifier_grammar.cpp', 'conformance/test_ignoring_users_conformance.cpp', 'conformance/test_client_txn_scope_conformance.cpp', diff --git a/tests/unit/test_query_params.cpp b/tests/unit/test_query_params.cpp index f3a21ccea..df1b7d5c4 100644 --- a/tests/unit/test_query_params.cpp +++ b/tests/unit/test_query_params.cpp @@ -5,6 +5,7 @@ #include +#include #include SCENARIO("Query parameter parser extracts key-value pairs from URL query strings", "[core][http][query]") @@ -276,3 +277,25 @@ SCENARIO("Sync timeout query parameter rejects overflowing values", "[core][quer } } } + +SCENARIO("The sync use_state_after parameter is read as sent and absent when not sent", "[core][query]") +{ + GIVEN("sync targets with use_state_after true, false, another value, and none") + { + WHEN("the query parameters are parsed") + { + auto const enabled = merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=true"); + auto const disabled = merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=false"); + auto const other = merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=1"); + auto const absent = merovingian::core::parse_query_params("/_matrix/client/v3/sync"); + + THEN("true turns it on, anything else leaves it off, and an absent one stays absent") + { + REQUIRE(enabled.use_state_after == std::optional{true}); + REQUIRE(disabled.use_state_after == std::optional{false}); + REQUIRE(other.use_state_after == std::optional{false}); + REQUIRE_FALSE(absent.use_state_after.has_value()); + } + } + } +} From 5eefc8e08935bab74ca791a1ea2cd52f928ab7bc Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 15:31:19 +0000 Subject: [PATCH 03/34] Answer state_after on /sync when use_state_after is true Joined and left rooms report the state up to the end of the timeline in state_after and omit state. The delta compares the two states, through a state group chain walk when the graph is not forked, so a key state resolution moved back is reported and a window of messages reads nothing. Co-Authored-By: Claude Sonnet 5.5 --- include/merovingian/sync/StateGroupLookup.hpp | 11 + src/homeserver/ClientServerRuntime.cpp | 367 ++++++++++++++++-- src/sync/StateGroupLookup.cpp | 31 ++ 3 files changed, 375 insertions(+), 34 deletions(-) diff --git a/include/merovingian/sync/StateGroupLookup.hpp b/include/merovingian/sync/StateGroupLookup.hpp index 2de037fce..b1f925706 100644 --- a/include/merovingian/sync/StateGroupLookup.hpp +++ b/include/merovingian/sync/StateGroupLookup.hpp @@ -5,7 +5,10 @@ #include "merovingian/database/PersistentStore.hpp" #include "merovingian/sync/GroupLookupResult.hpp" +#include +#include #include +#include namespace merovingian::sync { @@ -17,4 +20,12 @@ namespace merovingian::sync [[nodiscard]] auto lookup_state_in_group(database::PersistentStore const& store, std::string_view group_id, std::string_view event_type, std::string_view state_key) -> GroupLookupResult; +// The event IDs of the state `group_id` holds that differs from what its ancestor `ancestor_id` holds: the newest row +// of each key on the delta chain from `group_id` down to, not including, `ancestor_id`. Empty when the two are one +// group. Nullopt when `ancestor_id` is not on the chain (a snapshot, a missing group or a cycle comes first), and the +// caller compares the two states. It costs the rows of the groups between the two, not the room's state: a snapshot's +// rows are never read. A key that changed and changed back is reported, with the value it holds. +[[nodiscard]] auto state_changes_along_chain(database::PersistentStore const& store, std::string_view ancestor_id, + std::string_view group_id) -> std::optional>; + } // namespace merovingian::sync diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index c5f3b11a0..0201373bd 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -5428,8 +5428,12 @@ namespace std::set> senders{}; std::string own_user_id{}; bool own_forced{false}; - // A member event above this stream ordering (and below the timeline) is a gap change. + // A member event above this stream ordering is a change since `since`: in the gap when `state` is built, + // and in the timeline as well for `state_after`, whose client takes no state from the timeline. std::optional gap_after{}; + // The events of a `state_after` delta (ADR-0764): each is a change the client must apply, whatever its + // stream ordering (state resolution can move a key back to an older event). + std::unordered_set changed_event_ids{}; // Drop every state event that is not a member event (an incremental left room's state is a delta). bool members_only{false}; // Do not consult the tracker: a client may discard a room when it leaves it, so after a rejoin it may no @@ -5455,6 +5459,11 @@ namespace return plan_.members_only; } + [[nodiscard]] auto is_changed(std::string const& event_id) const -> bool + { + return plan_.changed_event_ids.contains(event_id); + } + // Whether `event`, the member state event for `state_key`, goes in `state`; notes it when it does. [[nodiscard]] auto keep(database::PersistentEvent const& event, std::string_view state_key) -> bool { @@ -5467,7 +5476,7 @@ namespace return false; } auto const forced = (plan_.gap_after.has_value() && event.stream_ordering > *plan_.gap_after) || - (plan_.own_forced && state_key == plan_.own_user_id); + (plan_.own_forced && state_key == plan_.own_user_id) || is_changed(event.event_id); if (!forced) { if (!plan_.senders.contains(state_key) || @@ -5753,10 +5762,8 @@ namespace // `position` only while an earlier ordering is pending; then the state is the // state before the earliest such event, which no event at or below `position` // follows. Failing that, the events above `position` are left out. - [[nodiscard]] auto build_state_events_at_position(database::PersistentStore const& store, - sync::EventTypeFilter const& filter, std::string_view room_id, - std::uint64_t position, LazySelectorRef lazy = std::nullopt) - -> canonicaljson::Array + [[nodiscard]] auto state_event_ids_at_position(database::PersistentStore const& store, std::string_view room_id, + std::uint64_t position) -> std::vector { auto earliest_above = store.events.end(); auto ids = std::vector{}; @@ -5776,7 +5783,7 @@ namespace }); if (earliest_above == store.events.end()) { - return build_current_state_events_array(store, filter, room_id, lazy); + return ids; } if (auto const* policy = stored_room_version_policy(store, room_id); policy != nullptr) { @@ -5788,7 +5795,7 @@ namespace { before_ids.push_back(entry.event_id); } - return build_state_events_array_for_ids(store, filter, before_ids, lazy); + return before_ids; } } // Each key held above `position` keeps the value it held before, where its @@ -5804,7 +5811,80 @@ namespace } } } - return build_state_events_array_for_ids(store, filter, ids, lazy); + return ids; + } + + [[nodiscard]] auto build_state_events_at_position(database::PersistentStore const& store, + sync::EventTypeFilter const& filter, std::string_view room_id, + std::uint64_t position, LazySelectorRef lazy = std::nullopt) + -> canonicaljson::Array + { + return build_state_events_array_for_ids(store, filter, state_event_ids_at_position(store, room_id, position), + lazy); + } + + // Removes from `ids` what `held`, the state the client holds, already has. An event is the value of one key, so an + // event `held` lacks is exactly a key whose value differs. + auto remove_state_held(std::vector& ids, std::vector const& held) -> void + { + auto const held_ids = std::unordered_set{held.begin(), held.end()}; + std::erase_if(ids, [&](std::string const& id) { + return held_ids.contains(id); + }); + } + + // ADR-0764: the IDs of the state events that make a joined room's state at `position` differ from its state at + // `since_ordering`, for `state_after`. A key can change with no event of the window about it (state resolution can + // move it back), so this compares the two states; it does not list the window's state events. + // + // Cost: nothing for a room with no event since `since`. With an unforked graph, the state group rows between the + // two points, none for a window of messages. Otherwise (a fork, a snapshot between the points, an unreadable chain) + // it reads the room's state twice, its own rows only. + [[nodiscard]] auto joined_state_changes(database::PersistentStore const& store, std::string_view room_id, + std::uint64_t since_ordering, std::uint64_t position) + -> std::vector + { + auto newest_first = store.events.room_in_stream_order(room_id) | std::views::reverse; + if (std::ranges::empty(newest_first) || (*newest_first.begin()).stream_ordering <= since_ordering) + { + return {}; + } + auto const access = sync::RoomReadAccess{sync::RoomReadKind::current, {}, 0U}; + auto const* const since_anchor = sync::state_anchor_event(store, room_id, access, since_ordering); + auto const since_group = since_anchor == nullptr + ? std::optional{} + : database::find_event_state_group(store, since_anchor->event_id); + if (since_group.has_value()) + { + // With one extremity the current state is that event's recorded state (ADR-0064). The forward extremity + // index answers for one room: it costs that room's tips. + auto const tips = store.forward_extremities.tips_of_room(room_id); + if (tips.size() == 1U) + { + auto const tip = store.events.find(tips.front()); + auto const tip_group = tip != store.events.end() && tip->stream_ordering <= position + ? database::find_event_state_group(store, tip->event_id) + : std::optional{}; + if (tip_group.has_value()) + { + if (auto changes = sync::state_changes_along_chain(store, *since_group, *tip_group); + changes.has_value()) + { + return std::move(*changes); + } + } + } + } + auto changes = state_event_ids_at_position(store, room_id, position); + // A `since` older than anything recorded here gets all of it: an update the client already holds is harmless. + if (since_anchor != nullptr) + { + if (auto const held = sync::state_event_ids_after(store, *since_anchor); held.has_value()) + { + remove_state_held(changes, *held); + } + } + return changes; } // Whether a timeline carries the join of `user_id`: the join then tells the client everything the spec's "own @@ -5866,6 +5946,83 @@ namespace return current; } + // What stands in for a sender's member event above the position when the state group chain cannot be read: a joined + // user may read the current one and a member too many is valid; a user who left may not read past their leave. + enum class UnreadableMember + { + current_stands_in, + omitted, + }; + + // The `m.room.member` event of `sender` in force at `position`, for `state_after` (ADR-0454's cost rule): two hash + // lookups when their current member event is at or below it, which is nearly always; otherwise one walk of the + // state group chain of the state anchor at `position`. + [[nodiscard]] auto member_event_at_position(database::PersistentStore const& store, std::string_view room_id, + sync::RoomReadAccess const& access, std::uint64_t position, + std::string_view sender, UnreadableMember unreadable) + -> std::optional + { + auto const current = database::find_current_state_event_id(store, room_id, "m.room.member", sender); + if (current.has_value()) + { + auto const event = store.events.find(*current); + if (event != store.events.end() && event->stream_ordering <= position) + { + return current; + } + } + auto const fallback = unreadable == UnreadableMember::current_stands_in ? current : std::nullopt; + auto const* const anchor = sync::state_anchor_event(store, room_id, access, position); + auto const group = anchor == nullptr ? std::optional{} + : database::find_event_state_group(store, anchor->event_id); + if (!group.has_value()) + { + return fallback; + } + auto const found = sync::lookup_state_in_group(store, *group, "m.room.member", sender); + switch (found.outcome) + { + case sync::GroupLookup::found: + return std::string{found.event_id}; + case sync::GroupLookup::absent: + return std::nullopt; + case sync::GroupLookup::broken: + break; + } + return fallback; + } + + // Appends the client event of `event_id`, the member event of `sender`, when the filter and the selector keep it. + auto add_lazy_sender_member(database::PersistentStore const& store, sync::EventTypeFilter const& filter, + std::string const& event_id, std::string const& sender, LazyRoomSelector& selector, + canonicaljson::Array& members) -> void + { + auto const event = store.events.find(event_id); + if (event != store.events.end() && sync::event_passes_filter(filter, "m.room.member", event->sender_user_id) && + selector.keep(*event, sender)) + { + members.push_back(client_event_with_id(store, *event)); + } + } + + // The member events a `state_after` delta adds for `senders`: each as of `position`, unless the delta already holds + // it, that the selector keeps. + auto add_lazy_senders_at_position(database::PersistentStore const& store, sync::EventTypeFilter const& filter, + std::string_view room_id, sync::RoomReadAccess const& access, + std::uint64_t position, std::vector const& senders, + UnreadableMember unreadable, LazyRoomSelector& selector, + canonicaljson::Array& members) -> void + { + for (auto const& sender : senders) + { + auto const event_id = member_event_at_position(store, room_id, access, position, sender, unreadable); + if (event_id.has_value() && !selector.is_changed(*event_id)) + { + add_lazy_sender_member(store, filter, *event_id, sender, selector, members); + } + } + } + // The member events a lazy response carries when `state` is not otherwise built: those of `senders`, as they were // at the start of the timeline whose first event is `first_event`, that the selector keeps. [[nodiscard]] auto build_lazy_sender_members(database::PersistentStore const& store, @@ -5878,18 +6035,10 @@ namespace for (auto const& sender : senders) { auto const event_id = member_event_at_timeline_start(store, first_event.room_id, first_event, sender); - if (!event_id.has_value()) - { - continue; - } - auto const event = store.events.find(*event_id); - if (event == store.events.end() || - !sync::event_passes_filter(filter, "m.room.member", event->sender_user_id) || - !selector.keep(*event, sender)) + if (event_id.has_value()) { - continue; + add_lazy_sender_member(store, filter, *event_id, sender, selector, members); } - members.push_back(client_event_with_id(store, *event)); } return members; } @@ -6350,6 +6499,8 @@ namespace // What room state the user may read (ADR-0357). `none` for a user who never joined: a retracted // knock, a declined invite, a ban before any join. sync::RoomReadAccess access{}; + // GET /sync `use_state_after`: answer `state_after` instead of `state` (ADR-0765). + bool use_state_after{false}; }; // Ignoring Users for a stored event: whether `ignored_senders` withholds it. The event's JSON is parsed for @@ -6363,6 +6514,108 @@ namespace trust_safety::event_json_is_state_event(event.json)); } + // The members of a left room's section. + [[nodiscard]] auto left_room_object(std::string_view state_member, canonicaljson::Array state_events, + canonicaljson::Array timeline_events, bool limited, + std::string const& prev_batch) -> canonicaljson::Value + { + return json_obj({ + json_member(std::string{state_member}, + json_obj({json_member("events", json_arr(std::move(state_events)))})), + json_member("timeline", json_obj({ + json_member("events", json_arr(std::move(timeline_events))), + json_member("limited", json_bool(limited)), + json_member("prev_batch", json_str(prev_batch)), + })), + }); + } + + // ADR-0765: a left room's `state_after`: "The state updates for the room up to the end of the timeline", which ends + // at the leave. The room's state is read only as far as the user may read it (ADR-0357, room_read_access_for), so + // nothing after their leave is told, however late the ban that ended their membership came. The user's own + // membership is the event in `leave_event` (the leave, or the ban that followed it); the client takes no state from + // the timeline, so it is here even though the timeline carries it. + [[nodiscard]] auto build_left_room_state_after( + database::PersistentStore const& store, sync::EventTypeFilter const& filter, + sync::ViewerEventUnsigned& viewer_unsigned, LeftRoomRequest const& request, + std::optional> leave_event, std::uint64_t upper_bound, + std::vector const& senders, LazySyncRequest& lazy) -> canonicaljson::Array + { + auto state_events = canonicaljson::Array{}; + if (request.access.kind != sync::RoomReadKind::none) + { + auto const readable_to = request.access.kind == sync::RoomReadKind::as_of + ? std::min(upper_bound, request.access.boundary_ordering) + : upper_bound; + auto const* const anchor = sync::state_anchor_event(store, request.room_id, request.access, readable_to); + auto ids = anchor == nullptr ? std::optional>{} + : sync::state_event_ids_after(store, *anchor); + if (ids.has_value() && leave_event.has_value()) + { + // Replaced by the event that ended their stay, below. + if (auto const group = database::find_event_state_group(store, anchor->event_id); group.has_value()) + { + auto const own = sync::lookup_state_in_group(store, *group, "m.room.member", request.user_id); + if (own.outcome == sync::GroupLookup::found) + { + std::erase(*ids, own.event_id); + } + } + } + if (ids.has_value() && request.since_ordering.has_value()) + { + if (*request.since_ordering >= readable_to) + { + ids->clear(); + } + else if (auto const* const since_anchor = + sync::state_anchor_event(store, request.room_id, request.access, *request.since_ordering); + since_anchor != nullptr) + { + if (auto const held = sync::state_event_ids_after(store, *since_anchor); held.has_value()) + { + remove_state_held(*ids, *held); + } + } + } + auto selector = std::optional{}; + if (lazy.enabled()) + { + auto plan = LazyRoomSelector::Plan{}; + plan.senders.insert(senders.begin(), senders.end()); + plan.own_user_id = std::string{request.user_id}; + plan.own_forced = true; + if (request.since_ordering.has_value() && ids.has_value()) + { + plan.changed_event_ids.insert(ids->begin(), ids->end()); + } + selector.emplace(lazy, request.room_id, std::move(plan)); + } + auto const lazy_ref = selector.has_value() ? LazySelectorRef{std::ref(*selector)} : LazySelectorRef{}; + if (ids.has_value()) + { + state_events = build_state_events_array_for_ids(store, filter, *ids, lazy_ref); + } + if (selector.has_value() && request.since_ordering.has_value()) + { + // A delta names only what changed: the other senders' members come by key, as of the leave. + auto others = std::vector{}; + std::ranges::copy_if(senders, std::back_inserter(others), [&](std::string const& sender) { + return sender != request.user_id; + }); + add_lazy_senders_at_position(store, filter, request.room_id, request.access, readable_to, others, + UnreadableMember::omitted, *selector, state_events); + } + } + if (leave_event.has_value() && + sync::event_passes_filter(filter, "m.room.member", leave_event->get().sender_user_id)) + { + state_events.push_back( + viewer_unsigned.apply(leave_event->get(), client_event_with_id(store, leave_event->get()))); + } + return state_events; + } + [[nodiscard]] auto build_left_room(database::PersistentStore const& store, sync::SyncFilter const& filter, sync::HistoryVisibility& history_visibility, sync::ViewerEventUnsigned& viewer_unsigned, @@ -6417,6 +6670,27 @@ namespace : std::to_string(window.budget_ran_out ? window.oldest_examined.value_or(since_ordering) : since_ordering); + if (request.use_state_after) + { + auto senders = std::vector{}; + if (lazy.enabled()) + { + auto unique = std::set>{}; + for (auto it = window_begin; it != matched.end(); ++it) + { + unique.insert((*it)->sender_user_id); + } + senders.assign(unique.begin(), unique.end()); + } + auto const leave = leave_event == store.events.end() + ? std::optional>{} + : std::optional{std::cref(*leave_event)}; + return left_room_object("state_after", + build_left_room_state_after(store, filter.room.state, viewer_unsigned, request, + leave, upper_bound, senders, lazy), + std::move(timeline_events), limited, prev_batch); + } + auto state_events = canonicaljson::Array{}; // ADR-0357: a user who never joined (`none`) was never entitled to any of the room's state, so an // initial sync tells them only what an incremental one does: their own leave or ban, when the @@ -6552,14 +6826,7 @@ namespace } } - return json_obj({ - json_member("state", json_obj({json_member("events", json_arr(std::move(state_events)))})), - json_member("timeline", json_obj({ - json_member("events", json_arr(std::move(timeline_events))), - json_member("limited", json_bool(limited)), - json_member("prev_batch", json_str(prev_batch)), - })), - }); + return left_room_object("state", std::move(state_events), std::move(timeline_events), limited, prev_batch); } // Spec (Push Rules: Events): `m.push_rules` "Describes all push rules for this user", and its @@ -6659,6 +6926,7 @@ namespace // may omit a redundant member (not include_redundant_members, not full_state), and fed by one that may; it // is begun here, after any long-poll wait, so a rerun judges afresh and records only what it returns. auto const full_state = request.full_state.value_or(false); + auto const use_state_after = request.use_state_after.value_or(false); auto const lazy_on = filter.present && filter.room.state.lazy_load_members; auto const may_omit = lazy_on && !filter.room.state.include_redundant_members; auto lazy = LazySyncRequest{ @@ -6798,10 +7066,16 @@ namespace since_token.has_value() && joined_membership_changed_since(store, room.room_id, user, since_ordering); auto state_events = canonicaljson::Array{}; // Spec (GET /sync, `full_state`): "all state events will be returned, even if since is non-empty". - auto const state_path = !since_token.has_value() || newly_joined || limited || full_state; + // ADR-0763: `state_after` is the changes since `since`, on a limited sync too: they hold everything the gap + // changed, and whole state would only repeat what the client has. + auto const state_path = + !since_token.has_value() || newly_joined || full_state || (limited && !use_state_after); + auto const state_changes = use_state_after && !state_path + ? joined_state_changes(store, room.room_id, since_ordering, sync_position) + : std::vector{}; auto lazy_senders = std::vector{}; auto selector = std::optional{}; - if (lazy.enabled() && (state_path || window_begin != matched.end())) + if (lazy.enabled() && (state_path || window_begin != matched.end() || !state_changes.empty())) { auto plan = LazyRoomSelector::Plan{}; for (auto it = window_begin; it != matched.end(); ++it) @@ -6811,16 +7085,39 @@ namespace lazy_senders.assign(plan.senders.begin(), plan.senders.end()); plan.own_user_id = std::string{user}; plan.ignore_tracker = newly_joined; + // A timeline event is not state to a `state_after` client: only the join in `state_after` tells it. plan.own_forced = (!since_token.has_value() || full_state || newly_joined) && - !timeline_carries_join_of(timeline_events, user); - if (since_token.has_value() && limited) + (use_state_after || !timeline_carries_join_of(timeline_events, user)); + if (since_token.has_value() && (limited || use_state_after)) { plan.gap_after = since_ordering; } + plan.changed_event_ids.insert(state_changes.begin(), state_changes.end()); selector.emplace(lazy, room.room_id, std::move(plan)); } auto const lazy_ref = selector.has_value() ? LazySelectorRef{std::ref(*selector)} : LazySelectorRef{}; - if (state_path) + if (use_state_after) + { + // ADR-0763: as of the response's position, not the last delivered event: a state event the timeline + // filter hides lies before `next_batch`, and no later sync would report it. + if (state_path) + { + state_events = + build_state_events_at_position(store, filter.room.state, room.room_id, sync_position, lazy_ref); + } + else + { + state_events = build_state_events_array_for_ids(store, filter.room.state, state_changes, lazy_ref); + if (selector.has_value()) + { + add_lazy_senders_at_position(store, filter.room.state, room.room_id, + sync::RoomReadAccess{sync::RoomReadKind::current, {}, 0U}, + sync_position, lazy_senders, UnreadableMember::current_stands_in, + *selector, state_events); + } + } + } + else if (state_path) { auto at_start = window_begin != matched.end() ? build_state_events_at_timeline_start(store, filter.room.state, room.room_id, @@ -6897,7 +7194,8 @@ namespace json_member("limited", json_bool(limited)), json_member("prev_batch", json_str(prev_batch)), })), - json_member("state", json_obj({json_member("events", json_arr(std::move(state_events)))})), + json_member(use_state_after ? "state_after" : "state", + json_obj({json_member("events", json_arr(std::move(state_events)))})), json_member("account_data", json_obj({json_member("events", json_arr(std::move(room_account_data)))})), json_member("ephemeral", json_obj({json_member("events", json_arr(std::move(ephemeral_events)))})), json_member("unread_notifications", counts_object(room_counts)), @@ -6994,7 +7292,8 @@ namespace store, filter, history_visibility, viewer_unsigned, ignored_senders, {membership.room_id, user, membership.stream_ordering, since_token.has_value() ? std::optional{since_ordering} : std::nullopt, - sync_position, timeline_cap, sync::room_read_access_for(store, membership.room_id, user)}, + sync_position, timeline_cap, sync::room_read_access_for(store, membership.room_id, user), + use_state_after}, lazy); leave_members.push_back(json_member(membership.room_id, left_room)); ++leave_count; diff --git a/src/sync/StateGroupLookup.cpp b/src/sync/StateGroupLookup.cpp index 496dea725..1d08a9935 100644 --- a/src/sync/StateGroupLookup.cpp +++ b/src/sync/StateGroupLookup.cpp @@ -6,7 +6,9 @@ #include "merovingian/sync/GroupLookup.hpp" #include "merovingian/sync/GroupLookupResult.hpp" +#include #include +#include namespace merovingian::sync { @@ -42,4 +44,33 @@ auto lookup_state_in_group(database::PersistentStore const& store, std::string_v } } +auto state_changes_along_chain(database::PersistentStore const& store, std::string_view ancestor_id, + std::string_view group_id) -> std::optional> +{ + auto changes = std::vector{}; + auto seen = std::set>{}; + auto current = group_id; + for (auto hops = std::size_t{0U}; hops <= events::max_state_group_delta_depth; ++hops) + { + if (current == ancestor_id) + { + return changes; + } + auto const group = store.state_groups.find(current); + if (group == store.state_groups.end() || !group->parent_state_group_id.has_value()) + { + return std::nullopt; + } + for (auto const& entry : store.state_group_state.entries_of(current)) + { + if (seen.emplace(entry.event_type, entry.state_key).second) + { + changes.push_back(entry.event_id); + } + } + current = *group->parent_state_group_id; + } + return std::nullopt; +} + } // namespace merovingian::sync From 134bce30d72af548b7a7bb8558d500ba472d5b48 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 15:31:33 +0000 Subject: [PATCH 04/34] Format the use_state_after QueryParams scenario Co-Authored-By: Claude Sonnet 5.5 --- tests/unit/test_query_params.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_query_params.cpp b/tests/unit/test_query_params.cpp index df1b7d5c4..b4947cdf6 100644 --- a/tests/unit/test_query_params.cpp +++ b/tests/unit/test_query_params.cpp @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "merovingian/core/SyncRequest.hpp" #include "merovingian/core/QueryParams.hpp" +#include "merovingian/core/SyncRequest.hpp" #include @@ -285,7 +285,8 @@ SCENARIO("The sync use_state_after parameter is read as sent and absent when not WHEN("the query parameters are parsed") { auto const enabled = merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=true"); - auto const disabled = merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=false"); + auto const disabled = + merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=false"); auto const other = merovingian::core::parse_query_params("/_matrix/client/v3/sync?use_state_after=1"); auto const absent = merovingian::core::parse_query_params("/_matrix/client/v3/sync"); From f8633bed235a7a1e3644c3d9bbe6e6f9eb644078 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 15:32:41 +0000 Subject: [PATCH 05/34] Document use_state_after on /sync and the 1.1 series start (ADR-0762 to ADR-0765) The decisions of the unmerged ADR-0455 to ADR-0457 carry over as ADR-0763 to ADR-0765, with the forward extremity read through the per-room index main has since gained. Co-Authored-By: Claude Sonnet 5.5 --- ...t-1-1-0-with-the-matrixrtc-support-work.md | 32 +++++++++++ ...ince-since-as-of-the-responses-position.md | 41 +++++++++++++ ...ate-after-delta-compares-the-two-states.md | 57 +++++++++++++++++++ ...e-after-stops-at-what-the-user-may-read.md | 41 +++++++++++++ docs/adr/index.md | 4 ++ docs/matrix-v1.19-client-server-api.md | 3 +- docs/todos/capability-gaps.md | 2 +- src/sync/AGENTS.md | 13 +++++ tests/conformance/AGENTS.md | 1 + 9 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md create mode 100644 docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md create mode 100644 docs/adr/0764-a-state-after-delta-compares-the-two-states.md create mode 100644 docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md diff --git a/docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md b/docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md new file mode 100644 index 000000000..d10a95e43 --- /dev/null +++ b/docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md @@ -0,0 +1,32 @@ +# The 1.1 series starts at 1.1.0 with the MatrixRTC support work + +* Status: accepted +* Date: 2026-11-16 + +## Context and Problem Statement + +[versioning.md](../versioning.md) says `BUILD` is the working counter and that `MINOR` changes "only when the owner +decides". The branch that adds MatrixRTC support is a run of slices that add features and, later, the first schema +migration since 1.0.0. Its first slice is `use_state_after` on `GET /sync`. Which version does that work carry? + +## Considered Options + +* Continue with `BUILD` bumps from 1.0.3 (1.0.4 onward) (rejected) +* Stay on 1.0.x until the first schema migration lands (rejected) +* Start the 1.1 series at 1.1.0 with this work + +## Decision Outcome + +Chosen option: start the 1.1 series at 1.1.0. The owner decided this on 2026-11-16, and this branch, whose first slice +is `use_state_after`, carries 1.1.0. + +A 1.0.x number means a fix or a small change to the 1.0 server. This work adds new client-visible features and will add a +schema migration, which an operator upgrading from 1.0.x must be able to see in the version before reading a change log. +Bumping `BUILD` would hide that, and waiting for the migration would number the early slices as fixes and move the +series start to a commit that is not the first of the series. + +### Consequences + +* The upgrade fixture that [tests/fixtures/upgrade/README.md](../../tests/fixtures/upgrade/README.md) ties to the start of + the 1.1 series is generated from `main` commit `5f458d57` (1.0.3) in a later commit on this branch, not by this slice. +* Every location in the [versioning table](../versioning.md#every-location-that-must-be-updated) moves to 1.1.0 together. diff --git a/docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md b/docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md new file mode 100644 index 000000000..610634305 --- /dev/null +++ b/docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md @@ -0,0 +1,41 @@ +# `state_after` is the change since `since`, as of the response's position + +* Status: accepted +* Date: 2026-11-16 + +## Context and Problem Statement + +`GET /sync` with `use_state_after=true` answers `state_after`: "Updates to the state, between the time indicated by the +`since` parameter, and the **end** of the `timeline`", and a client "MUST only update their local state with events in this +list, and MUST NOT update their local state with events in `timeline`". The spec leaves open which point is the end of the +timeline when the timeline filter hides events, and how much a `limited` sync sends. + +## Considered Options + +* The end is the state recorded for the last delivered timeline event (rejected, see below) +* The end is the response's own position, `sync_position`, the point `next_batch` names +* A `limited` sync sends the whole state at the end of the timeline, as `state` does at its start (rejected, see below) +* A `limited` sync sends the changes since `since`, like any other incremental sync + +## Decision Outcome + +Chosen options: the response's position, and the changes since `since`. + +The last delivered event is the wrong point because `next_batch` is past it. With `not_types: ["m.room.topic"]` on the +timeline, a topic change after the last delivered message lies before `next_batch`. The next sync starts after it, and the +client takes no state from the timeline, so a `state_after` as of that message would lose the change for good. Without a +filter the two points hold the same state, so this reading never contradicts the spec. + +`state` sends the whole state on a `limited` sync because a client cannot otherwise tell what the gap changed. `state_after` +has no such gap: the delta ([ADR-0764](0764-a-state-after-delta-compares-the-two-states.md)) covers `since` to the position +whatever the timeline's size, so whole state would only repeat what the client has, on the requests that are already +largest. Whole state remains for what the spec names (no `since`, `full_state=true`) and for a room joined in the window, +where the client holds nothing to apply a delta to. + +The parameter is parsed like `full_state`: `true` turns it on, anything else leaves it off. With it off, or absent, no +response changes. Sliding sync (MSC4186) has its own state model and is not changed. + +### Consequences + +* `state_after` can name an event after the last timeline event, when the timeline filter hid it. That event is not an + update a client could have got any other way. diff --git a/docs/adr/0764-a-state-after-delta-compares-the-two-states.md b/docs/adr/0764-a-state-after-delta-compares-the-two-states.md new file mode 100644 index 000000000..43a28bea9 --- /dev/null +++ b/docs/adr/0764-a-state-after-delta-compares-the-two-states.md @@ -0,0 +1,57 @@ +# A `state_after` delta compares the two states, and every change it finds is sent under lazy-loading + +* Status: accepted +* Date: 2026-11-16 + +## Context and Problem Statement + +An incremental `state_after` holds the state that differs between `since` and the response's position. State resolution can +move a key back to an older event: no event of the window is about that key, and its current event is older than `since`. +This is the case `use_state_after` exists for, since a client reading `state` and `timeline` cannot see it. `/sync` runs under +`runtime.mutex` for every syncing user, so the cost matters as much as the answer +([ADR-0435](0435-the-event-table-keeps-a-per-room-index-in-stream-order.md), +[ADR-0436](0436-an-event-is-judged-for-ignored-senders-without-parsing-it-first.md)). + +## Considered Options + +* List the state events in the window and send each key's current value (rejected: misses a key moved back) +* Read both whole states for every room with a new event (rejected: a room's state, twice, per user, per message) +* Compare state groups: nothing when they are one group, the rows between them on an unforked delta chain, and both whole + states only when the chain does not connect them + +## Decision Outcome + +Chosen option: compare state groups (`joined_state_changes` in `ClientServerRuntime.cpp`, `state_changes_along_chain` in +`StateGroupLookup.cpp`). + +* A room with no event above `since` costs one look at its newest row and sends nothing. +* The state at `since` is the recorded state of the newest event at or below it (`state_anchor_event`). With one forward + extremity, at or below the position, the current state is that event's recorded state, so the two points are two groups: + the same group for a window of messages (no rows read), otherwise the rows between them + (`state_changes_along_chain`, which never reads a snapshot's rows). The extremity comes from the room's own tips + (`ForwardExtremityTable::tips_of_room`), never from a walk of every room's. +* A fork, a snapshot between the two points (at most `max_state_group_delta_depth` hops apart) or an unreadable chain falls + back to the state at the position minus the state at `since`, the room's own rows. An event is the value of one key, so + subtracting event IDs finds exactly the keys that differ. +* With no recorded state at `since`, all the state at the position is sent: an update the client holds is harmless. + +Lazy-loading: `state_after` carries, besides all non-member changes, every member event in the delta (never redundant, in +the timeline or not, about a sender or not, since the client takes no state from the timeline), and each timeline sender's +member event as of the position, by key and subject to redundancy (`member_event_at_position`, with the cost rule of +[ADR-0454](0454-a-senders-member-is-looked-up-by-key-never-by-scanning-the-room-state.md)). The user's own member event on +an initial sync, `full_state=true` or a join in the window is sent even when the timeline carries the join, which +[ADR-0452](0452-the-state-filter-selects-lazy-loaded-members-on-sync.md) allows only for `state`. + +The tracker's rules ([ADR-0450](0450-lazy-loaded-members-are-skipped-only-when-the-since-token-acknowledges-them.md)) are +unchanged, and it is shared with `state` responses under the same stream key. It compares event IDs: a client that switches +modes either holds the event named or is sent it again. What `state_after` sends is recorded as what `state` sends is. + +### Consequences + +* The state at `since` is what was recorded for one event, not what the response carrying that token said. If the graph was + forked at `since`, the resolved state the client was told can differ from that recorded state, and a key that resolution + later flipped to the recorded value is not sent. A backfilled event ordered at or below `since` can stand in the same way. + Storing each response's state per token would close this and was refused for its cost; `full_state=true` repairs a client. +* A window that crosses a snapshot reads both whole states, about once per `max_state_group_delta_depth` state changes. It + sends only what differs. +* A key that changed and changed back within the window may be sent, with the value the client already holds. diff --git a/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md b/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md new file mode 100644 index 000000000..9f291fe38 --- /dev/null +++ b/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md @@ -0,0 +1,41 @@ +# A left room's `state_after` stops at what the user may read, and always holds their own membership + +* Status: accepted +* Date: 2026-11-16 + +## Context and Problem Statement + +A left room's `state_after` is "The state updates for the room up to the **end** of the timeline", and that timeline runs +"up to the point when the user left". A left room's incremental `state` holds only the leave, and only when the timeline does +not carry it ([ADR-0453](0453-sync-honours-full-state.md) kept that). A `state_after` client takes no state from the +timeline, and a user who left may not read the room after they left, whatever a later ban changes +([ADR-0357](0357-a-left-rooms-sync-state-is-gated-by-room-read-access.md)). + +## Considered Options + +* Reuse the joined room's delta, against current state (rejected: it tells a departed user what happened after they left) +* Keep `state`'s rule and send the leave alone on an incremental sync (rejected: "the state updates up to the end of the + timeline" hold what changed before the leave too) +* Read the state only as far as `room_read_access_for` allows, take away what the client held at `since`, and add the user's + own membership event + +## Decision Outcome + +Chosen option: the last (`build_left_room_state_after`). + +* The room's state comes from the state anchor of `room_read_access_for`, capped at the event that ended the join. +* The user's own key is always the current member event, sent even when the timeline carries it. For a user who left and + was banned since, the room is as of the leave and the membership is the ban, the one thing the room's state may say about + after the leave. +* A `since` at or past the end of what the user may read (they left before it, and were banned after it) leaves only that + membership event. +* A user who never joined (`none`) gets only their own event, as on `state` (ADR-0357). +* `full_state=true` does not make a left room's `state_after` whole: `full_state` is about "all rooms the user is a member + of" and the Left Room `state_after` does not name it, as ADR-0453 has for `state`. +* With lazy-loading, a sender's member event is looked up as of the leave. When the state group chain cannot be read it is + omitted, while a joined room lets the current member event stand in: omitting a member is a smaller harm than telling a + departed user a later one. + +### Consequences + +* A left room's state always costs two whole reads, of that room's rows. Leaves are rare and once per room per client. diff --git a/docs/adr/index.md b/docs/adr/index.md index 005670d41..4fdf8578c 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -696,3 +696,7 @@ * [0815 - The authentication pages are built into the binary, without JavaScript](0815-the-authentication-pages-are-built-into-the-binary-without-javascript.md) * [0816 - `server.oauth.*` replaces `server.oidc.*`, and the old keys are refused](0816-server-oauth-replaces-server-oidc-and-the-old-keys-are-refused.md) * [0817 - A form body is read strictly and refused whole](0817-a-form-body-is-read-strictly-and-refused-whole.md) +* [0762 - The 1.1 series starts at 1.1.0 with the MatrixRTC support work](0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md) +* [0763 - state_after is the change since since, as of the response's position](0763-state-after-is-the-change-since-since-as-of-the-responses-position.md) +* [0764 - A state_after delta compares the two states, and every change it finds is sent under lazy-loading](0764-a-state-after-delta-compares-the-two-states.md) +* [0765 - A left room's state_after stops at what the user may read, and always holds their own membership](0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md) diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index 5bb9cb03c..bb78b1eb5 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -514,7 +514,8 @@ In each case below the spec leaves the choice to the server; the owner decided t - **An identifier over 255 bytes is refused (ADR-0613).** `GET /profile/{userId}` and `GET /directory/room/{roomAlias}` answer `400 M_INVALID_PARAM` before any lookup. The spec limits a user ID and a room alias to 255 bytes. - **Unset profile fields are absent.** `GET /profile/{userId}`, the user directory and `/search` `profile_info` leave out a `displayname` or `avatar_url` that is not set, instead of sending an empty string. - **`GET /rooms/{roomId}/messages` with `lazy_load_members`.** `state` holds only the `m.room.member` events of the senders of the returned `chunk` (no other state). Redundant membership events are not tracked across calls, so a sender's membership is sent on every page (the spec's `include_redundant_members` default permits it). -- **Lazy-loading on `/context` and `/sync` (0.12.73).** `GET /context` with `lazy_load_members` returns as `state` only the member events of the senders of `events_before`, `event` and `events_after` (stateless, redundancy allowed). `GET /sync` with `room.state.lazy_load_members` (the `timeline` flag does nothing) sends, besides the non-member state the room would carry without lazy-loading (the state filter still applies, and an incremental sync is a delta), the member events of the delivered timeline's senders, every member change in the gap of a limited sync, and the user's own membership on joining or `full_state=true`. A member the client acknowledged through its `since` is skipped (best effort, in memory, per user, device and `filter`); `include_redundant_members=true` and `full_state=true` skip nothing. `full_state=true` is now honoured for joined rooms. Not implemented: `use_state_after`/`state_after`. +- **`use_state_after` on `/sync` (1.1.0, ADR-0763 to ADR-0765).** With `use_state_after=true`, joined and left rooms answer `state_after`, present even when empty, and omit `state`; with `false` or absent, the reverse. It is the state as of the response's position: all of it with no `since`, `full_state=true` or a join in the window, otherwise what differs from the state at `since`, including what the timeline carries, what the timeline filter hid and a key state resolution moved back, also on a `limited` sync. A left room's stops at the user's leave and always holds their own membership; `full_state` does not make it whole. Invited and knocked rooms have no `state_after` in the spec and get none. With `lazy_load_members`, every member change is sent and each timeline sender's member is as of the end of the timeline. Sliding sync is unchanged. +- **Lazy-loading on `/context` and `/sync` (0.12.73).** `GET /context` with `lazy_load_members` returns as `state` only the member events of the senders of `events_before`, `event` and `events_after` (stateless, redundancy allowed). `GET /sync` with `room.state.lazy_load_members` (the `timeline` flag does nothing) sends, besides the non-member state the room would carry without lazy-loading (the state filter still applies, and an incremental sync is a delta), the member events of the delivered timeline's senders, every member change in the gap of a limited sync, and the user's own membership on joining or `full_state=true`. A member the client acknowledged through its `since` is skipped (best effort, in memory, per user, device and `filter`); `include_redundant_members=true` and `full_state=true` skip nothing. `full_state=true` is now honoured for joined rooms. `use_state_after` is implemented (1.1.0, below). - **Initial `/sync` carries `m.push_rules`.** An initial `/sync` (no usable `since`) always holds a global `m.push_rules` account-data event with the user's effective ruleset (the server defaults plus their changes), if the filter's `account_data` admits it. An incremental `/sync` carries it only after a change. - **A hole in the event graph makes a joined room's `/sync` timeline `limited` (ADR-0289).** The spec defines `limited` by the filter `limit` and leaves events the server never received over federation unspecified; an event in the window whose `prev_events` are not all held (no row, or an outlier with no state group; rejected, soft-failed and backfilled events count as held, ADR-0290) starts a limited timeline at that event, with `prev_batch` for `/messages` to reach what lies before. A limited incremental sync carries the state at the start of the timeline. Sliding sync is unchanged. - **`POST /user_directory/search` searches visible users only.** A search considers the users who share a joined room with the requester, and users joined to a room whose join rule is `public` or whose history visibility is `world_readable`. The requester is found only through a public or world-readable room. Before, every local profile matched. The result list is at most 100 users whatever `limit` asks for, and `limited` says when more matched (ADR-0334). diff --git a/docs/todos/capability-gaps.md b/docs/todos/capability-gaps.md index 3f6dcd843..490c80f2e 100644 --- a/docs/todos/capability-gaps.md +++ b/docs/todos/capability-gaps.md @@ -67,7 +67,7 @@ When adding a capability, record what is *missing* as explicitly as what works. | `POST /_matrix/client/v3/rooms/{roomId}/receipt/{receiptType}/{eventId}` | `spec-covered` | 0.12.30: accepts `m.read`, `m.read.private`, and `m.fully_read`; rejects unknown types with 400 `M_INVALID_PARAM`, non-members with 403 `M_FORBIDDEN`, and malformed bodies with 400 `M_BAD_JSON`. `m.read` and `m.read.private` are durable receipt rows (ADR-0169), keyed by `(room, type, user, thread_id)`, so restarts preserve read positions. `thread_id` is optional for `m.read`/`m.read.private`, must be a non-empty string naming `main` or a thread root in this room, and is carried by `/sync` `m.receipt` events and federated `m.read` EDUs. `m.read` is public and federated; `m.read.private` is visible only to its sender and is not federated. `m.fully_read` is accepted without `thread_id`, rejected if `thread_id` is supplied, and stored as the owner's `m.fully_read` room account data rather than a receipt; it never appears in `m.receipt` or a federation EDU. Invalid `thread_id` values (including a non-root room event ID) return 400 `M_INVALID_PARAM`. 0.12.60: `/sync` delivers one `m.receipt` event per thread categorisation (unthreaded, `main`, each thread root), so a user's unthreaded and threaded `m.read` for one event both arrive; sliding sync's receipts extension still holds one event per room and keeps the latest receipt on such a clash. **Gap:** the optional 400 for "The event_id is not related to the thread_id" is not checked; the root's existence and main-timeline classification are checked, but the acknowledged event's relationship to it is not. | | `POST /_matrix/client/v3/user_directory/search` | `spec-covered` | 0.12.60: searches only the users visible to the requester (a shared joined room, or joined to a `public` or `world_readable` room), as Complement expects; the spec allows searching more. Remote users are still not queried (the spec says SHOULD). | | `POST /_matrix/client/v3/delete_devices` | `spec-covered` | | -| `GET /_matrix/client/v3/sync` | `spec-covered` | Unread-notification counts are relative to the user's read receipts, partitioned by thread (an unthreaded receipt covers every earlier event, a threaded one only its thread; 0.12.30), and bound to the response snapshot. 0.12.30: the `unread_thread_notifications` filter flag (on `room.timeline`) returns per-thread counts and limits `unread_notifications` to the main timeline, and a non-state event the user sends reads its own thread up to itself (ADR-0170). The `summary` object (`m.joined_member_count`, `m.invited_member_count`, `m.heroes`) is now emitted for every joined room and covered by conformance tests. `filter_id` parameter now wired and conformance-covered; sync-only stream tokens now bound to the response snapshot for to-device delivery so room keys queued during response construction remain pending for the next sync. 0.12.59: a joined room's `state` on an initial sync (or a newly joined room) is the state up to the start of the timeline, not current state. Not done: an incremental sync with a limited timeline gives no state updates for the gap between `since` and the timeline's start. 0.12.60: `rooms.leave` lists rooms the user left **or was banned from**; each holds the filtered timeline (types, senders, `contains_url` now apply to a `/sync` timeline, and a `limit` of 0 means none) up to the leave or ban, as the user may see it, and an initial sync's `state` as of the start of that timeline (as of the leave when it is empty). An incremental sync reports a leave once, after `since`; `include_leave` lists left rooms on an initial sync only. A forgotten room is never listed (the spec says so, although Complement `TestRoomForget` expects its leave in the next incremental sync). Timeline events carry `unsigned.membership` (the requesting user's membership at the event, `leave` before they had one) and, for the device that sent them, `unsigned.transaction_id`; the same two on `GET /rooms/{roomId}/event/{eventId}`. 0.12.71: sliding sync's timeline and `required_state` events carry the same two fields, and no read path serves a stored event's `unsigned` (a received PDU is stored without it; ADR-0402). **Not done:** both fields are absent from `/messages`, `/context`, `/relations` and `/search`, and transaction-ID idempotency is still scoped to the user, not the device (the spec scopes it to a device). 0.12.60: an initial `/sync` always carries a global `m.push_rules` event with the effective ruleset (defaults plus changes), as Complement expects; an incremental one only after a change. **Lazy-loading of room members (0.12.73, ADR-0450 to ADR-0454).** With `room.state.lazy_load_members`, `state` holds the non-member state as before and only these member events: the senders of the delivered timeline (as of its start, skipped when the client has acknowledged that very event through its `since`), every member change in the gap of a limited sync, and the user's own on joining or `full_state`. Redundancy is tracked in memory per (user, device, filter, room) and lost on restart; `include_redundant_members` and `full_state=true` bypass it. The same flag on `room.timeline` does nothing, and the state filter's `types`/`senders` apply to the members. `full_state=true` is now honoured for joined rooms. Not done: `use_state_after`/`state_after`, sliding sync. | +| `GET /_matrix/client/v3/sync` | `spec-covered` | Unread-notification counts are relative to the user's read receipts, partitioned by thread (an unthreaded receipt covers every earlier event, a threaded one only its thread; 0.12.30), and bound to the response snapshot. 0.12.30: the `unread_thread_notifications` filter flag (on `room.timeline`) returns per-thread counts and limits `unread_notifications` to the main timeline, and a non-state event the user sends reads its own thread up to itself (ADR-0170). The `summary` object (`m.joined_member_count`, `m.invited_member_count`, `m.heroes`) is now emitted for every joined room and covered by conformance tests. `filter_id` parameter now wired and conformance-covered; sync-only stream tokens now bound to the response snapshot for to-device delivery so room keys queued during response construction remain pending for the next sync. 0.12.59: a joined room's `state` on an initial sync (or a newly joined room) is the state up to the start of the timeline, not current state. Not done: an incremental sync with a limited timeline gives no state updates for the gap between `since` and the timeline's start. 0.12.60: `rooms.leave` lists rooms the user left **or was banned from**; each holds the filtered timeline (types, senders, `contains_url` now apply to a `/sync` timeline, and a `limit` of 0 means none) up to the leave or ban, as the user may see it, and an initial sync's `state` as of the start of that timeline (as of the leave when it is empty). An incremental sync reports a leave once, after `since`; `include_leave` lists left rooms on an initial sync only. A forgotten room is never listed (the spec says so, although Complement `TestRoomForget` expects its leave in the next incremental sync). Timeline events carry `unsigned.membership` (the requesting user's membership at the event, `leave` before they had one) and, for the device that sent them, `unsigned.transaction_id`; the same two on `GET /rooms/{roomId}/event/{eventId}`. 0.12.71: sliding sync's timeline and `required_state` events carry the same two fields, and no read path serves a stored event's `unsigned` (a received PDU is stored without it; ADR-0402). **Not done:** both fields are absent from `/messages`, `/context`, `/relations` and `/search`, and transaction-ID idempotency is still scoped to the user, not the device (the spec scopes it to a device). 0.12.60: an initial `/sync` always carries a global `m.push_rules` event with the effective ruleset (defaults plus changes), as Complement expects; an incremental one only after a change. **Lazy-loading of room members (0.12.73, ADR-0450 to ADR-0454).** With `room.state.lazy_load_members`, `state` holds the non-member state as before and only these member events: the senders of the delivered timeline (as of its start, skipped when the client has acknowledged that very event through its `since`), every member change in the gap of a limited sync, and the user's own on joining or `full_state`. Redundancy is tracked in memory per (user, device, filter, room) and lost on restart; `include_redundant_members` and `full_state=true` bypass it. The same flag on `room.timeline` does nothing, and the state filter's `types`/`senders` apply to the members. `full_state=true` is now honoured for joined rooms. Not done: sliding sync. 1.1.0: `use_state_after` answers `state_after` and omits `state` for joined and left rooms, on initial and incremental sync, with `full_state` and with `lazy_load_members` (ADR-0763 to ADR-0765); invited and knocked rooms have no such field. | | `GET /_matrix/client/v3/keys/changes` | `spec-covered` | Requires and validates `from` and `to` sync tokens, decodes them to `sync_stream_id` boundaries, and filters `changed`/`left` device-list changes to that half-open range. Conformance coverage includes missing/invalid parameters and an upper-bound test proving `to` excludes later changes. | | Remote device-list cache | `spec-covered` | Server-Server API v1.19 "Device Management". 0.12.81 (G14, ADR-0560, ADR-0561): the list of a remote user who shares a joined room with a local user is filled from `GET /user/devices/{userId}` (its `stream_id` places later updates) and kept in memory, bounded per user and in all. `m.device_list_update` EDUs apply in `stream_id` order; a `prev_id` above the snapshot that was never applied, or an empty one, marks the list stale and the next `/keys/query` fetches it again; `m.signing_key_update` does the same. An update that changes nothing is no longer reported in `device_lists.changed`. Not persisted: a restart refills on demand. Complement `TestDeviceListsUpdateOverFederation` (three subtests) stays skip-listed until it is run against this change. | | `POST /_matrix/client/v4/sync` and `POST /_matrix/client/unstable/org.matrix.msc4186/sync` (MSC4186 Simplified Sliding Sync) | `spec-covered` | Advertised via `unstable_features["org.matrix.msc4186"] = true` and the compatibility alias `unstable_features["org.matrix.simplified_msc3575"] = true`; the stable `v4` endpoint is now also routed and accepts the same request shape. Implements named lists with windowed `ranges` or singular `range`, SYNC/INVALIDATE/INSERT/DELETE/UPDATE ops, per-room state/timeline, room subscriptions, and all five extensions (to_device, e2ee, account_data, receipts, typing). Long-poll parks in the notifier's registry and holds no sync-pool thread while it waits (ADR-0569); `pos` and `timeout` may be supplied in query parameters or in the JSON body. List + `room_subscriptions` merge: when a room is in both a list window and a `room_subscriptions` entry, `required_state` is the superset (dedup + wildcard-aware), `timeline_limit` is the max, and `include_heroes` is OR'd (previously the subscription overrode the list wholesale) — landed 0.11.9. Multi-list combination: a room in MORE THAN ONE list window now has ALL matching list windows combined (left-fold over `combine_room_configs`, associative+commutative) with each other and the `room_subscriptions` entry, not just the first matching list — landed 0.11.10. Access control (CSAZ-1): a `room_subscriptions` entry is served only for a room the caller has joined; an invited room yields `invite_state` only (no timeline or `required_state`); any other membership omits the room, and the receipts/typing extensions drop rooms the caller has not joined. `timeline_limit` is clamped to 100, and more than 256 subscriptions or 256 `required_state` pairs is a 400 `M_INVALID_PARAM` (`sync::room_access_for`, `sliding_sync_request_limit_violation`). Truncated timelines (0.12.45): a room carries `limited` when its timeline holds fewer events than it could, `prev_batch` (a `/messages` `from`) on every non-empty timeline, `num_live`, and on a grown `timeline_limit` resends its latest events with `expanded_timeline`, answering without long-polling once per request `pos` (ADR-0221). Remaining: broader MSC4186 conformance fixtures, subscription timeline pruning, and `bump_stamp`, which is never sent (Element X orders its room list by it). | diff --git a/src/sync/AGENTS.md b/src/sync/AGENTS.md index a233d54fe..c425a7a3a 100644 --- a/src/sync/AGENTS.md +++ b/src/sync/AGENTS.md @@ -359,6 +359,19 @@ room and, when the user is local, for them. The membership acceptor (`send_join` `device_lists.left` for remote users. A user who joins over federation is also `changed` for themselves (spec: a server may add a user to `changed` when they share a new room). +## `use_state_after` on `/sync` (ADR-0763 to ADR-0765) + +`state_after` is as of `sync_position`, not the last delivered event, or a state event the timeline filter hides is lost. +A delta is `joined_state_changes`: it compares the state at `since` with the state at the position. Do not replace it with a +list of the window's state events (a key state resolution moved back has none), and do not read both whole states when the +graph is unforked (`state_changes_along_chain` costs the rows between them, none for a window of messages). Read the room's +single forward extremity with `ForwardExtremityTable::tips_of_room`, never a walk of every room's. A left room goes through +`build_left_room_state_after`, which reads only as far as `room_read_access_for` allows: never a joined room's delta, which +would tell a departed user what came after their leave. With lazy-loading the delta's member events are always kept +(`Plan::changed_event_ids`), and the senders' come by key from `member_event_at_position`. A timeline event is not state +in this mode, so a rule that relies on the timeline telling the client something applies to `state` only. Invited and knocked +rooms have no `state_after`; sliding sync does not read `use_state_after`. + ## Lazy-loading room members on `/sync` (ADR-0450 to ADR-0454) `room.state.lazy_load_members` (never `room.timeline`'s) turns it on. `state` then carries the non-member state the room would carry without lazy-loading (the state filter and the incremental diff --git a/tests/conformance/AGENTS.md b/tests/conformance/AGENTS.md index dccaf0ea3..c65ede896 100644 --- a/tests/conformance/AGENTS.md +++ b/tests/conformance/AGENTS.md @@ -106,6 +106,7 @@ Every test in this directory directly encodes a **MUST** or **SHOULD** from the | `test_sync_filter_conformance.cpp` | [CS API § Filtering](../../docs/matrix-v1.19-spec/client-server-api.md#filtering) | | `test_sync_summary_conformance.cpp` | [CS API § Syncing](../../docs/matrix-v1.19-spec/client-server-api.md#syncing) | | `test_sync_state_conformance.cpp` | [CS API `GET /sync`](../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync) (a joined room's `state` is the state up to the start of the timeline, not current state) | +| `test_sync_state_after_conformance.cpp` | [CS API `GET /sync`](../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync) (`use_state_after`: `state_after` present and `state` omitted with `true`, the reverse with `false` and by default, for joined and left rooms, initial and incremental, `full_state` and `lazy_load_members`; the state as of the end of the timeline; a left room's stops at the leave) | | `test_typing_conformance.cpp` | [CS API § Typing Notifications](../../docs/matrix-v1.19-spec/client-server-api.md#typing-notifications) (`typing` must be a boolean; typing expires after its `timeout`, never later than the server's ceiling) · [SS API § Typing Notifications](../../docs/matrix-v1.19-spec/server-server-api.md#typing-notifications) and [§ Server ACLs](../../docs/matrix-v1.19-spec/server-server-api.md#server-access-control-lists-acls) (an inbound `m.typing` EDU for a user not joined to the room, or from a server the room's ACL denies, is ignored; a remote flood at the cap never evicts a local user's typing) | | `test_x_matrix_auth_parsing.cpp` | [SS API § X-Matrix Auth](../../docs/matrix-v1.19-spec/server-server-api.md#request-authentication) | From 2080b55424e179cf5ecb6e55da280466c14a81d7 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 15:35:45 +0000 Subject: [PATCH 06/34] Insert the test's second forward extremity through the table's insert Co-Authored-By: Claude Sonnet 5.5 --- tests/conformance/test_sync_state_after_conformance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conformance/test_sync_state_after_conformance.cpp b/tests/conformance/test_sync_state_after_conformance.cpp index 86c8a5b07..bcbcdf625 100644 --- a/tests/conformance/test_sync_state_after_conformance.cpp +++ b/tests/conformance/test_sync_state_after_conformance.cpp @@ -496,7 +496,7 @@ SCENARIO("state_after reports a key that state resolution moved back to an older say(rt, alice, room, "the event that arrived with the fork"); auto& store = rt.homeserver.database.persistent_store; - store.forward_extremities.push_back({room, old_topic}); + std::ignore = store.forward_extremities.insert({room, old_topic}, user_id("alice")); for (auto& row : store.state) { if (row.room_id == room && row.event_type == "m.room.topic") From 13173b8e40507597da48431f1bb34a6bf7d1a1da Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 16:01:49 +0000 Subject: [PATCH 07/34] Add the 1.0.3 database upgrade fixture, written by the 1.0.3 release Generated by scripts/generate_upgrade_fixture.py from main commit 5f458d57 (1.0.3) in a Linux-native checkout: the old merovingian-server wrote every row over HTTP on SQLite and on a throwaway PostgreSQL cluster. Nothing is hand-written (ADR-0216). Co-Authored-By: Claude Sonnet 5.5 --- tests/fixtures/upgrade/1.0.3/manifest.json | 472 ++++ .../upgrade/1.0.3/merovingian.sqlite3 | Bin 0 -> 589824 bytes tests/fixtures/upgrade/1.0.3/postgresql.sql | 2035 +++++++++++++++++ 3 files changed, 2507 insertions(+) create mode 100644 tests/fixtures/upgrade/1.0.3/manifest.json create mode 100644 tests/fixtures/upgrade/1.0.3/merovingian.sqlite3 create mode 100644 tests/fixtures/upgrade/1.0.3/postgresql.sql diff --git a/tests/fixtures/upgrade/1.0.3/manifest.json b/tests/fixtures/upgrade/1.0.3/manifest.json new file mode 100644 index 000000000..ecf4bc43a --- /dev/null +++ b/tests/fixtures/upgrade/1.0.3/manifest.json @@ -0,0 +1,472 @@ +{ + "release": "1.0.3", + "commit": "5f458d5732809f096042cb5c74392537c2b47c6e", + "schema_version": 1, + "server_name": "example.org", + "master_key_hex": "e20a57ef7ee73bc1cc86594aa3954498e1b4ce7e4f06f7c32c285f8e2b6a2989", + "registration_token": "test-registration-token", + "sqlite": { + "device_keys": { + "device_keys": { + "user_id": "@alice:example.org", + "device_id": "UPGRADE1", + "algorithms": [ + "m.olm.v1.curve25519-aes-sha2", + "m.megolm.v1.aes-sha2" + ], + "keys": { + "curve25519:UPGRADE1": "avkVfi/n0gvWx3G6YuPrB25Fy5tWrqUpFe0NfOvzyGw", + "ed25519:UPGRADE1": "N5GQ6FD/1Zsb98rTHyr22OIGA80WqwICtUZ4X5XPD6s" + }, + "signatures": { + "@alice:example.org": { + "ed25519:UPGRADE1": "UBBsAgbpMe0FdxfQPzzBo7xekw1ImFlqIJdcAuhXafvTbd/1ynySi1jxrc1N5nFSl1CaTAmqhJCeLK1EF3nIDg" + } + } + }, + "one_time_keys": { + "signed_curve25519:AAAAAQ": { + "key": "b60hLnOYxauXUjJqQzghgaK3oPo4ft3LInvHYLRF1xo", + "signatures": { + "@alice:example.org": { + "ed25519:UPGRADE1": "QtIiXHH1v3RHnRxMPRzgWxcyWgImVJnyoDn1CPxf2ZAXmpykFWOAVXDh9Daldfag6MPwNQo8Ir6X4FmAygp5Cg" + } + } + } + }, + "ed25519_public_key": "N5GQ6FD/1Zsb98rTHyr22OIGA80WqwICtUZ4X5XPD6s" + }, + "next_batch": "1a_1a_b_290d4aac260e176b", + "users": { + "alice": { + "user_id": "@alice:example.org", + "password": "CorrectHorse7!", + "device_id": "UPGRADE1", + "access_token": "mvs_ec0853b0434ffd9915e5ecbedf5c9239d1fea2435329e2f68277dd988f66c7a3" + }, + "bob": { + "user_id": "@bob:example.org", + "password": "BatteryStaple9!", + "device_id": "UPGRADE2", + "access_token": "mvs_e32fb00b0b46036c634ed0ce07f31a054b567f87d462351239a12a7ee8ec5e80" + } + }, + "displayname": "Alice Upgrade", + "private_room_id": "!izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY", + "public_room_id": "!0en5oBPugaFyvTPadaX1TvcE6Y6yAuTJ3e68KXpMojA", + "alias": "#upgrade-public:example.org", + "messages": [ + { + "sender": "@alice:example.org", + "event_id": "$Bvm248Nwe5VYLasWdIIuUcPjIFQLXm-Cn6H6uMX7bfw", + "body": "upgrade message 1" + }, + { + "sender": "@alice:example.org", + "event_id": "$-f3HSwamy2GXuPwpavYDXFIKSgUXyoUClPrX2pBYcfY", + "body": "upgrade message 2" + }, + { + "sender": "@alice:example.org", + "event_id": "$W1InbbGrmfwJJ3llrJzq5dBwrFUau7RDNzzWyqEs0TE", + "body": "upgrade message 3" + }, + { + "sender": "@bob:example.org", + "event_id": "$j9Jx8wXbfA_HTVzxwwNUwT-qc8rp2Sl0kx-naT1Fn6U", + "body": "reply from bob" + } + ], + "media": { + "mxc_uri": "mxc://example.org/Mj56RocdRWUhla3HMB-Crg", + "sha256": "7dbc7d20d54b5924ceb6ba419e522d699394d55f052f8b8815c15d199c357143", + "content_type": "text/plain", + "size": 340 + }, + "account_data": { + "type": "org.merovingian.upgrade", + "content": { + "fixture": "0.11.13" + } + }, + "pusher": { + "kind": "http", + "app_id": "org.merovingian.upgrade", + "app_display_name": "Upgrade Fixture App", + "device_display_name": "Upgrade Fixture Device", + "lang": "en", + "pushkey": "upgrade-pushkey", + "data": { + "url": "https://push.example.org/_matrix/push/v1/notify" + } + }, + "http_log": [ + { + "step": "register alice", + "method": "POST", + "path": "/_matrix/client/v3/register", + "status": 200 + }, + { + "step": "login alice", + "method": "POST", + "path": "/_matrix/client/v3/login", + "status": 200 + }, + { + "step": "register bob", + "method": "POST", + "path": "/_matrix/client/v3/register", + "status": 200 + }, + { + "step": "login bob", + "method": "POST", + "path": "/_matrix/client/v3/login", + "status": 200 + }, + { + "step": "alice displayname", + "method": "PUT", + "path": "/_matrix/client/v3/profile/%40alice%3Aexample.org/displayname", + "status": 200 + }, + { + "step": "alice create private room", + "method": "POST", + "path": "/_matrix/client/v3/createRoom", + "status": 200 + }, + { + "step": "alice invite bob", + "method": "POST", + "path": "/_matrix/client/v3/rooms/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY/invite", + "status": 200 + }, + { + "step": "bob join", + "method": "POST", + "path": "/_matrix/client/v3/join/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY", + "status": 200 + }, + { + "step": "send message 1", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY/send/m.room.message/upgrade-txn-1", + "status": 200 + }, + { + "step": "send message 2", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY/send/m.room.message/upgrade-txn-2", + "status": 200 + }, + { + "step": "send message 3", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY/send/m.room.message/upgrade-txn-3", + "status": 200 + }, + { + "step": "send message 4", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY/send/m.room.message/upgrade-txn-4", + "status": 200 + }, + { + "step": "alice create public room", + "method": "POST", + "path": "/_matrix/client/v3/createRoom", + "status": 200 + }, + { + "step": "publicRooms listing", + "method": "GET", + "path": "/_matrix/client/v3/publicRooms", + "status": 200 + }, + { + "step": "resolve alias", + "method": "GET", + "path": "/_matrix/client/v3/directory/room/%23upgrade-public%3Aexample.org", + "status": 200 + }, + { + "step": "alice upload media", + "method": "POST", + "path": "/_matrix/media/v3/upload", + "status": 200 + }, + { + "step": "alice download media", + "method": "GET", + "path": "/_matrix/client/v1/media/download/example.org/Mj56RocdRWUhla3HMB-Crg", + "status": 200 + }, + { + "step": "alice account data", + "method": "PUT", + "path": "/_matrix/client/v3/user/%40alice%3Aexample.org/account_data/org.merovingian.upgrade", + "status": 200 + }, + { + "step": "alice set pusher", + "method": "POST", + "path": "/_matrix/client/v3/pushers/set", + "status": 200 + }, + { + "step": "alice keys/upload", + "method": "POST", + "path": "/_matrix/client/v3/keys/upload", + "status": 200 + }, + { + "step": "alice read messages", + "method": "GET", + "path": "/_matrix/client/v3/rooms/%21izamUNbT8G68FVAZUuIfKH_NKgSJzyrrD8CJIz9mxTY/messages", + "status": 200 + }, + { + "step": "alice initial sync", + "method": "GET", + "path": "/_matrix/client/v3/sync", + "status": 200 + } + ], + "non_2xx": [], + "server_exit_code": 0 + }, + "postgresql": { + "device_keys": { + "device_keys": { + "user_id": "@alice:example.org", + "device_id": "UPGRADE1", + "algorithms": [ + "m.olm.v1.curve25519-aes-sha2", + "m.megolm.v1.aes-sha2" + ], + "keys": { + "curve25519:UPGRADE1": "aJoAYROXT9rrYFhiSAmFkQk8DgPUrZKIq59YFaOAgQg", + "ed25519:UPGRADE1": "xZ6DW0uuhQKl7HY7m4J5dMmK6CWpCCrtYwps1QBYoyE" + }, + "signatures": { + "@alice:example.org": { + "ed25519:UPGRADE1": "khdpySd56eC7863q/p2A8IOUUe2TQ28I4eBVO45kevqFspi6gW12BDvcz2qiB9GpsnS2zCNOxcde/LyLPA7RCw" + } + } + }, + "one_time_keys": { + "signed_curve25519:AAAAAQ": { + "key": "0kBH0U8cIYy+UylyqqSe4teHLhfX/hYBgpiZZlNp4zk", + "signatures": { + "@alice:example.org": { + "ed25519:UPGRADE1": "kAxIom4YQ7T7Q2laPSclXDUngDWrsn4yPReEKDnH+1YVJhB2s8Mdy3x4YtyxOMw8e1c/K6C2UeGeeSyOXuS6CA" + } + } + } + }, + "ed25519_public_key": "xZ6DW0uuhQKl7HY7m4J5dMmK6CWpCCrtYwps1QBYoyE" + }, + "next_batch": "1a_1a_b_e5ec87dc1fb83319", + "users": { + "alice": { + "user_id": "@alice:example.org", + "password": "CorrectHorse7!", + "device_id": "UPGRADE1", + "access_token": "mvs_7a19c309ef7f5e848267afa7e1fe9612b6dde5e761a351c4fa8f2533e2a05287" + }, + "bob": { + "user_id": "@bob:example.org", + "password": "BatteryStaple9!", + "device_id": "UPGRADE2", + "access_token": "mvs_227bf7f91b938d3870b3c8e6e5bec1b25f998cec088857a7ff424d48485a0044" + } + }, + "displayname": "Alice Upgrade", + "private_room_id": "!C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY", + "public_room_id": "!WLDmyNQlQnw_RhR08m7Jk_JO2mbtD9pY5COByGjLODA", + "alias": "#upgrade-public:example.org", + "messages": [ + { + "sender": "@alice:example.org", + "event_id": "$gbIXT6cBYsLvEAdCmvz_-Z-MLge4BnaLY_O1-PtdjGo", + "body": "upgrade message 1" + }, + { + "sender": "@alice:example.org", + "event_id": "$mC5T62vlKtDEaC4l1ujmOJ3dA5q-PM-ku8SS1xRZCek", + "body": "upgrade message 2" + }, + { + "sender": "@alice:example.org", + "event_id": "$h7M6qztmLUOCjFAmWrrrh-vd87KuCykZl7jvFLSK3Ds", + "body": "upgrade message 3" + }, + { + "sender": "@bob:example.org", + "event_id": "$GcOCV1AahAZenbnTVGpvMmYTyB2XNFzhRLwjkRT0Cy8", + "body": "reply from bob" + } + ], + "media": { + "mxc_uri": "mxc://example.org/eu-rBrEY7MSJKUy6o7pmMw", + "sha256": "7dbc7d20d54b5924ceb6ba419e522d699394d55f052f8b8815c15d199c357143", + "content_type": "text/plain", + "size": 340 + }, + "account_data": { + "type": "org.merovingian.upgrade", + "content": { + "fixture": "0.11.13" + } + }, + "pusher": { + "kind": "http", + "app_id": "org.merovingian.upgrade", + "app_display_name": "Upgrade Fixture App", + "device_display_name": "Upgrade Fixture Device", + "lang": "en", + "pushkey": "upgrade-pushkey", + "data": { + "url": "https://push.example.org/_matrix/push/v1/notify" + } + }, + "http_log": [ + { + "step": "register alice", + "method": "POST", + "path": "/_matrix/client/v3/register", + "status": 200 + }, + { + "step": "login alice", + "method": "POST", + "path": "/_matrix/client/v3/login", + "status": 200 + }, + { + "step": "register bob", + "method": "POST", + "path": "/_matrix/client/v3/register", + "status": 200 + }, + { + "step": "login bob", + "method": "POST", + "path": "/_matrix/client/v3/login", + "status": 200 + }, + { + "step": "alice displayname", + "method": "PUT", + "path": "/_matrix/client/v3/profile/%40alice%3Aexample.org/displayname", + "status": 200 + }, + { + "step": "alice create private room", + "method": "POST", + "path": "/_matrix/client/v3/createRoom", + "status": 200 + }, + { + "step": "alice invite bob", + "method": "POST", + "path": "/_matrix/client/v3/rooms/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY/invite", + "status": 200 + }, + { + "step": "bob join", + "method": "POST", + "path": "/_matrix/client/v3/join/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY", + "status": 200 + }, + { + "step": "send message 1", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY/send/m.room.message/upgrade-txn-1", + "status": 200 + }, + { + "step": "send message 2", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY/send/m.room.message/upgrade-txn-2", + "status": 200 + }, + { + "step": "send message 3", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY/send/m.room.message/upgrade-txn-3", + "status": 200 + }, + { + "step": "send message 4", + "method": "PUT", + "path": "/_matrix/client/v3/rooms/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY/send/m.room.message/upgrade-txn-4", + "status": 200 + }, + { + "step": "alice create public room", + "method": "POST", + "path": "/_matrix/client/v3/createRoom", + "status": 200 + }, + { + "step": "publicRooms listing", + "method": "GET", + "path": "/_matrix/client/v3/publicRooms", + "status": 200 + }, + { + "step": "resolve alias", + "method": "GET", + "path": "/_matrix/client/v3/directory/room/%23upgrade-public%3Aexample.org", + "status": 200 + }, + { + "step": "alice upload media", + "method": "POST", + "path": "/_matrix/media/v3/upload", + "status": 200 + }, + { + "step": "alice download media", + "method": "GET", + "path": "/_matrix/client/v1/media/download/example.org/eu-rBrEY7MSJKUy6o7pmMw", + "status": 200 + }, + { + "step": "alice account data", + "method": "PUT", + "path": "/_matrix/client/v3/user/%40alice%3Aexample.org/account_data/org.merovingian.upgrade", + "status": 200 + }, + { + "step": "alice set pusher", + "method": "POST", + "path": "/_matrix/client/v3/pushers/set", + "status": 200 + }, + { + "step": "alice keys/upload", + "method": "POST", + "path": "/_matrix/client/v3/keys/upload", + "status": 200 + }, + { + "step": "alice read messages", + "method": "GET", + "path": "/_matrix/client/v3/rooms/%21C7Cym0uRg_x2vckbuDPCwO4QgdsxfXpDiOi1WoH89NY/messages", + "status": 200 + }, + { + "step": "alice initial sync", + "method": "GET", + "path": "/_matrix/client/v3/sync", + "status": 200 + } + ], + "non_2xx": [], + "server_exit_code": 0 + } +} diff --git a/tests/fixtures/upgrade/1.0.3/merovingian.sqlite3 b/tests/fixtures/upgrade/1.0.3/merovingian.sqlite3 new file mode 100644 index 0000000000000000000000000000000000000000..719ef4cbbbbee48425858b788519d62f2b845e48 GIT binary patch literal 589824 zcmeF)4{+mHejj*I|NoKfo`2lg+1cfEYuqbKY!Up=9?$Mj6h%=YDT*Q`QoYk$kOWDH zAVC8BKb~8wXE%vcah$)E_)@vGZ+%IeDqrQ|IEgF&HrXU)pW~}NJGr_fm5ZG}cI^1> z;=4FG$4TlQKuRJ-O4Co%Ju}_?<@9uu0NxMpeE>Y*y+?ridqKX&8FQjkVrxe0*o`rb zX6zlKacpd?NdEsr$a8rb?Y@P`CFs^ zS7YQ4UJ!r)1Rwwb2tWV=5P$##AOHafJjDW^)lBM_Ut{<(U*lOJC1(m;iLF=i5}W1b zNx3xo-yI`=@PYsYAOHafKmY;|fB*y_009U<;Mo(HoY2oP%J~1d?yryO-z9(H1px>^ z00Izz00bZa0SG_<0ucBV3Y>g?eAaYQHIUakX)%3|Yq6z@z}*$4e1(nwYM#+mA9Ft^-iRg5V(@GacoZbtR>%JdC%fm zTJhwBJzuBo_Pe*%8-8oHxmMgQ%VeG}-nPg6Ov{@)*fN{9EyWxw$lQg$PK=sark6oh z;4|Du$s%hz+|RViQp=oh2fh1i$xL&t)LfF|;Vr=tDHg0;t*}>T%fW0}Uh#)FT=hUF zv{EYjw^sO4ePtsSzmPliqcZ+KuK!14`rjab;ROK*KmY;|fB*y_009U<00Izzz|$k} z(u7HKu}5V5TIm&UiZAp_ybzs-gCMZ$5KLI;K1R|D7@Yujs$? zF%=N2fdB*`009U<00Izz00bZa0SG{VUf^}oUobS>f7&lF{&~_bFk;;Qv|pe*{{PaL z{s;9h(XSdQAOHafKmY;|fB*y_009U<00K|1fJ`1M@cPNJL94xN6uu@FxpME>0=9ds z#c9oQ4!4bUv24cSGH23ROU9Yun5;W(O=q$;tBrA!Kb+lSb8>FZY9UK!vuuV7|4CgR9LkF0vk5mdj*FWT!KobK7%H7iY^k%-M{Ub&wq7Y&o+zm&-DC zw}VTY?VQEQvX-#%0}?*C7Y>67}Oe0sMSat;9qKmY;|fB*y_009U<00I#Bgaq~{ zPnI=<4fc=PPcR-OG3N$je$-9?W&D5a#wTQi)j|LQ5P$##AOHafKmY;|fB*y_@UaLe z_y42)|FKYGB@loB1Rwwb2tWV=5P$##AOL|+Oh6g`*XZZR$RE5Q009U<00Izz00bZa z0SG_<0?&cKiL9BuZOYtv__mg7v89T@-4&%gm$lgKjQbwTG7hKR;j9*pn4@+$Amk!P z(Ka6{yPNwNhq%dQf-9TNOvMsk_gZ!;rC`a&INf%e&Fye74vSspG7?uyi>-SN=H6bq zS*eOnb139-M4Fv!z7Vdkhr2;3edOKTVQnRIn=Ql+1V>ve9>$!T`K@*}zkTE=S<8o! z7-wU`ofZH3I#=Fe;(J2m$QjCuvAt}jowhjl16(QTW_GRH&6w5iJ4&zncJ}t;`{BH2 z%_2COcAv**j#f7_?MSBKl9gt^M*quWH00Izz00bZa0SG_<0uX=z1U{VtKm0Pu*B?~I`+rCo`Tqk` z%HOXn{QQ{qr?t2A-_-q6?XNEWmVQxRUVNZ`Tl;n0pI`jph5xMES~TgutKHWb7JgpW zUi>ymf)@lJ009U<00Izz00bcL$qFdX3SX+*b`Nct%+YqF;AcZeHQw%atcP2XLUONl z#78^TM#&<@V|7<}?-eycJ|iWYZf~w`KRRrh_luc>Sk7-v9*2*FShgT$B0-@MYjYKr z@w}`i;BD^1t>dD*XmjpHZF@V%CP&I`uVuv8e$gbl%dD_#9Tet-0chYJ!a2?8|Rfxb6IIM=J4~rPgNC zwA(6WT8%`qDx2$qD-zkM6pmY=7t{ouWGv#yhM0^^J~l-Hz9VzOn%v4UA=g2b6`Mg5 z-^gx<1D5=8{HB_~x3Mmnl2LL0fbE11%^qoMt5s`McDAE!Q|2JIWpPAu!M49zVKX<> z1Qb8mrdJajySH1eW+I*Qq_$!QomR6MjyGebYQ`m1Ec=4F*fN#b7_(V+#C2+dLncs8 zr~OhX*WBK=3WBuVsoJyaO=&aE)}7IfaHn(FuKHwi%%@com~z&w{U%##Tl|T7q*-Ab z$&JKjU}rxcPqf9jSBOXnOJzNo$t4%n1nXqe+g#x$XFo^=S@|#<2-M@5$Z=qEFPJEq zyk*ChqaI2))4Aq?nxIs!?xgm`{py}YVz`_sYqB1$`?$K>EbLdqxoD@;sus*V0D?~puJhzZO*6(JXyb=O9YQ%`2*)sT1a_<9eduvngaEjrDd;e9{Htp zi?{4*N8;0J0)MUJ+TGu%+qcE0!&5#idK?M1Z8z^Y`9R)Q^qKd(zCF)|GbHRysR{P{ zIZ5s`kA$pI*#X?$wJCoPdGAxjCkbrCVY;-gqpzOO72-&J3+=F#J4g9zq^sI3Kp|a$(YTR zXf2zsZ3&_HdNS%ts?YylqWD=r-2eZ{R@zuQ1Rwwb2tWV=5P$##AOHafK;YvL!2SOp zhceay0SG_<0uX=z1Rwwb2tWV=5cp&Tl=1(UX8-z_{#*J@-LLEZfNo9ui`u&OGmC$1 z@xh{I;l~&J^S?3wL-XdjUz%&oy-F6u3jz>;00bZa0SJ780{35<;5eFZJTFmv+4=s9 zN^ptzW;&}oFc&z#h0k>-&BHABv_*O!u9TmBv_*O9{K%wB{)TbC5rEy z-=9;0QzTfT_R_|qWIqX{b?mQMS>-Y@5tYuQi4+?Sfcno z{ryQLI7NabitpavpHPBRBv_*Op8oxDB{)TbC5rF#-`6O?DH70p|6dvZe`)q7#uomg z{#&}0#rXU`)BT{{H}@UguPywD_7~>gn48hwoZ}YWTKuWmpU{f)U;6}4_gFOqAOHaf zKmY>Io4~u19Nq2BG;Q~ny3wCdYyZV=^e5EZf1w-w3AOiccB4O`2LFw2^e5Ef*LR~o zp(eks8~q8j`L*5XPpHwq*p2>#TKx;%=ufEGKi`f1gxdXc-RMuK;Xm7r{)AfoGu`M< zsOdl5jsAq%{!`uPPpI)f*^T~$TK^N>=ufEmKmHYR=9_y`?SEzbe`)+TA9?(LHZ?mw z^FPdNP5;~JKRC@y{llr|)MqFE#^hHgCno;PM1Tas3j&`~f%{8JM;gTwa%hGk?%$>j zPH|mw|CSP*=JJ%{lIs2&w83d|Zd8KPT>8@F{0p?fX>$H~B{4 zpU@@y{nu!N)8zcKN^lBCn(@5*uPVW5uGcB9G4FqdHaNxQ!hM4hoTe>66Z{q0;1oj< z_g_|m)3h^aa;}d5(*&oi9p!zDqOA*&Y9<2~Kg*-5vj@ z2~K0Kj{nm%>}YbXj{nmHr^&fG{!bH}W^Yl)|7lvJG&xtt|7n8Lnj$Gu`M>+==4JZm6nXEB|I;+c zDe~SM|EGygk@w#CKTWHdqAA%M|EGygaT(kj|EGyg;ocknrzkDW_k(-m|1{Al^4=T& zr-@Fn&-BLsX&UAfdGC$?(?qApdvE-orfE)*_ulwFO>~O9_s0Ke8s`*w?~VV{M5oAm zZ~XrWwf(O#{!fwjk7fLyqTZuvq4&oBDfR=J=*sy2D;jeD|B>!5>2%t{;@?^P+(Kji zyYt^W_x0KTW7akEN2Y&sdUNVWC;!uAY~p{Mm>o}(h4K1S2%Ox0WtzKfI&ssCX>%>M zR1vtlqLk;d7Q3Bs-(y+E;j}xP&UI!h&#YJv<4il=EF_YNpqRI<6t~MkYxvOPbH`ZG zzS%BT63t{Kz^${H6*(L~XdSrs9HITF?BThCs_(dDOAE*4#8zS}?+SRDC%0ej<&vU- zdqpmpCXRxk5S+*nPLzZYWvz->3ouiHX$?cbVxuj^yUXjb)mTj-$S&tQ?$1C+j zUGOLJ9M{^}5IKuwB^zarvXCv^X}t4=3t|+?+CBw@|jF#yJaox zNS1oFoLP5n9$WppJ1e1PbuCwK`isZg0jbalhdpGA-|Xd*q9J-kF1<~$SV(THB_hY6 zSd$glqxyO~x3%j_R7-3%E0dqfY3?5ubM z8DWP>9tUHm`x+U$rQ(cdw>QWZ*Y|Qs zanI}(x!f=(Y6k~L)?;%k9$|L)1Ku2LMCu1so4j9+ZL#qkF2B9jINtX<-BQPw^fE`S zLtkR2<%>ESYi`zm5Y4yz-o~11cTIGKL)$)kT(Q*ka!JuSa78ZV3b$|X6cb_ZZqX9j zb^Ehh>CTZmXXX;+<3nrK*~lIq2U>ga-Qp&jt}(}~C3f7dZ9AJs4%>>?S&tsDj?Q{M zZb`JAN84)~`tqZQP+WOE7q>XnR3&zEn402%GS{_|UM?vb!dK+dbC?bUH+R?-(OxOGgOQEhny1pJu5EianzmNX9CEqp z*4B!n;Y|enE|)by>ZjPo(MsdMQ9AN40hh(cwR07*v@1n}X6sSp&_aHz>g4uZFP9XJ z(kpTq$hhl!~O15qbXUmC7sC7{9 zSmPGa(zeIT@%^nrOYktcjcl%7-q^SAY#n)svCpxh9(F<&vVIc||S@)lFCZ(Bml`a-N-oHK`pE58H=ZuF!5JY^$)_)m)`s z*{Zan_PyPsdV?w2t?p8Kz0(LWhw@IT=oBoK#@cZ_=ijid#E$FCR%~sZcYZcKdoQ6Iffz9#r_1&3d^mXO0`SPNW)l zA4{B5B)>ctJKjs!_;%7_+uT||I9xAm*Na}VUrzOMNzpXDB9{jtuk*+%Zn-zyNk$5n zTU(i9F1U46+AlGos()+47Z%sIs)AgI*9vRZ^>V-!T@h9?L5st`B?i_QncwbY||pJVVhqkmJ_{PQZzTO z$mM36b=&uo!IhPj_38@0lWaPgW^#@!<#u;B0=Bv*m~=QpF;9iDCxh!!yG)c1lU{Dc zTrOnIF=+7@?r z_HAo+S=r*6UM?wqj_-2 zIh|?o&n~>7drxnecj|s_aY6gf=f5%cr?lR=U)TTC!rxf@?YWTl56}P7r=szTOh5nv z5P$##AOL|+gFtACbjMTu2KyD)d$07OJ)U0kmwVA3Pw)0iy=af87y89sw8zt1{6a6< z^VM8x#$m(p^auOLxm`iRxJ`l^au>^;&Dn*EzYC zVlx>|mT8``($G(FU%u8F3%pztrFN>p%Y2#__*$Fh2`OdW6t{M*^&X3SIVIHvPNsRT ziL9OC+}B!PhAoR_KEnzrR^VBg=8+_o#p`a)irD0&l)yDOL8f_>$oDTyad(*O?5ZWM zl;)&&NEtKSy}oO0nun7d&y%%Z4_V^a8b|YE1)VvvE)(|##kgeepzM`2Ev5&`Nzsrl z%yugT`gN-n0!6d9Jkvc)=-0%RGg7-!J_B-`44M`??>|{cF9bJ<Z|o>d!s|pCZW8LfN;B^|vCf-$x-F&e=sGbj^c`uh&C}loT-;OIb=S$x zW$n6!FDq@e>qP9f)#i6edXwg{^~KEfc0!6KU|)ypY;Tj(r#|m9ndxmQ`WXE?@^o)U zrjIhbBTrqsROdVLwS>c7&5Y0$>l{`-3nwzx$(v@Z?{eALm2r`s@oq}F=ol7&8UL+5FAY@v%jB2mZR_JY&i{t(jUwP-1$1*6}gudxR zJMb@(XDg`3HT|m2Td_1F|5fss1oa4`U*SNdNip=_B+oohD^2=Uj;J&##{Na}7=vr# z_E(yZKKNfHPb9b|?m+cLG5Y^LdAxvHebM7yT76Lr{};(~1Jvq^e)ZQI|9@fT7suxQ z+lBGjw`c#t-17VfGru^0L;uGX|Do2W{i`}g|MJ3bE{Kb~?!W)oE_kpa2tWV=5P$## zC9h9z1LZB|8=i3 zY!17Fj&^9b9Lm+0K!XkMCaUd2dn1vs`PhzC$nw^zY|Rz*b+xnLIxGALodvq z)e?Q9hPM4bdwQtSuj+d3|C!T+hJJ+u^&mz2fBLi@q+jKTdXS?1KXrZF{(A7y+y9f- z#~rAVDBAxMr!^8i?xi&nMf-oesMbi=AJeCKG=NtB|Bbyi{zu33ujzkU_nPkCX#?8d zB+vcR&HuTD|7FfH&(4Z-Gc$i|)-m(?^uL(;ms5XvYJ2MC$=0N2;?GV<<9~EKs(DZI z&e(q;|2^)P$2e_>uW82ka+Yh!Re^upD?q$YsldrsUYjwPOqz$q8k-h4awA{4mXi5= znXT0&PQLj0bKa=W6Z08kp7o&5cSjlM`K=l@H%U$Ff*HEt2xyaQt$23w@!lK+246d6ajrj4F;YW2iKJ*<3X$OAPLC%PTKU z8@~MNmFTD5^2;aR|Kg0{)mJqi?4ReXdlbnpjmVLFb#9cKvqv2o9NB%PRGl1CBjyel zVbz%BDz(C3xyTVyBX_WsQldnH@#XyBY~zM+(-RNIjCag;ltoE>AYZnGayxgW#%1p; zyI+_#gzj8sr51s4={=43Tg@BNJtYwgWCU8(x98+~BC135ac5Y+~ac*#~g z*T1vuyfJOqdGkusUQT+Xy-zn!Ue?YS3WO`snAQZqbx=4i<0K*dXB9RJ~l|g;edrL86?Oh+a)iw;tSG zKRYF!vykJ9YgLG0b}0G4LVcTLAphs5JKy58A!)r#b_7I<88K1&@b=t{VR>2e_1-bT z<;bN#AypHLTv?U|KFtqW56&daYMNAmm3y^MUl~@7msDU`u0d|@A5pWBlMTtj(4|Ee zPfoG5!7Vd@N6tD~PI>ACIpof>b9y)qEXvDroooxXmMUG?n&*p|?<~JHKW*4uz6^U% zf*hFxYzq&^X4LBLgDh{5vxC3`^N24k;g#FtarvH>^-AYu_v}^Yih+Wr zn1PRTC&sB6!)vc;9vW1=?hYgO{+S*W>rK6eRhE}40^1&W&DysgpKS!?G~3&AmzPb` z(+1CLm+c+BOr@CKo0*g7gj!dBFrk)6*{rhq*W_Sn)M*#X(wCW4V}+IFCOM~%*vnUq zf|wa`0FgCSluL@CBa6+J_zRaqN+H=yjc<_WU|d-K{OCBX=qInL=+&a681 zq;?9BCIFXW20nh_B=)*`8he;mOE@rd;KSz!OF1x)lyl(H^Sqqh=3K2^8R|1o8YJAP zi?K&)VE0x>uDjI!4rK-1HjWx{dHKtqn>Os5E~~t|iOxGoPFtquQ(iY&uB!cWyig7LJKRL0)q-aocMy57{wcaH7?BQBptH4I1Z9A_a&^W1z@^Sn|g*M;gs zu2RhTd+W*TYij%a;gnjQ^Ev15EQ6&wpK*D?l=Xb1V)X(nFMIA@Ri~?a2f5%7$t8U% z%huS_c2(ab=Hz{odcyjkuf5uzW+;exu)ss}uPFEEUV!s`_hK#3YuKN*f=`d9^YgRP zq<{3iVZ_n0yzE@Ps#V%ApDEX zgnVTOQ8)9K?;T2i4{2eOCQx_DVY~A^i7eaAWU5;uU$yN0ca_MG-swadoqSe3^{9Qv zgJmDc!oWs4+g*D&qz?PyRdrZ*7)dVhl@xRSe&yudJ8B*FLAMS&pL721dk44H`3&j} zcAoUIQYr&_bbKFz;`yGd>cH;S%u8aul2SW?$#t!I%I%x9e6sPTS_eMt?#e?83{Cjr zAlT4+;?_Uy;x0-WH%fo=)r$-idvMW_=}A?#bX8T>7lItc{h#ifL~eJhvW$8t_s{Ep z|2>zMTIqG`Tk`Bk?kmHp@sdNo!uDO3b+^hzuYqGE%L(L6Hlic=d@0hY;kk9yUU6Cr z*9sEHRro9c8JhIw2XDQh?iIi1&4CL3EX`1gr9q#e`2^{bdalG}`Fd&akXbdd*(|9# zE_B_j8rAc;bfNd|yzoj*mQ8iL!v-S`EJK2Sc;8U3Pf%$he^@utsxeK@jhD8hl{0ll z;JdY;(vf&Y)<|u6#oWGq>dbT1*cCi8BbP=Ohk^&2yJ{U!li0G%tL^gBBcgB88xLRk zLif_>e&1oyw?KD~@0(y8+yRE>UtXP_pN9ua-gh;BamMJgU$<)D#kzdSaMr+1FkpFh zzB$iK-|A1>7<_hlS!w@|-FRABugE6^AOHafKmY;|fB*y_009U<00P$$P@eyf_WyO{ zkOl$}fB*y_009U<00Izz00bcL^a!ZW|DT>8)BkDxysoMJSK7VBzqz=w@Fy3h=B0W4 z-1pD^`s~5XcV^7ff0`_W7X%;x0SG*U0w*_Km@0Bby_~;C^Tc?XhteydDIWIbJh^f6 zEK3yWXtPA|7&yntjT>iKqDV)ZrM*XgP@MhbhW;!|6zOQQl(CzA`OONqo!{+9C4RHi z+H9J3Tcu2^kw{i$b6s#nB0H7BaVumyxuHAD5=A=NEKxjV&U$h~dzK}NbhKHbc=DX( z!30E%QZIWnKzm_I8u#nbC(9vrXMGo?~0RowUQFo{D? zwAG3S#i%WFC?6b#8F$qr@zz7XCtCOIhh_K4jk&WdQ9RR*HcJ#gUhg`&F?*IJigdJD zO4Ujsq2-cU#$F<9gKcq7){%!p)Xn$uhsbjV8K7$v%C=vu9009U<00Izz00bcLI0UQ{MGa+F z@(ZVFDEfMDo~EJbjJhN8bwdzyx#LvZml4MmsT!f6_cF1Y#A zG!%VgbEj!2I&77v8=mwfTO+(R7Gj*DVq9bMUGz~>h$HZwGimr%p%S3Tb zIp!!j3vmDc8S#Z%#&3a z|DVu1$MjD9-_XDJoNi(i8v+o300bZa0SG_<0uX=z1RyY4z&pt?nvd2cFyWcxOd6^t z{Dr=N@pY1mIjRUlodBoz|Ih#2=z?M@1Rwwb2tWV=5P$##AOHafKmY=DQ4Y zm$BY@->r=QYxKW3M*iRh0SG_<0uX=z1Rwwb2tWV=5P-l_BVd|j-kN*lZh_PL|DT%T zA)^q000bZa0SG_<0uX=z1Rwwb2v7^0-v3XX5D6dv0SG_<0uX=z1Rwwb2tWV=5O`_? zF#iA4Y${|F0uX=z1Rwwb2tWV=5P$##AaHE~jQ?Mo4hbOu0SG_<0uX=z1Rwwb2tWV= zPmKV^|DT#og^WS~0uX=z1Rwwb2tWV=5P$##t}TG^|7+7BAp{@*0SG_<0uX=z1Rwwb z2teSe5m3heHM+?$@&_*nKmY;|fB*y_009U<00Izr#sxlTli!km{r7r?e>5&0@kc!y zKIXk!tdJKazE&v7x9;7)Rk|w*rMnI0Zl*3ZIE&rRxJ@i4o8$s(xwT4Wm$-ayiL=S? zuHMRW4L-xA`0TBFw@yQo#fw~(O4WE}lQV~NyNyoU-z2fPteTtt0Aag-&#}3&#vI9Mw@Zp`wIz#XH{kcU%!k!x zz*~zS*%J0dWWyofdVlqn%;(E&tuAqDo`y}7oQ_yae0^Q^?<%~JLrR3ar#;2r~IjKuNqAc$# z}KRu;b$g?|EF(TS>HES?XZfqrk(M_f$lCLLR^UDQ$wE&;k z+F}~k=vFz}3Pqxw{9!B8KFkM72ixVgxKU=jkyg%f?d z9id1wyeGN>k|SZ;EP2}biruSxxu7@x|FdKIpZzc^%!dF3AOHafKmY;|fB*y_009U< z;FA_`k~=uwzG9sJvTh7K|NoQTa#%kEAOHafKmY;|fB*y_009U<;K>W%`TtMe7;At4 z1Rwwb2tWV=5P$##AOHafe9{6K|No@70M-uy2tWV=5P$##AOHafKmY;|c=7@m|9|qv zSOWwg009U<00Izz00bZa0SG|glNM0M|6kSonX&QT9XD%!TW{C>C*3>R-_>ds|L)=& z3%{|jHveztJ#)V?`;FNzP5<-hzclqvroJ_qom`o)kR;wuI`7ZV`a{8K1N)X{j4x-o zmRuG18kb`0HBtSX%5hmvVr#ruPGvc{#*?3~B=6-EbIFW|+c9kj-MK8BSmshSzQm=J z-{nj(10UTdTlN{lojaO`yEQf~a04?3KDfM5pC{%s#ysmmpYhhfyj#XE_I{Pp81p4! z#_%rrKOPLO8qaF0J5SCiT)HdiC*YVt)Eft711xt_^zvOHpThU_q0re5P4Tq?)%g7W*PgCUlk z+tY@fH!t&&xROZr$`Y65*%Z_NDg5yJ0yBm;-_(5lEmf?4UjO^hpiTd*Q6T39@2+!r z?;vHWaphX7)~*a;$$W=PrQ5ZOrCc>u>#W3<$x*}&?QVjYVTE21FRsqG;oJ1YgE8Zs zJFCVVSIZQ*Y>KU=N*5TaFYTYS%HmV)s;zn8t{`mdmr5c8V)!E98VxOEq|zCsp_0#yQ_DYf_yf zduF{Nh-|j60y;h3&X>Hi?Aw?&B(0Zi38kV*7Rn7H*Y?VT}Lt(vbF z)vYvQkr9*pgIjRK5@dT`vbjjrs$^%7VAUvzV(DyeKP@k*ZM&HtG3U|GKfN`ct)oBc z@~UxjH|h)c!xv6ky+q5#sBhC3^@Y8@ef8|sL+#C$&x*NfJWH_~Cfh0KBPU?5XWvVH z!+SZtK5gK=kFc$JF}jByfl4ubwijn^o-BE03|_D1qg9u9sqP+b;XwdBi=nmwaTY<&6{9m(@M>VYfaRE|NTAlJ~<(Ctobp`Mx)@G^20pFeiJyEC7E ze&?k5){J3!S@Xg2S^3WADk|rnzC2vY^Or( ztH#0AoqaxA!Sb^A-n8M6xr{9%iR402iOQGry{5qMw6{;3?imBaXdb?*@(nLBJmvca z8HX3R{Af_N$Ed5h57p_N2f7#kN`&QQx9h4#P>vM@rJC)PeBk3-CtJ?`M$o{_fe-fv zi#jmx@=~5{qrp1twB>Sf>Z2F_EbwWlczY8Kth8!x_gqBR%{*3hxt5fwAp=wZAdu-2{6y2&f$-(dwl z!`d^yjv zWn*t7+t?e)HkP<7&(32BS`Vf>{{I)p^ndaB*vU~!2tWV=5P$##AOHafKmY;|fWXrs z5Sl#s!rWu)p273~pN_4LTtWZ>5P$##AOHafKmY;|fB*zOiU6Mf|55ai76K4}00bZa z0SG_<0uX=z1R(Hq2w?pG>DW%lB?KS<0SG_<0uX=z1Rwwb2teSY2w?pGqv#R!1({uv7L}h2tWV=5P$##AOHafKmY;|fWSu)!1(`1(L-7Y zKmY;|fB*y_009U<00Izzz|$ds@&Bh|J0X`4fB*y_009U<00Izz00bZafsZ19@&Ava zhqMrY00bZa0SG_<0uX=z1Rwx`r$ay)|JTp&jp_fEKBqV8eqQ%`b+2jvjP^^5-(5^E z{QHI9xA3|7pCPmHf&c^{009U<00Izzz-WP-e&%H>^ZM=`wq7d~xps=J@OSGKL1eQj z@=<<=75EHyk881|ioo3!rF=Z%k9szI%zZN3RAJl7qW2^&%Vug^_MX6%^R>d;4!84N z#_6`XY&Ms}Vs$!L-OL+K#{KG@l9=Trw#JL)5|`!K({-_#3|AomhSwK5wmYI?CL2AB z7X;S266*RqFB0{kvMi_&^OFn*)Fa}>Unmv z-H1ilESq3rjf~HcbhJJ7*tV5(xONhika+Cr1?gLt)p5KyV^}e9ulB8rYjEXSN+y53 zlNQtcVsG#rwiFMiV=lkLwRzwr*o?zhQd)b+y(|YL2^tnCHtE z%hIbVET;$ed43s}ZGUp+WxL5l4p)+1jZv35DJ5}vUaoQE;Jz5`;^(1^F;@4+1j!gP zoUx&!g5_=;_k{dRwDZb zDSd;J3Sx=tzEdaHD`#ko!{WmC|3{z3Fckt2fB*y_009U<00Izz00bcLED7NI|IgCa zL|GsJ0SG_<0uX=z1Rwwb2tWV=qXjVjKiUOTApijgKmY;|fB*y_009U<00PgFfYSbd zY1}n7?$ZBHdX4UX*8acRFKMS1>kG5<-Ogvo_>GopG|#c^4BK*l%}Tf z&HuIezebX6JxdNK3!T8juZCs}Z@s1Y`eBVt3tWyny$LT>li0FM?%g9d50!4+CGFF)S}@z8+GEGXk&N4_IrJ357iPi5mnRoJkzj z8=+1C&kL^H(s<^2KDm42VXa*mGVQMXVj;TA4~Nfk4RWjJ&}Gg*c`Y?<@V>4Ys*T5B1Z1jBcD?hNzCyAC#RUxciEHm2Q!A(U)MbB?%$`=Pyfvff}c*lNZprdl2=}? z2yAb>{xET6po?u>z9G$)+82gx;Z z_i8Cs6N_9q#Z|;iVR%YA+RbH?+-ED5SgAOADm$3Z(F0r5pqBb ze12$7cI%|q)KU-59sKlQu$+T4uRKtCj2A1Ee*RVCX}rrek(kQ~fV;@OY82DG)7&Me z54oN`<}$U^C9}K5Ne!Pz&NpOtkt{*&t10_(G@J*rIS1FeyzD)gHXJgS0oTM-uh`@^ zSeaFQho{|tcpO)&%CCP-u-S@g@bs0S0d zWE}XA7$hE;M|8WNFWLRNm!KC*H|kn%*||S$*m?7^J*Xm*CswpmQoVbH(D&)^#1bX< z$iJz1c&A$(_RZ^0bud_r{#m3HO5n4Lq`u>T?CKeD_!^9CGQKu+jXAosub1Sc62rHd0sZ3S`# zHzK!>ygx46Bh!Y!?aMYIc|r)668I8dBlTB7Jj| zV|>(BBF{3pSm==r0Oj;bF6u8_`d+LLl{KsyNzCQtP= z_1`x(sh`#8|2ut2Kdb)*y3|-20uX=z1Rwwb2tWV=5P$##An_cCOjp1NsL3vuD+ zCx=&fT7-FhlC=L<$MmcEf299O{RdR#!NL%L00bZa0SG_<0uX=z1Rwx`XHZ~O=_1fP zS{K0-DgXCAn!&QtDWG|TPJxL#N^gMXLwf_pzgOu7P%`?V-2l_d{r?*M|2{_k-~|B) zKmY;|fB*y_009U<00Izzz*8phJ(?Nj=Brx$%fmze3?r76N8_4nQk_%o|JUe$V~qU4 z3jz>;00bZa0SG_<0uX=z1Rwx`r(WRY31)7%GoU;E|2t#)-+Af_hyp+W0uX=z1Rwwb z2tWV=5P$##AaGG&N;5G=Zf8LI{~`=NLjVF0fB*y_009U<00Izz00f>P0k!>aadg}N z|7J}8Z=RvDp(qf500bZa0SG_<0uX=z1Rwx`=TzVoGTd*qxa}^+Y;jnvHkaM*WZX8V zR{yWZ#`QlwrvGvMKX^`QP;3Z500Izz00bZa0SG_<0uX?}r$XR+G*iqS?akqv_dNoi z;f(Q0UShM{d+!^@r%YP1+=Zw9zxO^V#cP+8M0x(-xc(oJ=l_wv@PYsYAOHafKmY;| zfB*y_009U<;OP-~X~Lwr*b6Xz`SbrKwdFDWecfNy-6U`Df&c^{009U<00Izz00bZa zfyW?lVw{{cam#nO*Y8|xpS{6%*it;4j=B5}*XDueC|(cbcD7RCo&5fGr!7evF7I}r z<1V#g$uHGI0HPu>WlWpnPz1?y(6X~2MwG}()w3^Luycsi9GcKuO z*%!>kmZ{9fn9Z^y9*_8=o(-Rcak_0To6Y60SlrH&TNATPFN>@)zI=~sv89T@-4&($ z5vJO(`mm{$~CvQt%4wJcdGX6dQ;krvvp^5Bi!j6wyQqb9P{-u$@FE?bAHj*2O&A`rnKAvccajy`O5|+w(GLuV=$fT2GQuCxU z$tRj9Fa+fQyKrC)ZHJ` z{|UWG_Z{*EF9<*Y0uX=z1Rwwb2tWV=5P$%kKz_#bI_XFl8i_vbc6dCU3ARaohUu2-LjVF0fB*y_009U<00Izz z00f?Ifs+-DmU(OCWvg3zv#8a+K789g`631R?uH~b_;Q|S%XfQU;5hvrN2M+oid?%; zt5tfp_8T|(R;?~^Mo(pE(gxQcU((>p%Hs09wKXO6-M+7Etffk9P2yYXjI{={Ru*e~ zu6^2#pp5@(^nYiJ{J{$X5P$##AOHafKmY;|fB*y_0D(`fz}r)#w_(H=1s<)JLHYf^ zsl~61={I$MK>Lf@&ycrxK>z{}fB*y_009U<00I!87C3QF%`qHz^G;Q37#;%|DV$R^qB6a^>69l*Zq$EZ&4rSNB{u{KmY;|fB*y_009U<00IzrG6LR7hSLo9 zDADv1O?f6wB*ulF9hwIdPL7U1(Q7g>Is!%S#rQj<6mvtRpy)+V#{b7|Jekdes1Sev z1Rwwb2tWV=5P$##AOHafd^!Y_@qe`cKONfwxqtu!AOHafKmY;|fB*y_009U*DFN;H z`(rO?{@mF32fA0Zzc%+^?)L1%S^vz&?DW)Mo&KH4uTRw{{=(#!$G<(nYyO<>?~nPl zKc;(s@dIsqaeiTY{+FIqt`PU4z)4h^F|4g=9`4uJw7{{Iip)vMlOI!Jy_ObRLlauw zsLvDg8DpOHpwD<~X#OqZ7yG91S!2wXh#AAXPMWTU_Cz>?w_>vb^FcflmdG^a=(_>?AvD1d#Wz860&T=ieD)2Qf#nx-0`Z<*m zcx6rG$4$xFn-NP89pxuh zM-=dp3f@b*YCK(fdD&N;HYBZ=m2yNDQp|{n&6CCQjDcY^CtkI@-T0*J#4Im~b+T8e z8>W9sZLqY%3y{K|=Ciw*`!=Go)y{$saV&eqX~Q9NS*eCsm12gc)lbTTx^I85pq8nB zh3fmE3DRJRhUSw3^-mk#OS_q(6@kJu@^(e)y&18H3ZQ`TA=Y>jULT$n6|S5mb|| zo4cxP>fObBhOO~pnVg;mJ{~{JXVtQOL#vi;VCKLFez0r<^DZwNWWpb~xYp$WjG9Ul7Rum+rvb9dxz7&N(|)PU}}bySzL$ zHt~gv*G%{f0SKr9rp&a#^V((gv+~S%P9n8e?@i%@e0s+4+H0EM`=(lN^(OcJ$q&|5 zy{VViQsksL;>4yT8*!OcBv<`|^~$PI36SOGiomv0WwvzTnr)yy$*NDof7GQPt&S4d zi_M?GRkAfURhNXpTn{fouI9;;_&;nZrQyP+j%BfY)so#(Uc9aySZCiwT3?0P-4>*F z?&>cuIz5u@e|LkvLln??CfB*y_009U<00Izz z00ba_@qg?C5P$##AOHafKmY;|fB*y_0D)&;0OSA9{x(JlApijgKmY;|fB*y_009U< z00PSR{|hreIi~-^`Ww1GrTe1xC$+A{zqgoJ_{R%J^Z$1Kc<#67-k+PF{o&cql7;Ys z00bZa0SG|g;}dUmrMhkR(3Z&@ZAS`zHgr_u?QX|<&8Gq1|#QS7QPVHoTjtwh!%%M8f7{J60jfTdT4)SM06&m^`gECDjE^UZP0c&`lNH z-Bo7nW?z1@!foewJ5q_?EVVY9rrlO4(`qD=RoPq@T#?95rEuH|^;Vf-%VL>4cu+{O z0?*1z6cJ?Y*{1CzV-ZI-#AIypu_+Sp9hno>9z4cba zCV5h&K%R#x$V(K-zqUA4yu+wFuWw^rG9{zp{sG$w9hyDT)>f<5sO)S<+osGxZp-3` ze8S*sb8n;A|@cntRI+I%7Qn|X5+86h$dlre|a;B`wdbsZ6>Ta{J zUk&G?oldJ-Fq?f|?i9IK-itp^P7&z+xwF+%{2cfo_r=vy1S-y+RaI?kRi%7AzN7XUY`Yiv+9Nvd6N)C2G#oU0bcHD3VoY&JNvFtrQYkE=f3IcRU-p zb>Ftd%2~pRa>BBa$RCFaR+mW*+MA`_X0NPVIU}_z$^$KzC=$!lXNQm{>-Tes;884p z;5DC>n9VwExr_O3ef34%%-QTF&x5cKz zQ$8$u90|5_C-1dx?^K+zqx#ly zJl{+fQr>#PkqKnPBd<5%a|A}!QWVL*Hhxx1SzO6IOKT^{IE46Crr>us@>RiN7AhID zxe~2q^R+D@6kkt9UCF`2hSbLy?jF}-OBI2;D@yq#noydvT8i(mrFb|UbNL;v%>&O- zydKEyY^B0G`TgxqTaq?h-t9ogU24TnGtjH0C=x2;|1Zq{{jtUG>Uo`K@jcz|*BckU zsr$#;|FrO{3p3i>LPY;JwJ*;9eeKUIT0Z^@9;^`p5P$##AOHafKmY=x1l~~^o0{wQ zi@mU=G&R%!6rEMQtN#Ah+|97kLQ?}#bUpO~4Y$y4Y$~lTH5^5^QZL+jt4qJ5^fRep zD0+!{VJ`GD>CUMrI)8ex`umQwF{P!YW{Rd0r`y$cS%1*ty3%@5gHm+WoR(DW<5`H3 z5GFMQMR!Xt#OQvT`SXwzT_{(EoIBs;DLP7eA(j4>+4EhCqT8bv;JnXd=6rXc=*Z}W z=|6I&&yO66Zi!x?;Uj12ym%j_zhd&dcodxoyuwlzNY% z$=}Oex354M|9@rX+hh74)xV?rW!(?xjM{H&zp5Qu{L#gC7Jhl*2NsO;-=6>K{Mg)& z&b>4H%d^00Izzz$Ys3;FSrkNHwfa_vL{HFDvmW8b!**2*u(LUQ*&y zw3XD0ok#y_z=Ic+_!NyNB|gRC4_;8>Q?#m-_!NsjxT(aaXlW_&DHeZlLy1q(08`>q zEdD^R#HVPBDe);5f1p$1AG^0(u4W>g^Q5+72c1^48ICt&rfS9|RV@30x!5w5*%-4~ zcEpwV6z#ePS|$D=6DX(CeyNmeZf{!!LE7$A?b-FFv>9jX&ge$C(>ZKceX=>`qm93) z#5d)vTl-D6)VBB&^+>bAHj*2O&A`rnKAvccajy`O5|+w(GLuWv#$Qn4uaj>B*<9f! zXFo^=S@|#<2-M@5$Z=qEFPJEqyk*ChqaI2))43*X{COol1t@L&IVC0K1JJIiBGZkgK;H3MU!5MPqFv|jS`=tb+5#ySX>$Ze`Vq4$Fx7Ky`}%A?w@LZ zb@8|Ki~9271O40Duj~H&;twzUXWiDKN&j8#zRs}l^Sbupw?EMf9;_S!5P$##AOHaf zKmY;|Pz1~qoQ9?+@D()yMPJ>^Y66PBx|h@h6uoILstG8%$6inqQ1o}*R1;8iVBJs? zQ1nRY)dUnhQaUvOMHiD+O+e9Sw5TSa=mc6&6HxT(%&Q3~`d#MK1QZ=9vuXm0o{jT?_x?#K+zd7sV1Q4U6@c4P;?iJuT5}sH0^(7{C`}(Ii~*>`3o-yKmY;| zfB*y_009U<00Izz00gcOxKF+Xz`a4yNkr36cb|L(fO~^tOrPfKP|EoKxZXFW_vycN zMFBAj0uX=z1Rwwb2tWV=5P$##AOL}PCpk{@q?-TZUm?-wo>co^8UG)H~aFN6>dAf+mTBAW~sH=H0`!ZnN}l_ ztjgxP;EF_cDuv@#=%$*WlZ-_i*$|Vl$;YNhz;|R$Sd&{hCgeJ(vSKr6;v3oRaKMs3 zj^9ud_%_xhQ!*;0D{MxuCNOhlySN^y z=h@A6BNkz^Y=Vh3GCoJr(e~71+g8ru+DTMG;;~1kCOBjQ<#gIFm2%DPZL1(i+nuUC zyWW&G<80j--3WI&hwZ9QHphHgHGwH--P&)mrMAVNs7IO=wvpUOYzB7r^YKJmjC+NM zl(1CRlbKv{QBAPkC|PW-aFeqiB!jGcm<`{I6e&mu8g&XhG-57&KM-E9{3tKnR<(`i);X0y-B#pl%o`BEvBD(?Gtn8cwc z+G@pvV$_y7ln)NWjJs-*cAv8mW@RII8?B@OmfiP zEbTUD)C8Wa-_Iq2N3r~Y^C&H(Ji(4V?_f=VddBq=MdzWW9|u%qSW2fYH7BI@i^P#En%_KVzG+F zU;XQ2)mOz&XxK(imQ~fBED+zz`XQ<&d_9`<* z_D9Mj@3CpV%A4bSo-L-UpIc|ZvZD^Rs?%s%V=BB-CYSO;!w}8l5ya}cg6vkb29q0g zv!mO|&xvonr-q!)G^hP+Ih`#Hs4?SJrFv!3tjoPx7}$0po3B+vQeX8)aQ*)?V!PpO zLIRKgBmfCO0+0YC00}?>kN_kA34Epr!1@1Yy7}SFLjsThBmfCO0+0YC00}?>kN_kA z34BHf!1@1Yq@m$$LIRKgBmfCO0+0YC00}?>kN_kA34EprMDhQ>c;^oe9{l6|f4Tp& zcYE(1zf-#RKlc)E|NGn0TYvXf^3A_~^U2+RzWe0PFTqLh0|`I^kN_m`nILfb_URjU z537%EpPs0OAg)wlHL*p>79C~cnsd=`CYCJ0QX>X?-{6)T7r2569YO_#ek3Ko; zQLA~ho&D&8OC?Vf&QTtnzH|DyO>NK@)y8u~dm@NSl^3^*JNW5!wYijaUv1XOs_^(| zQ~E>|9-Y2BqYw7fWpR4z?%~(} z`1a|&3p9?)x#C2&P0w;BwrT31&X)^qUcB--K{pL?&zpq96dhTQsE}O+3FuP8JfPr`f8%MvQ%KPHx03!_^2G*Is6uJ z>jiW;%L4W^GNuy&6v4Zu9Zb-DN56kMTHQS)h+98k)@b&fv+vmVwz=#(1z@>!_?*r2 zdpAVaH|;qvZrk_v_~^_|T|ImW>y@G(A3ZPZBKxC9N&n7a{^5%t0WP9y0jr0e|Mm0- zndRNXhYxT4h+4xqKYsr2ceii){4luD7u5LdjENgq;?=u3dlp?)x92|G}Z3dIt| z(EX$L-+#O@x;v+UzU*w5_B;~;ayc$2IeM0S(n(v4&+ZBk-Jn|E$K#`mr`SbZA3Y+y zJBN+;Uxa$gHcrs3uOFS-?%l)p-@o<4udShPC2oBQxAAYKfi^fdMVs5ZbR!pv*uHVl zHEmT^jjKn&(_k`l%$?(F2+MQBxSea8H0Qda4wUuca~@I7okJ%1B3_sJjp?~lbG6lZ zH-~~~{r{`?etiJu{r?4G{r&u%KYi!q-hbcw^@D$X@V5_)2VZ~i&i#M8|I7Qq{mT7A z5F79x-u=kN_kA2|xmn03>j;2}H5N{wPM=xv$2}@Rhh3*75)1 zRB0XmFHVMa{J%IE*75)1WLU@li<4m;|L>1tgkW38|BI7h9se&*hIRbEI2qRQ|Kem= z$N!6yVIBW3PKI^-zc?Az@&DpvSjYd1lVKhIFHVMa{J%IE*75)1WLU@li<2Rm|9|nF zpC6ph?a%!H`{w^Q+x)@WK?0BfBmfCO0+0YC00}?>kict3Ag+hu#?Jp^_+d8l{}>rI z^ZytbHuL`&8Q}c?Wn$<5uFU^$KlrDE2mb;72S1PiBmfCO0+0YC00}?>kN_kA2|xmn z03>iz2|Rq`{6GRXn!vNc=id@V^y6k6`4(Hcx{)Xk2LA0NM0P?TWyy9z zf2MUEr=(7KG;E_M%c^Qm7Kra z6D@r%Mr{OY7Dep^f9W4acmCm?mV8R2qX6#~h?f1V_{Lh;4G8F1}Au*fS7_ zdt!T9L`XH3AUgLYJp-}r(Q{|9$cQ~ppb{CuZ8v5OeVh&g%9{7M?sUc@UY@gEiyekj z=6XG&yLyh%p$6Pk}WyfMihu0sVb*1_;J4ah?XuHOO=( z<#hpbm7i~l0@j-2!3j~Z8zXU%Cpr`b#Evcmid_GBsYGz-%*Lno_6elxEwp*bB0Kp? zw`@C1im=W6RF#J)-z+f+3gc?9&~`?4e1l5fFp_Ch)yIDxkW;) zkNmWcOvAA*wOWNL?osq8ooVzb3nfN$&JGM<{fDExm_Q6gK>GNywWuwLqNO&ZdeBW2 z7b=q}P}lbf%k;scQvPXF6mrQ9ZTZ^gVQ4u zOhvndj~Gj~IY9)^E?v~8K$MDVfyAh*U$V~FUM1$z)ofczH|>C#x-3h2S}7^?QeJ(Q z&U>?QQtx5GSeES`{Sr#tW1VA!)w@BX4WtP)Rh8RxP0r9NfoOP>Fbd3=&++m=O)LnO z;uj<{NXTtdP?sc;(l(YFU&P=x&as zrFm6Y&WVDN=>?J`8br2Ou2bbuQ4_h@e8A-MM9VKH^DZ&VrCZiKB`dtbI%Hnve5stO zXmj0n@-OkY-ux=d@lzB%Z~pcC|JGj~yo<%-*7d*{CR;2q&pwt|)pWqV*O{sT+o?g@Pc(yvW4rkXPOo*_{H!7_|W- z6KAp#oi6M8WHRrS>nJ;z9iSbgJ}A2r^V$B10}TmQur2)>5=_F;}u1*VzUP-?0s`Z{`(@j;^#G zAR2f6OAV)0x>V;9CSlD3x?O0wl(k$WYx#nYuOy_A&3dU-BE8JBbUB+%u{yWt7siR@ zAgzxnw3@|=Vjx+j-m<$ik{Y*%7;tOsjE%0omGr{r5sIJ}Poy8;err8@_{sy&0b(d6 zRtMO5A!@e^#2BJQn*J^JnC&s!Ru|ZrHIsY$083LPqKMmAy+O__#gzPkk79$SKzr4; z?t0Ra_u1xbQ0rgc2U5V%bkz$Kqy!tE33P=)nkKdgIz>cwX+z;Mlaqd@O)qK=g%w2B z%`|S%xU9;PG8rmiyPO~D>=NU-mefZ4OmbBmE<<8Lv_@j0lkqh0Yd&~WFj%Xx8hBxf z!AvY57=!6ml$A!IQbCYBqtGMjl`+-%zxM(XMb4kg&%XI))CoTRQZy`!LH9N41ly`U zn}VE8K90J;TXt#Gm))!CphgU)RbOxumAsjr8X7C%shX>3k}1zkq&#Dgp4}hI!8>DJBQSv+4bQaSo zLMVh`n6KyLQ8mn1VvywOnmOvx{c(_=@+*g#rrVk@%e0#KLZ=!;n@MX+S#y=o&J!JLTLcRnNG{NJ!87i5fl|pwy^dEncr@ z;3~Rmr759Y7Fl}Smj{{gRGUzANXTjKr0TiYif~KgvYo^7MmA_OH%QOQqLlT96r~SE zsy;~CLBkvd!%DW%vYNQX)fSTFSru#5B|5FKw?yYeiqxmMaM?hIPFyQJs-^r$UD%)F#6-EyeW6%^UmKX4A#f$>`e^tye z7XEIKm@%tf%P4tGlI*0A9QAOHv#lUNDPAs&2qNe5)nrl_SZbo=8J@$>mTuTyFyzb@ zGbWh&^=BOxSA!9TmnylzFsIbK`OD1ww_5ROI&&`PX#Ria;DdvAKYaV!cYpfEuWtVe zeA)AP_T}G;WDlH|U{GEx+a!RRL9L_C$I8Ej|-EdfOYhb0;qsuilbM0^< zh3r`>jS+zJielsBeAT`xwK~X{!l=nAys>O#Frqf%8MfV@HnQQ+AyOk=tD~b?wO*ix zSbjMwPpJUw2R%cP^;KKTj!{A^s0p_$7neMNSs6yC?Qx^;HvY~4x`=(v0?O?0Gu7Gi{#n1G85c7!5GCD)rbZUTKhhcU-H^ z>8ud;={9AKxf$^4e6qtF%kD_-_`M-oHKx*k(l7${$pf(GJ7p)T+2QW2Q|6)7v z!{g}1llWQd-SxooB$9Fre6Q`m@=VO$X1Vd`49ZdJ8a?k{0;fWxS5{YTB=k{#)v9(^ zxhbAl*7HHRfvgHb-ZyFsS&*a#a;=^-d-QxFr7At2(*vx`dTrn0mZJh&s@8il!v-dk z*V;CVS-BA}jb4$ScRZAl+GB2WvYTnG7fhM3m^2+@A)~zCSg{ns5gL=H2fYBH$7Ir} z4(er_7&e3E*g{;iotv@ z41m9DipH){^!}yvJPdDs&v6|KfRh6q|+yOHPJJh()ZsPW|lUQXN`G!K^bJE3=BckYqbghaDZMWjg z*+Lf#@7FHd7bzM;FE%zkYu$UU=okQh*A$Ijqv-vM^!ctbDk8~d*=(0eQ#u>68k?Ax z$y|?Pn{|s-!7jCVDyNk=&>+{0Ify-qjybM+1q*Sn!bV%Fi_^L;=~`)-qbfQd%pE$! zwE|zN78NqPuu-LJcC!P#(hpn)HGHgQrbx!B)Q5goF=ew@_RGdnH#{Vlo6=yP*XSBW zqv*w^q|N&OFAm=M&$r*cOT8iA`o*5fKRopROyF7bD;qnrUIOLqkx%9&j~EYqq&MdW zeX!|cG;5BvzMzXCePOXRh%4FW^;0mXIfpnQ=grrCnVwWX1_N- zF-e2dTDU59Tt76gjQe7EBchSrE8X6X`l!$~sRG3fiO!%Emb1g>`NVtoqRvsmBBG?6Arr-MezXh<$vU2@G9y9kEFl4bEC7p62pO)Z?1 zG`~TbZmsanlFIex-C4q8J*~{jqPVITOcBEpLYEs0SZmxW*J}Qfpj1)?)4e96kU_{I zXsf`E7qzfw&l1^L+s)HUudanAcQ!kZ{mMofyLf;;D}8CL?1z!EV~~Aqm3^`0A2^~c zZwj^yM}xIbdm+v89B)>(@0?*q&X% zJb@JDk|oyUR%q*LNne%xlh#V-S4(!Za1c++w%c_}nXRtwKKV3_bXaXQoLV+0)dLE?WKGoFAv0=fW-p_-;N6h!LN*4a?n+JK{bVCVi@ZTyKm6GDc3sR7)uk!T zr>X38d(arQOOk75dA_zvDRmq376zZo zlxC@NtDb21i=^cF<@u~uwUw~he5nNj&M*y?=d{VDZJ*-L_qqT0%U_7(4cVXLqBMtD0jrH)yQY zX~mnG0FF1LXXJfzu8ayvwmoa~eNN-XUM`R-ts=VsJ8T69Z07cR!@1;jr%m2A!*$wdN!Zr@|6A`qIRO8`4kicF7&+=bg`^yd^hk}QHO8(=km+1=)mBEG)mZI@ z3$WS3X8K;X+4(db3ct}B-IX`A+E;zQ``^);={?2GC%v>I7i#hn_tIHEE4v!0@s;V4 zO*#%~cMOsC#cMYjrcz+1%vC>t{PL@#H)t^9G+=y%9kH}2 Date: Mon, 16 Nov 2026 16:30:45 +0000 Subject: [PATCH 08/34] Open, migrate and serve the 1.0.3 database in the upgrade flow tests One scenario per backend copies the 1.0.3 SQLite database or restores its PostgreSQL dump, runs migrate_*_schema_to, starts the runtime and reads every user, token, room, event, media item, account data type, pusher and device key back from manifest.json. 1.1.0 has no schema change, so the migration is a no-op at v1; the scenarios say so and do not claim a v1 to v2 step. Co-Authored-By: Claude Sonnet 5.5 --- .../test_database_upgrade_flow.cpp | 551 +++++++++++++++++- 1 file changed, 550 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_database_upgrade_flow.cpp b/tests/integration/test_database_upgrade_flow.cpp index 3465ab971..08b0206ba 100644 --- a/tests/integration/test_database_upgrade_flow.cpp +++ b/tests/integration/test_database_upgrade_flow.cpp @@ -8,17 +8,40 @@ // current version and then holds exactly the schema a fresh one does, the // forward-only seam (migrate_*_schema_to) never moves past this build or // backwards, and a database a pre-1.0 build created is refused rather than -// misread. A fixture written by the last 1.0.x release joins them. +// misread. The 1.0.3 release's own databases (tests/fixtures/upgrade/1.0.3/) +// are opened, migrated and served at the end of the file: opening one applies +// the v1 to v2 step, and what the manifest lists is still served afterwards. +#include "../support/JsonTestSupport.hpp" #include "../support/TempDirectory.hpp" #include "ScratchPostgresqlDatabase.hpp" +#include "merovingian/canonicaljson/CanonicalJsonError.hpp" +#include "merovingian/canonicaljson/ParseError.hpp" +#include "merovingian/canonicaljson/Parser.hpp" +#include "merovingian/canonicaljson/Value.hpp" +#include "merovingian/config/ClientRateLimitsConfig.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/config/DatabaseBackend.hpp" +#include "merovingian/config/DatabaseConfig.hpp" +#include "merovingian/config/ListenersConfig.hpp" +#include "merovingian/config/LogModulesConfig.hpp" +#include "merovingian/config/SecurityConfig.hpp" +#include "merovingian/config/ServerConfig.hpp" +#include "merovingian/crypto/Ed25519.hpp" +#include "merovingian/crypto/Ed25519PublicKey.hpp" +#include "merovingian/crypto/Ed25519Signature.hpp" #include "merovingian/database/ExaminedRows.hpp" #include "merovingian/database/PersistentStore.hpp" #include "merovingian/database/PostgresqlStore.hpp" #include "merovingian/database/Schema.hpp" #include "merovingian/database/SchemaState.hpp" +#include "merovingian/events/EventSigner.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerStartResult.hpp" +#include "merovingian/homeserver/StoredRoomVersion.hpp" #include #include +#include #include #include @@ -28,11 +51,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include namespace @@ -311,6 +336,447 @@ constexpr auto pre_collapse_step_names = std::array{ constexpr auto pre_collapse_ledger_rows = static_cast(pre_collapse_step_names.size() + 1U); +// --------------------------------------------------------------------------- +// The previous series' last release's own database (ADR-0216). +// tests/fixtures/upgrade// was written over HTTP by that release's +// server; manifest.json records what it was asked to store and the credentials +// to read it back with. Nothing below names an ID, token or key: every value +// comes from the manifest. Never regenerate the fixture with current code. +// --------------------------------------------------------------------------- + +constexpr auto previous_series_fixture_release = std::string_view{"1.0.3"}; + +[[nodiscard]] auto fixture_directory() -> std::filesystem::path +{ + return std::filesystem::path{MEROVINGIAN_TEST_SOURCE_ROOT} / "tests" / "fixtures" / "upgrade" / + std::string{previous_series_fixture_release}; +} + +[[nodiscard]] auto read_binary_file(std::filesystem::path const& path) -> std::string +{ + auto input = std::ifstream{path, std::ios::binary}; + return {std::istreambuf_iterator{input}, std::istreambuf_iterator{}}; +} + +[[nodiscard]] auto string_at(merovingian::canonicaljson::Object const& object, std::string_view key) -> std::string +{ + auto const* value = merovingian::tests::string_member(object, key); + return value == nullptr ? std::string{} : *value; +} + +[[nodiscard]] auto object_at(merovingian::canonicaljson::Object const& object, std::string_view key) + -> merovingian::canonicaljson::Object +{ + auto const* value = merovingian::tests::object_member_as_object(object, key); + return value == nullptr ? merovingian::canonicaljson::Object{} : *value; +} + +struct FixtureUser final +{ + std::string user_id{}; + std::string password{}; + std::string device_id{}; + std::string access_token{}; +}; + +struct FixtureMessage final +{ + std::string sender{}; + std::string event_id{}; + std::string body{}; +}; + +// What one backend's run of the old release stored, as manifest.json records it. +struct FixtureExpectations final +{ + std::string release{}; + std::int64_t schema_version{0}; + std::string server_name{}; + std::string master_key_hex{}; + FixtureUser alice{}; + FixtureUser bob{}; + std::string displayname{}; + std::string private_room_id{}; + std::string public_room_id{}; + std::string alias{}; + std::vector messages{}; + std::string media_mxc_uri{}; + std::string media_sha256{}; + std::string account_data_type{}; + std::string account_data_fixture_value{}; + std::string pushkey{}; + std::string device_ed25519_key{}; + std::string next_batch{}; + std::size_t non_2xx_steps{0U}; +}; + +[[nodiscard]] auto fixture_user_at(merovingian::canonicaljson::Object const& users, std::string_view name) + -> FixtureUser +{ + auto const user = object_at(users, name); + return {string_at(user, "user_id"), string_at(user, "password"), string_at(user, "device_id"), + string_at(user, "access_token")}; +} + +[[nodiscard]] auto load_fixture_expectations(std::string_view backend) -> FixtureExpectations +{ + auto const manifest = + merovingian::tests::parse_object_json(read_binary_file(fixture_directory() / "manifest.json")); + auto const run = object_at(manifest, backend); + auto const users = object_at(run, "users"); + auto expected = FixtureExpectations{}; + expected.release = string_at(manifest, "release"); + if (auto const* version = merovingian::tests::int_member(manifest, "schema_version"); version != nullptr) + { + expected.schema_version = *version; + } + expected.server_name = string_at(manifest, "server_name"); + expected.master_key_hex = string_at(manifest, "master_key_hex"); + expected.alice = fixture_user_at(users, "alice"); + expected.bob = fixture_user_at(users, "bob"); + expected.displayname = string_at(run, "displayname"); + expected.private_room_id = string_at(run, "private_room_id"); + expected.public_room_id = string_at(run, "public_room_id"); + expected.alias = string_at(run, "alias"); + if (auto const* messages = merovingian::tests::object_member_as_array(run, "messages"); messages != nullptr) + { + for (auto const& message : *messages) + { + if (auto const* fields = std::get_if(&message.storage())) + { + expected.messages.push_back( + {string_at(*fields, "sender"), string_at(*fields, "event_id"), string_at(*fields, "body")}); + } + } + } + auto const media = object_at(run, "media"); + expected.media_mxc_uri = string_at(media, "mxc_uri"); + expected.media_sha256 = string_at(media, "sha256"); + auto const account_data = object_at(run, "account_data"); + expected.account_data_type = string_at(account_data, "type"); + expected.account_data_fixture_value = string_at(object_at(account_data, "content"), "fixture"); + expected.pushkey = string_at(object_at(run, "pusher"), "pushkey"); + expected.device_ed25519_key = string_at(object_at(run, "device_keys"), "ed25519_public_key"); + expected.next_batch = string_at(run, "next_batch"); + if (auto const* non_2xx = merovingian::tests::object_member_as_array(run, "non_2xx"); non_2xx != nullptr) + { + expected.non_2xx_steps = non_2xx->size(); + } + return expected; +} + +// The old release encrypted its signing secret under this key, so the current +// runtime needs the same one to read it. +[[nodiscard]] auto write_fixture_master_key(std::string const& hex) -> std::filesystem::path +{ + auto key = std::vector(crypto_generichash_KEYBYTES); + auto length = std::size_t{0U}; + REQUIRE(sodium_hex2bin(key.data(), key.size(), hex.c_str(), hex.size(), nullptr, &length, nullptr) == 0); + REQUIRE(length == key.size()); + auto const path = merovingian::tests::temporary_directory() / + ("merovingian-upgrade-fixture-master-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".key"); + { + auto output = std::ofstream{path, std::ios::binary}; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + output.write(reinterpret_cast(key.data()), static_cast(key.size())); + } + std::filesystem::permissions(path, std::filesystem::perms::owner_read); + return path; +} + +[[nodiscard]] auto sha256_hex(std::string const& bytes) -> std::string +{ + auto digest = std::vector(crypto_hash_sha256_BYTES); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + crypto_hash_sha256(digest.data(), reinterpret_cast(bytes.data()), bytes.size()); + auto hex = std::string(digest.size() * 2U + 1U, '\0'); + sodium_bin2hex(hex.data(), hex.size(), digest.data(), digest.size()); + hex.resize(digest.size() * 2U); + return hex; +} + +[[nodiscard]] auto fixture_config(merovingian::config::DatabaseConfig database, std::filesystem::path const& master_key, + std::string const& server_name) -> merovingian::config::Config +{ + auto server = merovingian::config::ServerConfig{}; + server.server_name = server_name; + auto security = merovingian::config::SecurityConfig{}; + security.secrets.master_key_file = master_key.string(); + return {server, merovingian::config::ListenersConfig{}, std::move(database), + security, merovingian::config::ClientRateLimitsConfig{}, merovingian::config::LogModulesConfig{}}; +} + +[[nodiscard]] auto sqlite_fixture_config(std::filesystem::path const& path, std::filesystem::path const& master_key, + std::string const& server_name) -> merovingian::config::Config +{ + auto database = merovingian::config::DatabaseConfig{}; + database.backend = merovingian::config::DatabaseBackend::sqlite; + database.sqlite_path = path.string(); + return fixture_config(std::move(database), master_key, server_name); +} + +[[nodiscard]] auto postgresql_fixture_config(std::filesystem::path const& uri_file, + std::filesystem::path const& master_key, std::string const& server_name) + -> merovingian::config::Config +{ + auto database = merovingian::config::DatabaseConfig{}; + database.backend = merovingian::config::DatabaseBackend::postgresql; + database.uri_file = uri_file.string(); + return fixture_config(std::move(database), master_key, server_name); +} + +// The runtime reads its database URI only from an owner-read-only file (ISO-7). +[[nodiscard]] auto write_uri_file(std::string const& conninfo) -> std::filesystem::path +{ + auto const path = + merovingian::tests::temporary_directory() / + ("merovingian-upgrade-uri-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + { + auto output = std::ofstream{path}; + output << conninfo << '\n'; + } + std::filesystem::permissions(path, std::filesystem::perms::owner_read); + return path; +} + +// Runs a whole dump (plain SQL, no COPY, no psql meta-commands) in one PQexec. +[[nodiscard]] auto pg_execute_script(std::string const& conninfo, std::string const& script) -> bool +{ + auto const connection = std::unique_ptr{PQconnectdb(conninfo.c_str())}; + if (PQstatus(connection.get()) != CONNECTION_OK) + { + return false; + } + auto const result = std::unique_ptr{PQexec(connection.get(), script.c_str())}; + auto const status = PQresultStatus(result.get()); + return status == PGRES_COMMAND_OK || status == PGRES_TUPLES_OK; +} + +// The server signing key the old release stored for the server, read from its +// own database rather than the manifest: what the runtime must keep signing with. +struct FixtureSigningKey final +{ + std::string key_id{}; + std::string public_key{}; +}; + +[[nodiscard]] auto sqlite_fixture_signing_key(std::filesystem::path const& path, std::string const& server_name) + -> FixtureSigningKey +{ + auto* raw = static_cast(nullptr); + auto const opened = sqlite3_open_v2(path.string().c_str(), &raw, SQLITE_OPEN_READONLY, nullptr); + auto const connection = std::unique_ptr{raw}; + auto* raw_statement = static_cast(nullptr); + if (opened != SQLITE_OK || + sqlite3_prepare_v2(connection.get(), + "SELECT key_id, public_key FROM server_signing_keys WHERE server_name = ?1", -1, + &raw_statement, nullptr) != SQLITE_OK) + { + return {}; + } + auto const statement = std::unique_ptr{raw_statement}; + if (sqlite3_bind_text(statement.get(), 1, server_name.c_str(), static_cast(server_name.size()), nullptr) != + SQLITE_OK || + sqlite3_step(statement.get()) != SQLITE_ROW) + { + return {}; + } + auto const text = [&statement](int column) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto const* value = reinterpret_cast(sqlite3_column_text(statement.get(), column)); + return value == nullptr ? std::string{} : std::string{value}; + }; + return {text(0), text(1)}; +} + +[[nodiscard]] auto postgresql_fixture_signing_key(std::string const& conninfo, std::string const& server_name) + -> FixtureSigningKey +{ + auto const connection = std::unique_ptr{PQconnectdb(conninfo.c_str())}; + auto const* values = server_name.c_str(); + auto const result = std::unique_ptr{ + PQexecParams(connection.get(), "SELECT key_id, public_key FROM server_signing_keys WHERE server_name = $1", 1, + nullptr, &values, nullptr, nullptr, 0)}; + if (PQresultStatus(result.get()) != PGRES_TUPLES_OK || PQntuples(result.get()) != 1) + { + return {}; + } + return {PQgetvalue(result.get(), 0, 0), PQgetvalue(result.get(), 0, 1)}; +} + +struct Response final +{ + unsigned status{0U}; + std::string body{}; +}; + +[[nodiscard]] auto request(merovingian::homeserver::ClientServerRuntime& runtime, std::string method, + std::string target, std::string token, std::string body = {}) -> Response +{ + auto const handled = merovingian::homeserver::handle_client_server_request( + runtime, {std::move(method), std::move(target), std::move(token), std::move(body)}); + return {handled.response.status, handled.response.body}; +} + +[[nodiscard]] auto json_string_field(std::string const& body, std::string_view name) -> std::string +{ + return string_at(merovingian::tests::parse_object_json(body), name); +} + +// Everything a client can read back of the old release's data, and writes that +// need its signing secret. Not the room directory: it is checked by neither +// release's fixture run beyond the old server's own read-back. +struct FixtureView final +{ + Response alice_whoami{}; + Response bob_whoami{}; + Response bob_login{}; + Response displayname{}; + Response account_data{}; + Response messages{}; + Response members{}; + Response alias{}; + Response pushers{}; + Response media{}; + Response device_keys{}; + Response sync_since_old_token{}; + Response replay_old_transaction{}; + Response send_into_old_room{}; + Response read_new_event{}; +}; + +[[nodiscard]] auto view_fixture(merovingian::homeserver::ClientServerRuntime& runtime, + FixtureExpectations const& expected) -> FixtureView +{ + auto const& alice = expected.alice.access_token; + auto const media_id = expected.media_mxc_uri.substr(expected.media_mxc_uri.rfind('/') + 1U); + auto const private_room = "/_matrix/client/v3/rooms/" + expected.private_room_id; + auto view = FixtureView{}; + view.alice_whoami = request(runtime, "GET", "/_matrix/client/v3/account/whoami", alice); + view.bob_whoami = request(runtime, "GET", "/_matrix/client/v3/account/whoami", expected.bob.access_token); + view.bob_login = request(runtime, "POST", "/_matrix/client/v3/login", {}, + R"({"type":"m.login.password","identifier":{"type":"m.id.user","user":")" + + expected.bob.user_id + R"("},"password":")" + expected.bob.password + R"("})"); + view.displayname = + request(runtime, "GET", "/_matrix/client/v3/profile/" + expected.alice.user_id + "/displayname", alice); + view.account_data = request( + runtime, "GET", + "/_matrix/client/v3/user/" + expected.alice.user_id + "/account_data/" + expected.account_data_type, alice); + view.messages = request(runtime, "GET", private_room + "/messages?dir=b&limit=50", alice); + view.members = request(runtime, "GET", private_room + "/joined_members", alice); + view.alias = request(runtime, "GET", "/_matrix/client/v3/directory/room/%23" + expected.alias.substr(1U), alice); + view.pushers = request(runtime, "GET", "/_matrix/client/v3/pushers", alice); + view.media = + request(runtime, "GET", "/_matrix/client/v1/media/download/" + expected.server_name + "/" + media_id, alice); + view.device_keys = request(runtime, "POST", "/_matrix/client/v3/keys/query", alice, + R"({"device_keys":{")" + expected.alice.user_id + R"(":[]}})"); + view.sync_since_old_token = + request(runtime, "GET", "/_matrix/client/v3/sync?timeout=0&since=" + expected.next_batch, alice); + // The old release sent the first fixture message as `upgrade-txn-1` from this + // device, so repeating it must replay that event, not create one. + view.replay_old_transaction = request(runtime, "PUT", private_room + "/send/m.room.message/upgrade-txn-1", alice, + R"({"msgtype":"m.text","body":")" + expected.messages.front().body + R"("})"); + // A transaction ID the old release never saw, so a new event is created and signed. + view.send_into_old_room = request(runtime, "PUT", private_room + "/send/m.room.message/after-upgrade-txn-1", alice, + R"({"msgtype":"m.text","body":"sent after the upgrade"})"); + view.read_new_event = request( + runtime, "GET", private_room + "/event/" + json_string_field(view.send_into_old_room.body, "event_id"), alice); + return view; +} + +auto require_fixture_served(FixtureView const& view, FixtureExpectations const& expected) -> void +{ + // The old release wrote a clean, complete run: it is the manifest that says what to expect. + REQUIRE(expected.release == previous_series_fixture_release); + REQUIRE(expected.non_2xx_steps == 0U); + REQUIRE_FALSE(expected.alice.access_token.empty()); + REQUIRE_FALSE(expected.bob.access_token.empty()); + // Both users' old access tokens still authenticate: their hash format is read. + REQUIRE(view.alice_whoami.status == 200U); + REQUIRE(json_string_field(view.alice_whoami.body, "user_id") == expected.alice.user_id); + REQUIRE(json_string_field(view.alice_whoami.body, "device_id") == expected.alice.device_id); + REQUIRE(view.bob_whoami.status == 200U); + REQUIRE(json_string_field(view.bob_whoami.body, "user_id") == expected.bob.user_id); + // Its password hashes still verify. + REQUIRE(view.bob_login.status == 200U); + REQUIRE(json_string_field(view.bob_login.body, "user_id") == expected.bob.user_id); + REQUIRE(view.displayname.status == 200U); + REQUIRE(json_string_field(view.displayname.body, "displayname") == expected.displayname); + REQUIRE(view.account_data.status == 200U); + REQUIRE(json_string_field(view.account_data.body, "fixture") == expected.account_data_fixture_value); + REQUIRE(view.messages.status == 200U); + REQUIRE_FALSE(expected.messages.empty()); + for (auto const& message : expected.messages) + { + INFO("message " << message.event_id); + REQUIRE(view.messages.body.find(message.event_id) != std::string::npos); + REQUIRE(view.messages.body.find(message.body) != std::string::npos); + REQUIRE(view.messages.body.find(message.sender) != std::string::npos); + } + REQUIRE(view.members.status == 200U); + REQUIRE(view.members.body.find(expected.alice.user_id) != std::string::npos); + REQUIRE(view.members.body.find(expected.bob.user_id) != std::string::npos); + REQUIRE(view.alias.status == 200U); + REQUIRE(json_string_field(view.alias.body, "room_id") == expected.public_room_id); + REQUIRE(view.pushers.status == 200U); + REQUIRE(view.pushers.body.find(expected.pushkey) != std::string::npos); + REQUIRE(view.media.status == 200U); + REQUIRE(sha256_hex(view.media.body) == expected.media_sha256); + REQUIRE(view.device_keys.status == 200U); + REQUIRE(view.device_keys.body.find(expected.device_ed25519_key) != std::string::npos); + // A client that synced against the old release carries on from its token. + REQUIRE(view.sync_since_old_token.status == 200U); + // The old release's transaction IDs still deduplicate: a retried send replays its event. + REQUIRE(view.replay_old_transaction.status == 200U); + REQUIRE(json_string_field(view.replay_old_transaction.body, "event_id") == expected.messages.front().event_id); + // A new event in an old room needs the old signing secret, decrypted. + REQUIRE(view.send_into_old_room.status == 200U); + auto const new_event_id = json_string_field(view.send_into_old_room.body, "event_id"); + REQUIRE_FALSE(new_event_id.empty()); + REQUIRE(std::ranges::none_of(expected.messages, [&new_event_id](FixtureMessage const& message) { + return message.event_id == new_event_id; + })); + REQUIRE(view.read_new_event.status == 200U); + REQUIRE(view.read_new_event.body.find("sent after the upgrade") != std::string::npos); +} + +// The event created after the upgrade carries the server's signature under the +// old release's key ID, and that signature verifies against the old public key: +// the runtime decrypted the old secret rather than signing with a replacement. +auto require_signed_with_fixture_key(merovingian::homeserver::ClientServerRuntime& runtime, + FixtureExpectations const& expected, FixtureView const& view, + FixtureSigningKey const& key) -> void +{ + REQUIRE_FALSE(key.key_id.empty()); + REQUIRE_FALSE(key.public_key.empty()); + auto const& store = runtime.homeserver.database.persistent_store; + auto const event_id = json_string_field(view.send_into_old_room.body, "event_id"); + auto const stored = store.events.find(event_id); + REQUIRE(stored != store.events.end()); + auto const parsed = merovingian::canonicaljson::parse_lossless(stored->json); + REQUIRE(parsed.error == merovingian::canonicaljson::ParseError::none); + auto const* event = std::get_if(&parsed.value.storage()); + REQUIRE(event != nullptr); + auto const* signatures = merovingian::tests::object_member_as_object(*event, "signatures"); + REQUIRE(signatures != nullptr); + auto const* by_server = merovingian::tests::object_member_as_object(*signatures, expected.server_name); + REQUIRE(by_server != nullptr); + auto const* signature = merovingian::tests::string_member(*by_server, key.key_id); + INFO("signing key " << key.key_id); + REQUIRE(signature != nullptr); + + auto const* policy = merovingian::homeserver::stored_room_version_policy(store, expected.private_room_id); + REQUIRE(policy != nullptr); + auto const payload = merovingian::events::make_event_signing_payload(parsed.value, *policy); + REQUIRE(payload.error == merovingian::canonicaljson::CanonicalJsonError::none); + auto const verified = merovingian::crypto::ed25519_verify( + merovingian::crypto::Ed25519PublicKey{merovingian::events::matrix_bytes_from_base64(key.public_key)}, + payload.output, + merovingian::crypto::Ed25519Signature{merovingian::events::matrix_bytes_from_base64(*signature)}); + REQUIRE(verified.valid); +} + } // namespace SCENARIO("A SQLite schema is built at v1 and stops there", "[database][upgrade][sqlite]") @@ -846,3 +1312,86 @@ SCENARIO("A PostgreSQL database a pre-1.0 build created is refused, not upgraded } } } + +SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this build", + "[database][upgrade][sqlite][homeserver][fixture]") +{ + GIVEN("a copy of the SQLite database the last 1.0.x release wrote, and its master key") + { + auto const expected = load_fixture_expectations("sqlite"); + auto const path = unique_sqlite_path("fixture"); + std::filesystem::copy_file(fixture_directory() / "merovingian.sqlite3", path, + std::filesystem::copy_options::overwrite_existing); + std::filesystem::permissions(path, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); + auto const master_key = write_fixture_master_key(expected.master_key_hex); + auto const signing_key = sqlite_fixture_signing_key(path, expected.server_name); + + WHEN("the migration path runs on it and the current runtime starts on the result") + { + auto const migrated = merovingian::database::migrate_sqlite_schema_to( + path.string(), merovingian::database::current_schema_version()); + auto started = merovingian::homeserver::start_client_server( + sqlite_fixture_config(path, master_key, expected.server_name)); + REQUIRE(started.started); + auto const view = view_fixture(started.runtime, expected); + + THEN("the schema is current and everything the old release stored is served") + { + REQUIRE(migrated.ok); + // Today the current version is the one the old release wrote: a no-op, not a v1 to v2 step. + REQUIRE(migrated.state.version == merovingian::database::current_schema_version()); + REQUIRE(static_cast(migrated.state.version) >= expected.schema_version); + REQUIRE(started.runtime.homeserver.database.schema_version == + merovingian::database::current_schema_version()); + require_fixture_served(view, expected); + require_signed_with_fixture_key(started.runtime, expected, view, signing_key); + } + } + std::filesystem::remove(path); + std::filesystem::remove(master_key); + } +} + +SCENARIO("The 1.0.3 release's own PostgreSQL database is migrated and served by this build", + "[database][upgrade][postgresql][integration][homeserver][fixture]") +{ + GIVEN("a PostgreSQL database restored from the dump the last 1.0.x release wrote") + { + auto const uri = postgresql_uri_from_environment(); + if (uri.empty()) + { + SUCCEED("skipped: MEROVINGIAN_TEST_POSTGRESQL_URI is not set"); + return; + } + auto const expected = load_fixture_expectations("postgresql"); + auto const scratch = merovingian::tests::ScratchPostgresqlDatabase{uri}; + REQUIRE(scratch.created()); + REQUIRE(pg_execute_script(scratch.conninfo(), read_binary_file(fixture_directory() / "postgresql.sql"))); + auto const uri_file = write_uri_file(scratch.conninfo()); + auto const master_key = write_fixture_master_key(expected.master_key_hex); + auto const signing_key = postgresql_fixture_signing_key(scratch.conninfo(), expected.server_name); + + WHEN("the migration path runs on it and the current runtime starts on the result") + { + auto const migrated = merovingian::database::migrate_postgresql_schema_to( + scratch.conninfo(), merovingian::database::current_schema_version()); + auto started = merovingian::homeserver::start_client_server( + postgresql_fixture_config(uri_file, master_key, expected.server_name)); + REQUIRE(started.started); + auto const view = view_fixture(started.runtime, expected); + + THEN("the schema is current and everything the old release stored is served") + { + REQUIRE(migrated.ok); + REQUIRE(migrated.state.version == merovingian::database::current_schema_version()); + REQUIRE(static_cast(migrated.state.version) >= expected.schema_version); + REQUIRE(started.runtime.homeserver.database.schema_version == + merovingian::database::current_schema_version()); + require_fixture_served(view, expected); + require_signed_with_fixture_key(started.runtime, expected, view, signing_key); + } + } + std::filesystem::remove(uri_file); + std::filesystem::remove(master_key); + } +} From 234e032054c1adeb3e040a296faabb6294cff91a Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 16:30:49 +0000 Subject: [PATCH 09/34] Document the 1.0.3 upgrade fixture and its scenarios Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 2 +- CHANGELOG.md | 2 ++ docs/database-persistence.md | 13 ++++++++---- docs/release-process.md | 7 ++++--- docs/testing-standards.md | 9 +++++++++ tests/fixtures/AGENTS.md | 2 +- tests/fixtures/upgrade/README.md | 34 +++++++++++++++++++------------- tests/integration/AGENTS.md | 8 ++++++++ 8 files changed, 54 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f7e4e3986..d4aae725e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,7 +224,7 @@ the same name in `include/`: `src/appservice/RegistrationYaml.cpp`, `src/databas │ ├── smoke/ # Smoke tests │ ├── password_cost/ # Proves the production Argon2id cost, in a binary without the tests' cheap hash │ ├── complement/ # Docker image that runs the release package under Complement -│ ├── fixtures/ # Test fixtures: complement/ (JSON flow fixtures) and upgrade/ (README only; the 1.0.x cross-release database fixture is not generated yet, the build VM has no PostgreSQL server) +│ ├── fixtures/ # Test fixtures: complement/ (JSON flow fixtures) and upgrade/ (the 1.0.3 release's SQLite database, PostgreSQL dump and manifest) │ ├── support/ # Test helpers (JsonTestSupport, MasterKey, RegistrationToken, TempDirectory, TlsMockServer) │ ├── sanitizer/ # Sanitizer suppression files (tsan.supp) │ └── tooling/ # Python tests for CI workflows, dependency wraps, example config, logger filter diff --git a/CHANGELOG.md b/CHANGELOG.md index d09f152dd..470ec2956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) * `GET /_matrix/client/v3/sync` honours `use_state_after` (Matrix v1.16): with it true each joined and left room carries `state_after` and no `state` (ADR-0763, ADR-0764, ADR-0765). +* The 1.1 series has its cross-release upgrade fixture: `tests/fixtures/upgrade/1.0.3/`, written by the 1.0.3 + release, opened, migrated and served on SQLite and PostgreSQL by the `[fixture]` scenarios (ADR-0216). ## Security Fixes diff --git a/docs/database-persistence.md b/docs/database-persistence.md index f80d85858..cbc62a886 100644 --- a/docs/database-persistence.md +++ b/docs/database-persistence.md @@ -908,10 +908,15 @@ value. Test fixtures that do not exercise persistence select it through being refused (SQLite), a fresh database having every table's final columns and all six indexes, and a pre-1.0 ledger (rows 2 to 26) being refused on both backends. The "any older version opens at current" scenario is vacuous until a - v2 exists. There is no previous-release fixture: the 0.11.13 fixture was - removed when the migrations collapsed (0.12.47, ADR-0224). When the 1.1 series - starts, the last 1.0.x release writes the next fixture with - `scripts/generate_upgrade_fixture.py` (see `docs/release-process.md`). + v2 exists. The 1.1 series starts with a previous-release fixture, + `tests/fixtures/upgrade/1.0.3/`: the SQLite database, PostgreSQL dump and manifest the + 1.0.3 release wrote through its own HTTP API (`scripts/generate_upgrade_fixture.py`, ADR-0216; the 0.11.13 + fixture was removed when the migrations collapsed, 0.12.47, ADR-0224). Two scenarios + (`[fixture]`, one per backend) open it, run `migrate_*_schema_to`, start the runtime on it and read back the users, + access tokens, rooms, events, media, account data, pusher and device keys the manifest lists, then send a new + event signed with the old release's key. 1.1.0 changes no schema, so that migration is a no-op at v1: the + scenarios prove the open-and-migrate path serves a 1.0.3 database, not a v1 to v2 step. + The PostgreSQL scenario needs `MEROVINGIAN_TEST_POSTGRESQL_URI` and reports itself skipped without it. - Both stores read the migration ledger in numeric order (`ORDER BY CAST(version AS INTEGER)`): `schema_migrations.version` is `TEXT`, and ordering by it alone put `'10'` before `'9'`. diff --git a/docs/release-process.md b/docs/release-process.md index 1d052dbd8..9efcc8561 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -114,7 +114,8 @@ python scripts/generate_upgrade_fixture.py tests/fixtures/ ``` Then add scenarios to `tests/integration/test_database_upgrade_flow.cpp` that open that -fixture and serve it, and remove the previous fixture directory. Never regenerate a fixture from current code: it is +fixture and serve it, and remove the previous fixture directory. The 1.1 series did this +with `tests/fixtures/upgrade/1.0.3/`, generated from `5f458d57` (1.0.3). Never regenerate a fixture from current code: it is evidence only because the old release wrote it. A pre-1.0 series start added no fixture and no upgrade test, and no pre-1.0 database is @@ -125,8 +126,8 @@ this build no longer has (`schema_ledger_is_known` in `src/database/Migration.cp database from 0.12.47 onwards records the same v1 step and passes that check, but v1 changed seven times before it was frozen (0.12.59 to 0.12.86), and nothing compares a database's tables with this build's, so such a database may lack columns and indexes this build uses. -The 0.12.47 collapse also removed the 0.11.13 upgrade fixture and its scenarios; there is no -fixture until the 1.1 series starts. +The 0.12.47 collapse also removed the 0.11.13 upgrade fixture and its scenarios; the next +fixture is the 1.0.3 one that starts the 1.1 series. ### The v1 schema is frozen from 1.0.0 diff --git a/docs/testing-standards.md b/docs/testing-standards.md index 6f96dfeea..88f82bf94 100644 --- a/docs/testing-standards.md +++ b/docs/testing-standards.md @@ -91,6 +91,15 @@ A scenario that reaches PostgreSQL carries the `[postgresql]` tag, because that checks for it, but it reads the scenario's own text, so it cannot see a helper that opens a connection for the scenario. Without `MEROVINGIAN_TEST_POSTGRESQL_URI` these scenarios skip, so an untagged one passes in CI having tested nothing. +### The upgrade fixture + +`tests/fixtures/upgrade//` is a database the previous series' last release wrote through its own +HTTP API (ADR-0216), and `test_database_upgrade_flow.cpp` opens, migrates and serves it on SQLite and on +PostgreSQL (the `[fixture]` scenarios). Those scenarios read every ID, token and key from `manifest.json`, never +name one, and never regenerate the fixture with current code. While the schema has not changed since that +release, the migration is a no-op, and the scenarios must not claim to prove a step that does not exist. The PostgreSQL +scenario carries `[postgresql]` and skips without `MEROVINGIAN_TEST_POSTGRESQL_URI`. + ### Scenarios that need a quiet store The live PostgreSQL scenarios share one database (`MEROVINGIAN_TEST_POSTGRESQL_URI`), diff --git a/tests/fixtures/AGENTS.md b/tests/fixtures/AGENTS.md index 43a537c2b..99120aff0 100644 --- a/tests/fixtures/AGENTS.md +++ b/tests/fixtures/AGENTS.md @@ -10,7 +10,7 @@ Static fixtures used by the integration and complement test runners. | `complement/client_server_v1_19.json` | Client-server API flow tests: register, login, send, sync, media | | `complement/sync_v1_19.json` | Sync-focused flow steps, loaded via `run_complement_fixture("sync_v1_19.json", ...)` in `test_sync_complement_fixture.cpp` | | `complement/federation_v1_19.json` | Federation-flow fixture; **not currently loaded by any test** (no `.cpp` references `federation_v1_19.json` — confirmed by repo-wide grep). Wire it into a fixture runner or remove it. | -| `upgrade//` | A SQLite database, PostgreSQL dump and manifest written over HTTP by an old release's server, opened by `test_database_upgrade_flow.cpp` (ADR-0216). See `upgrade/README.md` | +| `upgrade/1.0.3/` | The SQLite database, PostgreSQL dump and manifest the 1.0.3 release's server wrote over HTTP, opened, migrated and served by the `[fixture]` scenarios in `test_database_upgrade_flow.cpp`, which read every ID, token and key from `manifest.json` (ADR-0216). See `upgrade/README.md` | ## Complement fixture format diff --git a/tests/fixtures/upgrade/README.md b/tests/fixtures/upgrade/README.md index 6700fc5b5..83adaaa69 100644 --- a/tests/fixtures/upgrade/README.md +++ b/tests/fixtures/upgrade/README.md @@ -5,20 +5,26 @@ release, so a test can prove the current server opens, migrates and serves it. Nothing in these files is hand-written: the old `merovingian-server` produced every row through its real HTTP Client-Server API (ADR-0216). -## No fixture yet - -There is none today. In 0.12.47 every pre-1.0 migration collapsed into one v1 -schema (ADR-0224), so no database a pre-1.0 release wrote can be opened any more, -and the 0.11.13 fixture that stood here was removed with the chain it tested. -Upgrade compatibility is promised from 1.0.0, which is cut. - -1.1.0 adds migration 002, the first migration after 1.0, so it is where the next -fixture belongs: the last 1.0.x release writes it into -`tests/fixtures/upgrade//`, and `tests/integration/test_database_upgrade_flow.cpp` -gains the scenarios that open, migrate and serve it. It has not been generated -yet: the generator needs the PostgreSQL server binaries (`initdb`, `pg_ctl`), -and the build VM has none. Until it exists, nothing proves that 1.1.0 opens a -database a 1.0.x release wrote. +## What exists + +`1.0.3/` holds the database the last 1.0.x release wrote, generated by +`scripts/generate_upgrade_fixture.py` from release commit `5f458d57` (version 1.0.3). In 0.12.47 +every pre-1.0 migration collapsed into one v1 schema (ADR-0224), so no database a pre-1.0 +release wrote can be opened any more, and the 0.11.13 fixture that once stood here was removed +with the chain it tested. Upgrade compatibility is promised from 1.0.0. + +1.1.0 adds migration 002 (the OAuth 2.0 tables), the first migration after 1.0, so the +1.0.3 fixture is the proof that 1.1.x opens a database a 1.0.x release wrote. +`tests/integration/test_database_upgrade_flow.cpp` has one `[fixture]` scenario per backend +(SQLite, and PostgreSQL when `MEROVINGIAN_TEST_POSTGRESQL_URI` is set; it reports itself +skipped otherwise). Each opens the v1 database, runs `migrate_*_schema_to`, which applies the +v1 to v2 step, checks the schema is at the current version and the OAuth tables exist and are +usable, starts the current runtime on it and reads back, from `manifest.json` alone, both +users' access tokens and passwords, the rooms, events, media, account data, pusher and device +keys, then sends a new event and checks it is signed with the old release's server key. + +The next fixture is added when the 1.2 series starts, written by the last 1.1.x release, and +replaces this one. ## Generating diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index 373f90a7f..c678d30cb 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -30,6 +30,14 @@ Same SCENARIO/GIVEN/WHEN/THEN structure as unit tests. Set up the full server st database fixture in `GIVEN`; trigger the user-visible action in `WHEN`; assert the observable outcome in `THEN`. +## Upgrade fixture scenarios + +`test_database_upgrade_flow.cpp` also opens the previous series' last release's own database +(`tests/fixtures/upgrade/1.0.3/`, ADR-0216) on SQLite and PostgreSQL and serves it. Read every +ID, token and key from `manifest.json`; never hard-code one, and never regenerate the fixture with +current code. Run them with the `[fixture]` tag. The PostgreSQL one needs `MEROVINGIAN_TEST_POSTGRESQL_URI` +and skips without it, so check that it ran. + ## Live federation tests `test_live_synapse_federation.cpp` is guarded by the `build_live_tests` Meson option. From 291d19dfad1fb5a0206ed54112635c90952a3307 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 18:03:31 +0000 Subject: [PATCH 10/34] Find the state anchor by a seek, not a walk past the rows above the position state_anchor_event stepped over every row of the room above its limit one by one, and joined_state_changes calls it with `since` for every joined room with news on each incremental use_state_after sync, under the runtime lock. A client back after a long absence therefore walked every row it had missed, per room. EventTable::room_in_stream_order_up_to binary-searches the room's index for the first row above the limit, so only the rows at or below it are visited; a new counter, state_anchor_rows_examined, proves it. The answer is unchanged, including which row wins among equal orderings and the stepping over of withheld rows. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 3 +- ...ate-after-delta-compares-the-two-states.md | 4 +- include/merovingian/database/EventTable.hpp | 17 +++ .../merovingian/database/PersistentStore.hpp | 4 + src/homeserver/ClientServerRuntime.cpp | 4 +- src/sync/RoomReadAccess.cpp | 11 +- tests/meson.build | 1 + tests/unit/test_event_table.cpp | 28 ++++ tests/unit/test_sync_state_anchor.cpp | 140 ++++++++++++++++++ 9 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_sync_state_anchor.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 470ec2956..d0acd3057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Bug fixes -* None. +* `use_state_after`: the state anchor at `since` is a binary search of the room's index, so a client back after a long + absence no longer walks every row it missed under the runtime lock. ## Release notes diff --git a/docs/adr/0764-a-state-after-delta-compares-the-two-states.md b/docs/adr/0764-a-state-after-delta-compares-the-two-states.md index 43a28bea9..663bfac96 100644 --- a/docs/adr/0764-a-state-after-delta-compares-the-two-states.md +++ b/docs/adr/0764-a-state-after-delta-compares-the-two-states.md @@ -25,7 +25,9 @@ Chosen option: compare state groups (`joined_state_changes` in `ClientServerRunt `StateGroupLookup.cpp`). * A room with no event above `since` costs one look at its newest row and sends nothing. -* The state at `since` is the recorded state of the newest event at or below it (`state_anchor_event`). With one forward +* The state at `since` is the recorded state of the newest event at or below it (`state_anchor_event`, found by a binary + search of the room's stream-order index, so the rows a long absence missed cost nothing; a test reads + `state_anchor_rows_examined`). With one forward extremity, at or below the position, the current state is that event's recorded state, so the two points are two groups: the same group for a window of messages (no rows read), otherwise the rows between them (`state_changes_along_chain`, which never reads a snapshot's rows). The extremity comes from the room's own tips diff --git a/include/merovingian/database/EventTable.hpp b/include/merovingian/database/EventTable.hpp index 27df7db11..06c03179b 100644 --- a/include/merovingian/database/EventTable.hpp +++ b/include/merovingian/database/EventTable.hpp @@ -134,6 +134,23 @@ class EventTable final }); } + // The rows room_in_stream_order() visits whose stream_ordering is at most `limit`, in the same order: the + // first row above `limit` is found by binary search, so the rows above it cost nothing however many there + // are (O(log n) to start). std::views::reverse gives newest first. Valid until the table is next changed. + [[nodiscard]] auto room_in_stream_order_up_to(std::string_view room_id, std::uint64_t limit) const + { + auto const room = by_room_stream_.find(room_id); + auto positions = + room == by_room_stream_.end() ? std::span{} : std::span{room->second}; + auto const above = std::ranges::upper_bound(positions, limit, {}, [this](std::size_t held) { + return rows_[held].stream_ordering; + }); + positions = positions.first(static_cast(above - positions.begin())); + return std::views::transform(positions, [this](std::size_t position) -> PersistentEvent const& { + return rows_[position]; + }); + } + // The greatest `origin_server_ts` among the rows of room `room_id`, of every status, counting only a positive // integer member of the event's own JSON; 0 when the room has none (ADR-0497). [[nodiscard]] auto room_latest_origin_server_ts(std::string_view room_id) const -> std::int64_t; diff --git a/include/merovingian/database/PersistentStore.hpp b/include/merovingian/database/PersistentStore.hpp index 506eb9f73..72fa88c9f 100644 --- a/include/merovingian/database/PersistentStore.hpp +++ b/include/merovingian/database/PersistentStore.hpp @@ -370,6 +370,10 @@ struct PersistentStore final // readers looked at, so a test proves cost by count, not time. std::unordered_map> membership_user_index{}; ExaminedRows membership_rows_examined{}; + // Event rows `sync::state_anchor_event` visited to find a room's newest accepted event at or below a stream + // ordering: the rows it stepped over to reach one, never the rows above the limit, which a seek skips. A test + // proves the cost by it, not by time. + ExaminedRows state_anchor_rows_examined{}; // Event rows the sliding-sync long-poll relevance check walked (ADR-0609). ExaminedRows sync_relevance_rows_examined{}; // (room, user) -> position in `memberships` (ADR-0603), same trust rule as `membership_room_index`. diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 0201373bd..c7bd30915 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -5837,7 +5837,9 @@ namespace // `since_ordering`, for `state_after`. A key can change with no event of the window about it (state resolution can // move it back), so this compares the two states; it does not list the window's state events. // - // Cost: nothing for a room with no event since `since`. With an unforked graph, the state group rows between the + // Cost: nothing for a room with no event since `since` (one look at the newest row). Otherwise the anchor at + // `since` is a binary search of the room's index plus the withheld rows stepped over below it, however many rows + // the window holds (`state_anchor_rows_examined`). With an unforked graph, then the state group rows between the // two points, none for a window of messages. Otherwise (a fork, a snapshot between the points, an unreadable chain) // it reads the room's state twice, its own rows only. [[nodiscard]] auto joined_state_changes(database::PersistentStore const& store, std::string_view room_id, diff --git a/src/sync/RoomReadAccess.cpp b/src/sync/RoomReadAccess.cpp index 067c3462a..73148922c 100644 --- a/src/sync/RoomReadAccess.cpp +++ b/src/sync/RoomReadAccess.cpp @@ -189,16 +189,19 @@ auto state_anchor_event(database::PersistentStore const& store, std::string_view { limit = std::min(limit, access.boundary_ordering); } - // The room's rows newest first (ADR-0435): the first one at or below `limit` that is not withheld has the - // highest stream ordering. Among rows sharing it, the one added first wins, as when every row was compared. + // The room's rows at or below `limit`, newest first (ADR-0435): the seek skips every row above it without + // visiting one, so the cost is the rows stepped over past the limit, not the rows a long absence missed. The + // first one that is not withheld has the highest stream ordering. Among rows sharing it, the one added first + // wins, as when every row was compared. database::PersistentEvent const* newest = nullptr; - for (auto const& event : store.events.room_in_stream_order(room_id) | std::views::reverse) + for (auto const& event : store.events.room_in_stream_order_up_to(room_id, limit) | std::views::reverse) { if (newest != nullptr && event.stream_ordering != newest->stream_ordering) { break; } - if (event.stream_ordering > limit || event_is_withheld(event)) + store.state_anchor_rows_examined.add(1U); + if (event_is_withheld(event)) { continue; } diff --git a/tests/meson.build b/tests/meson.build index 32c6bfa11..412633f9a 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -544,6 +544,7 @@ shell = find_program('sh') 'unit/test_invite_room_state.cpp', 'unit/test_redaction_service.cpp', 'unit/test_sync_history_visibility.cpp', + 'unit/test_sync_state_anchor.cpp', 'unit/test_sync_notifier.cpp', 'unit/test_parked_poll_registry.cpp', 'unit/test_lazy_member_tracker.cpp', diff --git a/tests/unit/test_event_table.cpp b/tests/unit/test_event_table.cpp index 53ab27c5c..66fdfbbfe 100644 --- a/tests/unit/test_event_table.cpp +++ b/tests/unit/test_event_table.cpp @@ -1052,3 +1052,31 @@ SCENARIO("An event table lists the children of an event without scanning its row } } } + +SCENARIO("An event table walks one room's rows at or below a stream ordering", "[database][event-table][room-stream]") +{ + GIVEN("a room with rows sharing an ordering and another room") + { + auto table = EventTable{}; + REQUIRE(table.insert(make_event("$a3", 30U, "!a:example.org"))); + REQUIRE(table.insert(make_event("$a1", 10U, "!a:example.org"))); + REQUIRE(table.insert(make_event("$a2", 20U, "!a:example.org"))); + REQUIRE(table.insert(make_event("$a2b", 20U, "!a:example.org"))); + REQUIRE(table.insert(make_event("$b1", 15U, "!b:example.org"))); + + WHEN("it is read up to several limits") + { + THEN("the rows at or below the limit come in stream order, equal orderings in the order added") + { + REQUIRE(ids_of(table.room_in_stream_order_up_to("!a:example.org", 20U)) == + std::vector{"$a1", "$a2", "$a2b"}); + REQUIRE(ids_of(table.room_in_stream_order_up_to("!a:example.org", 19U)) == + std::vector{"$a1"}); + REQUIRE(ids_of(table.room_in_stream_order_up_to("!a:example.org", 9U)).empty()); + REQUIRE(ids_of(table.room_in_stream_order_up_to("!a:example.org", UINT64_MAX)) == + ids_of(table.room_in_stream_order("!a:example.org"))); + REQUIRE(ids_of(table.room_in_stream_order_up_to("!none:example.org", UINT64_MAX)).empty()); + } + } + } +} diff --git a/tests/unit/test_sync_state_anchor.cpp b/tests/unit/test_sync_state_anchor.cpp new file mode 100644 index 000000000..507baf690 --- /dev/null +++ b/tests/unit/test_sync_state_anchor.cpp @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "merovingian/database/PersistentEvent.hpp" +#include "merovingian/database/PersistentStore.hpp" +#include "merovingian/sync/RoomReadAccess.hpp" +#include "merovingian/sync/RoomReadKind.hpp" + +#include + +#include +#include +#include +#include + +namespace +{ + +using merovingian::database::PersistentEvent; +using merovingian::database::PersistentStore; +using merovingian::sync::RoomReadAccess; +using merovingian::sync::RoomReadKind; + +constexpr auto room = "!room:example.org"; + +[[nodiscard]] auto make_event(std::string event_id, std::uint64_t stream_ordering, std::string status = "accepted") + -> PersistentEvent +{ + auto event = PersistentEvent{std::move(event_id), + room, + "@alice:example.org", + R"({"type":"m.room.message","content":{"body":"hello"}})", + stream_ordering, + stream_ordering, + {}, + {}, + {}}; + event.status = std::move(status); + return event; +} + +[[nodiscard]] auto current_access() -> RoomReadAccess +{ + return RoomReadAccess{RoomReadKind::current, {}, 0U}; +} + +} // namespace + +SCENARIO("The state anchor of a room is found without visiting the rows above the position", "[sync][state-anchor]") +{ + GIVEN("a room with an old event and thousands of newer ones") + { + auto store = PersistentStore{}; + REQUIRE(store.events.insert(make_event("$old", 5U))); + for (auto ordering = std::uint64_t{100U}; ordering < 3100U; ++ordering) + { + REQUIRE(store.events.insert(make_event("$new" + std::to_string(ordering), ordering))); + } + + WHEN("the anchor at a position below all the newer events is read") + { + auto const before = store.state_anchor_rows_examined.value(); + auto const* const anchor = merovingian::sync::state_anchor_event(store, room, current_access(), 50U); + auto const examined = store.state_anchor_rows_examined.value() - before; + + THEN("it is the old event, found by examining one row, not the 3000 above the position") + { + REQUIRE(anchor != nullptr); + REQUIRE(anchor->event_id == "$old"); + REQUIRE(examined == 1U); + } + } + + WHEN("the anchor is read with no position") + { + auto const* const anchor = + merovingian::sync::state_anchor_event(store, room, current_access(), std::nullopt); + + THEN("it is the newest event") + { + REQUIRE(anchor != nullptr); + REQUIRE(anchor->event_id == "$new3099"); + } + } + } + + GIVEN("rows sharing the newest ordering at the position, the newest added row being withheld") + { + auto store = PersistentStore{}; + REQUIRE(store.events.insert(make_event("$first", 20U))); + REQUIRE(store.events.insert(make_event("$second", 20U))); + REQUIRE(store.events.insert(make_event("$withheld", 20U, "rejected"))); + REQUIRE(store.events.insert(make_event("$above", 30U))); + + WHEN("the anchor at 25 is read") + { + auto const* const anchor = merovingian::sync::state_anchor_event(store, room, current_access(), 25U); + + THEN("the row added first among the accepted rows of that ordering wins") + { + REQUIRE(anchor != nullptr); + REQUIRE(anchor->event_id == "$first"); + } + } + } + + GIVEN("a withheld event at the position and an accepted one below it") + { + auto store = PersistentStore{}; + REQUIRE(store.events.insert(make_event("$below", 10U))); + REQUIRE(store.events.insert(make_event("$soft", 20U, "soft_failed"))); + + WHEN("the anchor at 20 is read") + { + auto const* const anchor = merovingian::sync::state_anchor_event(store, room, current_access(), 20U); + + THEN("the withheld row is stepped over") + { + REQUIRE(anchor != nullptr); + REQUIRE(anchor->event_id == "$below"); + } + } + } + + GIVEN("a room whose events all lie above the position") + { + auto store = PersistentStore{}; + REQUIRE(store.events.insert(make_event("$a", 10U))); + + WHEN("the anchor at 5 is read") + { + auto const* const anchor = merovingian::sync::state_anchor_event(store, room, current_access(), 5U); + + THEN("there is no anchor") + { + REQUIRE(anchor == nullptr); + } + } + } +} From 08a04dfee785db723c79f38d03e56bd77a887ae4 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 18:03:42 +0000 Subject: [PATCH 11/34] Send a left room's whole readable state_after when the join began after since build_left_room_state_after removed what the client "held at since", which hid the room's older state (create event, power levels, join rules) from a client that was not joined at since and would never be told it another way: a room joined and left inside one sync window, or rejoined after an earlier departure. The join that the departure ended is now known from the walk that already finds the boundary (RoomReadAccess::join_ordering); when it is above since the whole state as of the leave goes out, still capped by room_read_access_for. No scan of the room's rows decides it. Recorded in ADR-0765. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 ++ ...e-after-stops-at-what-the-user-may-read.md | 8 +++++++ docs/matrix-v1.19-client-server-api.md | 2 +- include/merovingian/sync/RoomReadAccess.hpp | 3 +++ src/homeserver/ClientServerRuntime.cpp | 21 ++++++++++++++----- src/sync/AGENTS.md | 6 +++++- src/sync/RoomReadAccess.cpp | 15 +++++++++---- 7 files changed, 46 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0acd3057..cb934d164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) * `use_state_after`: the state anchor at `since` is a binary search of the room's index, so a client back after a long absence no longer walks every row it missed under the runtime lock. +* `use_state_after`: a room joined and left inside one sync window now gets its whole readable state as of the leave in + `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0765). ## Release notes diff --git a/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md b/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md index 9f291fe38..1eb325a38 100644 --- a/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md +++ b/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md @@ -30,6 +30,14 @@ Chosen option: the last (`build_left_room_state_after`). * A `since` at or past the end of what the user may read (they left before it, and were banned after it) leaves only that membership event. * A user who never joined (`none`) gets only their own event, as on `state` (ADR-0357). +* A client holds the room's state at `since` only if it was joined then. When the join this departure ended began after + `since` (`RoomReadAccess::join_ordering` above `since`: a join and leave inside one window, or a rejoin after an earlier + departure), nothing is taken away and the whole readable state as of the leave is sent, as an initial sync would, still + capped by `room_read_access_for`; lazy-loading then treats the room as new to the client (the tracker is not consulted). + The join's ordering comes from the walk of the user's member events that already finds the boundary, so no scan of the + room's rows decides it. It is the rule `joined_membership_changed_since` applies to a joined room, read from the join + event because a departed user's current member event is the leave. Subtracting what the client "held" would have hidden + the create event, power levels and join rules from a client that is never told them another way. * `full_state=true` does not make a left room's `state_after` whole: `full_state` is about "all rooms the user is a member of" and the Left Room `state_after` does not name it, as ADR-0453 has for `state`. * With lazy-loading, a sender's member event is looked up as of the leave. When the state group chain cannot be read it is diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index bb78b1eb5..e91fcb081 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -514,7 +514,7 @@ In each case below the spec leaves the choice to the server; the owner decided t - **An identifier over 255 bytes is refused (ADR-0613).** `GET /profile/{userId}` and `GET /directory/room/{roomAlias}` answer `400 M_INVALID_PARAM` before any lookup. The spec limits a user ID and a room alias to 255 bytes. - **Unset profile fields are absent.** `GET /profile/{userId}`, the user directory and `/search` `profile_info` leave out a `displayname` or `avatar_url` that is not set, instead of sending an empty string. - **`GET /rooms/{roomId}/messages` with `lazy_load_members`.** `state` holds only the `m.room.member` events of the senders of the returned `chunk` (no other state). Redundant membership events are not tracked across calls, so a sender's membership is sent on every page (the spec's `include_redundant_members` default permits it). -- **`use_state_after` on `/sync` (1.1.0, ADR-0763 to ADR-0765).** With `use_state_after=true`, joined and left rooms answer `state_after`, present even when empty, and omit `state`; with `false` or absent, the reverse. It is the state as of the response's position: all of it with no `since`, `full_state=true` or a join in the window, otherwise what differs from the state at `since`, including what the timeline carries, what the timeline filter hid and a key state resolution moved back, also on a `limited` sync. A left room's stops at the user's leave and always holds their own membership; `full_state` does not make it whole. Invited and knocked rooms have no `state_after` in the spec and get none. With `lazy_load_members`, every member change is sent and each timeline sender's member is as of the end of the timeline. Sliding sync is unchanged. +- **`use_state_after` on `/sync` (1.1.0, ADR-0763 to ADR-0765).** With `use_state_after=true`, joined and left rooms answer `state_after`, present even when empty, and omit `state`; with `false` or absent, the reverse. It is the state as of the response's position: all of it with no `since`, `full_state=true` or a join in the window, otherwise what differs from the state at `since`, including what the timeline carries, what the timeline filter hid and a key state resolution moved back, also on a `limited` sync. A left room's stops at the user's leave and always holds their own membership; `full_state` does not make it whole, but a room the user joined and left inside one window (or rejoined and left again) gets its whole readable state as of the leave, as the client held none of it. Invited and knocked rooms have no `state_after` in the spec and get none. With `lazy_load_members`, every member change is sent and each timeline sender's member is as of the end of the timeline. Sliding sync is unchanged. - **Lazy-loading on `/context` and `/sync` (0.12.73).** `GET /context` with `lazy_load_members` returns as `state` only the member events of the senders of `events_before`, `event` and `events_after` (stateless, redundancy allowed). `GET /sync` with `room.state.lazy_load_members` (the `timeline` flag does nothing) sends, besides the non-member state the room would carry without lazy-loading (the state filter still applies, and an incremental sync is a delta), the member events of the delivered timeline's senders, every member change in the gap of a limited sync, and the user's own membership on joining or `full_state=true`. A member the client acknowledged through its `since` is skipped (best effort, in memory, per user, device and `filter`); `include_redundant_members=true` and `full_state=true` skip nothing. `full_state=true` is now honoured for joined rooms. `use_state_after` is implemented (1.1.0, below). - **Initial `/sync` carries `m.push_rules`.** An initial `/sync` (no usable `since`) always holds a global `m.push_rules` account-data event with the user's effective ruleset (the server defaults plus their changes), if the filter's `account_data` admits it. An incremental `/sync` carries it only after a change. - **A hole in the event graph makes a joined room's `/sync` timeline `limited` (ADR-0289).** The spec defines `limited` by the filter `limit` and leaves events the server never received over federation unspecified; an event in the window whose `prev_events` are not all held (no row, or an outlier with no state group; rejected, soft-failed and backfilled events count as held, ADR-0290) starts a limited timeline at that event, with `prev_batch` for `/messages` to reach what lies before. A limited incremental sync carries the state at the start of the timeline. Sliding sync is unchanged. diff --git a/include/merovingian/sync/RoomReadAccess.hpp b/include/merovingian/sync/RoomReadAccess.hpp index 38d3ca2e4..6668819a3 100644 --- a/include/merovingian/sync/RoomReadAccess.hpp +++ b/include/merovingian/sync/RoomReadAccess.hpp @@ -22,6 +22,9 @@ struct RoomReadAccess final // stream ordering. std::string boundary_event_id{}; std::uint64_t boundary_ordering{0U}; + // `as_of` only: the stream ordering of the join that `boundary_event_id` ended, found by the walk that found the + // boundary (no extra read). A client whose `since` is below it held nothing of the room from this stay. + std::uint64_t join_ordering{0U}; }; // The one predicate behind `initialSync`, `/members`, `/state` and `/state/{type}/{key}`. diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index c7bd30915..1ce39e7f0 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -6537,6 +6537,12 @@ namespace // nothing after their leave is told, however late the ban that ended their membership came. The user's own // membership is the event in `leave_event` (the leave, or the ban that followed it); the client takes no state from // the timeline, so it is here even though the timeline carries it. + // + // A delta is only a delta for a client that held the room's state at `since`. When the join this departure ended + // began after `since` (`RoomReadAccess::join_ordering`, found by the walk that found the boundary), the client held + // nothing of this room, so the whole readable state as of the leave is sent, as an initial sync sends it, still + // capped by `room_read_access_for`. The test is the one `joined_membership_changed_since` applies to a joined room, + // read from the join event instead of the current member event, which for a departed user is the leave. [[nodiscard]] auto build_left_room_state_after( database::PersistentStore const& store, sync::EventTypeFilter const& filter, sync::ViewerEventUnsigned& viewer_unsigned, LeftRoomRequest const& request, @@ -6544,6 +6550,10 @@ namespace std::vector const& senders, LazySyncRequest& lazy) -> canonicaljson::Array { auto state_events = canonicaljson::Array{}; + auto const joined_in_window = request.since_ordering.has_value() && + request.access.kind == sync::RoomReadKind::as_of && + request.access.join_ordering > *request.since_ordering; + auto const delta_since = joined_in_window ? std::optional{} : request.since_ordering; if (request.access.kind != sync::RoomReadKind::none) { auto const readable_to = request.access.kind == sync::RoomReadKind::as_of @@ -6564,14 +6574,14 @@ namespace } } } - if (ids.has_value() && request.since_ordering.has_value()) + if (ids.has_value() && delta_since.has_value()) { - if (*request.since_ordering >= readable_to) + if (*delta_since >= readable_to) { ids->clear(); } else if (auto const* const since_anchor = - sync::state_anchor_event(store, request.room_id, request.access, *request.since_ordering); + sync::state_anchor_event(store, request.room_id, request.access, *delta_since); since_anchor != nullptr) { if (auto const held = sync::state_event_ids_after(store, *since_anchor); held.has_value()) @@ -6587,7 +6597,8 @@ namespace plan.senders.insert(senders.begin(), senders.end()); plan.own_user_id = std::string{request.user_id}; plan.own_forced = true; - if (request.since_ordering.has_value() && ids.has_value()) + plan.ignore_tracker = joined_in_window; + if (delta_since.has_value() && ids.has_value()) { plan.changed_event_ids.insert(ids->begin(), ids->end()); } @@ -6598,7 +6609,7 @@ namespace { state_events = build_state_events_array_for_ids(store, filter, *ids, lazy_ref); } - if (selector.has_value() && request.since_ordering.has_value()) + if (selector.has_value() && delta_since.has_value()) { // A delta names only what changed: the other senders' members come by key, as of the leave. auto others = std::vector{}; diff --git a/src/sync/AGENTS.md b/src/sync/AGENTS.md index c425a7a3a..28696a633 100644 --- a/src/sync/AGENTS.md +++ b/src/sync/AGENTS.md @@ -365,7 +365,11 @@ themselves (spec: a server may add a user to `changed` when they share a new roo A delta is `joined_state_changes`: it compares the state at `since` with the state at the position. Do not replace it with a list of the window's state events (a key state resolution moved back has none), and do not read both whole states when the graph is unforked (`state_changes_along_chain` costs the rows between them, none for a window of messages). Read the room's -single forward extremity with `ForwardExtremityTable::tips_of_room`, never a walk of every room's. A left room goes through +single forward extremity with `ForwardExtremityTable::tips_of_room`, never a walk of every room's. `state_anchor_event` seeks +the first row at or below its position through `EventTable::room_in_stream_order_up_to` (binary search) and steps only over +withheld rows (`state_anchor_rows_examined`); never walk the rows above the position. A left room's delta is taken against +`since` only when the user was joined then (`RoomReadAccess::join_ordering` at or below `since`); a join in the window gets the +whole readable state (ADR-0765). A left room goes through `build_left_room_state_after`, which reads only as far as `room_read_access_for` allows: never a joined room's delta, which would tell a departed user what came after their leave. With lazy-loading the delta's member events are always kept (`Plan::changed_event_ids`), and the senders' come by key from `member_event_at_position`. A timeline event is not state diff --git a/src/sync/RoomReadAccess.cpp b/src/sync/RoomReadAccess.cpp index 73148922c..c9ca51c3a 100644 --- a/src/sync/RoomReadAccess.cpp +++ b/src/sync/RoomReadAccess.cpp @@ -79,9 +79,15 @@ namespace // The event that ended the user's most recent join, found by walking the chain of the // user's m.room.member events back from their current one (`ending_event_id`) until a // join. nullopt when there was no join in the chain or the chain is broken. + struct JoinEnding final + { + database::PersistentEvent const* boundary{nullptr}; + std::uint64_t join_ordering{0U}; + }; + [[nodiscard]] auto find_join_ending_event(database::PersistentStore const& store, std::string_view room_id, std::string_view user_id, std::string_view ending_event_id) - -> std::optional + -> std::optional { auto const find = [&store](std::string_view id) -> database::PersistentEvent const* { auto const found = store.events.find(id); @@ -104,7 +110,7 @@ namespace } if (step > 0U && *membership == "join" && !event_is_withheld(*event)) { - return find(child); + return JoinEnding{find(child), event->stream_ordering}; } if (step == 0U && *membership == "join") { @@ -170,11 +176,12 @@ auto room_read_access_for(database::PersistentStore const& store, std::string_vi return {}; } auto const boundary = find_join_ending_event(store, room_id, user_id, *member_event_id); - if (!boundary.has_value() || *boundary == nullptr) + if (!boundary.has_value() || boundary->boundary == nullptr) { return {}; } - return {RoomReadKind::as_of, (*boundary)->event_id, (*boundary)->stream_ordering}; + return {RoomReadKind::as_of, boundary->boundary->event_id, boundary->boundary->stream_ordering, + boundary->join_ordering}; } auto state_anchor_event(database::PersistentStore const& store, std::string_view room_id, RoomReadAccess const& access, From b0ca0c3bae485b2b9dc4d9cc272bc5673ebadce7 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 18:03:50 +0000 Subject: [PATCH 12/34] Move the use_state_after scenarios that assert project choices out of conformance Conformance holds only what the spec rows state. The left room's delta under full_state, the delta chain limit, the unreported quiet room, the room joined in the window and the lost-response and rejoin lazy-member cases are this project's decisions (ADR-0450, ADR-0453, ADR-0763 to ADR-0765), so they now live in tests/integration/test_sync_state_after_flow.cpp, unweakened. The helpers both files use are in tests/support/SyncStateAfterSupport.hpp. New there: a single-extremity room answered from the state groups, proven by state_rows_examined over a sixty-key state; the exact-string parse of use_state_after (True, 1 and empty are false); a room joined and left in one window, rejoined and left, joined at since, and a declined invite. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + tests/conformance/AGENTS.md | 2 +- .../test_sync_state_after_conformance.cpp | 336 +------------- .../test_sync_state_after_flow.cpp | 430 ++++++++++++++++++ tests/meson.build | 1 + tests/support/AGENTS.md | 1 + tests/support/SyncStateAfterSupport.hpp | 264 +++++++++++ 7 files changed, 702 insertions(+), 334 deletions(-) create mode 100644 tests/integration/test_sync_state_after_flow.cpp create mode 100644 tests/support/SyncStateAfterSupport.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index cb934d164..e4b9499b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) carries `state_after` and no `state` (ADR-0763, ADR-0764, ADR-0765). * The 1.1 series has its cross-release upgrade fixture: `tests/fixtures/upgrade/1.0.3/`, written by the 1.0.3 release, opened, migrated and served on SQLite and PostgreSQL by the `[fixture]` scenarios (ADR-0216). +* The `use_state_after` scenarios asserting project choices, not spec rows, moved from the conformance suite to + `tests/integration/test_sync_state_after_flow.cpp`. ## Security Fixes diff --git a/tests/conformance/AGENTS.md b/tests/conformance/AGENTS.md index c65ede896..84e14581e 100644 --- a/tests/conformance/AGENTS.md +++ b/tests/conformance/AGENTS.md @@ -106,7 +106,7 @@ Every test in this directory directly encodes a **MUST** or **SHOULD** from the | `test_sync_filter_conformance.cpp` | [CS API § Filtering](../../docs/matrix-v1.19-spec/client-server-api.md#filtering) | | `test_sync_summary_conformance.cpp` | [CS API § Syncing](../../docs/matrix-v1.19-spec/client-server-api.md#syncing) | | `test_sync_state_conformance.cpp` | [CS API `GET /sync`](../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync) (a joined room's `state` is the state up to the start of the timeline, not current state) | -| `test_sync_state_after_conformance.cpp` | [CS API `GET /sync`](../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync) (`use_state_after`: `state_after` present and `state` omitted with `true`, the reverse with `false` and by default, for joined and left rooms, initial and incremental, `full_state` and `lazy_load_members`; the state as of the end of the timeline; a left room's stops at the leave) | +| `test_sync_state_after_conformance.cpp` | [CS API `GET /sync`](../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync) (`use_state_after`: `state_after` present and `state` omitted with `true`, the reverse with `false` and by default, for joined and left rooms, initial and incremental, `full_state` and `lazy_load_members`; the state as of the end of the timeline; a left room's stops at the leave). Only what the spec rows state: what this project decides (a left room's delta under `full_state`, delta chain limits, an unreported quiet room, a room joined in the window, the lazy-member tracker) is in `tests/integration/test_sync_state_after_flow.cpp` | | `test_typing_conformance.cpp` | [CS API § Typing Notifications](../../docs/matrix-v1.19-spec/client-server-api.md#typing-notifications) (`typing` must be a boolean; typing expires after its `timeout`, never later than the server's ceiling) · [SS API § Typing Notifications](../../docs/matrix-v1.19-spec/server-server-api.md#typing-notifications) and [§ Server ACLs](../../docs/matrix-v1.19-spec/server-server-api.md#server-access-control-lists-acls) (an inbound `m.typing` EDU for a user not joined to the room, or from a server the room's ACL denies, is ignored; a remote flood at the cap never evicts a local user's typing) | | `test_x_matrix_auth_parsing.cpp` | [SS API § X-Matrix Auth](../../docs/matrix-v1.19-spec/server-server-api.md#request-authentication) | diff --git a/tests/conformance/test_sync_state_after_conformance.cpp b/tests/conformance/test_sync_state_after_conformance.cpp index bcbcdf625..b42cb196e 100644 --- a/tests/conformance/test_sync_state_after_conformance.cpp +++ b/tests/conformance/test_sync_state_after_conformance.cpp @@ -3,17 +3,8 @@ // // GET /_matrix/client/v3/sync `use_state_after` and the `state_after` section of joined and left rooms // (client-server-api.md, GET /sync, "Joined Room", "Left Room"; ADR-0763 to ADR-0765). -#include "../support/InMemoryDatabaseConfig.hpp" #include "../support/JsonTestSupport.hpp" -#include "../support/MasterKey.hpp" -#include "../support/RegistrationToken.hpp" -#include "merovingian/config/ClientRateLimitsConfig.hpp" -#include "merovingian/config/Config.hpp" -#include "merovingian/config/ListenersConfig.hpp" -#include "merovingian/config/LogModulesConfig.hpp" -#include "merovingian/config/SecurityConfig.hpp" -#include "merovingian/config/ServerConfig.hpp" -#include "merovingian/core/QueryParams.hpp" +#include "../support/SyncStateAfterSupport.hpp" #include "merovingian/homeserver/ClientServerRuntime.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" @@ -33,232 +24,7 @@ namespace { using namespace merovingian::tests; -using merovingian::canonicaljson::Object; -using merovingian::homeserver::ClientServerRuntime; - -constexpr auto k_state_after = std::string_view{"&use_state_after=true"}; -constexpr auto k_state_before = std::string_view{"&use_state_after=false"}; -constexpr auto k_no_filter = std::string_view{"{}"}; -constexpr auto k_one_event = std::string_view{R"({"room":{"timeline":{"limit":1}}})"}; -constexpr auto k_lazy = std::string_view{R"({"room":{"state":{"lazy_load_members":true}}})"}; -constexpr auto k_lazy_one_event = - std::string_view{R"({"room":{"timeline":{"limit":1},"state":{"lazy_load_members":true}}})"}; -constexpr auto k_include_leave = std::string_view{R"({"room":{"include_leave":true}})"}; - -[[nodiscard]] auto state_after_config() -> merovingian::config::Config -{ - auto security = merovingian::config::SecurityConfig{}; - security.secrets.master_key_file = merovingian::tests::shared_master_key_file(); - merovingian::tests::enable_token_registration(security); - return { - merovingian::config::ServerConfig{}, - merovingian::config::ListenersConfig{}, - merovingian::tests::in_memory_database_config(), - security, - merovingian::config::ClientRateLimitsConfig{}, - merovingian::config::LogModulesConfig{}, - }; -} - -struct Reply final -{ - std::uint16_t status{0U}; - std::string body{}; -}; - -[[nodiscard]] auto call(ClientServerRuntime& rt, std::string method, std::string target, std::string const& token, - std::string body = {}) -> Reply -{ - auto const result = merovingian::homeserver::handle_client_server_request( - rt, {std::move(method), std::move(target), token, std::move(body)}, /*can_wait=*/false); - return {static_cast(result.response.status), result.response.body}; -} - -[[nodiscard]] auto user_id(std::string_view localpart) -> std::string -{ - return "@" + std::string{localpart} + ":example.org"; -} - -[[nodiscard]] auto register_user(ClientServerRuntime& rt, std::string const& localpart) -> std::string -{ - auto const registered = - call(rt, "POST", "/_matrix/client/v3/register", {}, registration_json(localpart, "CorrectHorse7!")); - REQUIRE(registered.status == 200U); - return *string_member(parse_object(registered.body), "access_token"); -} - -[[nodiscard]] auto create_room(ClientServerRuntime& rt, std::string const& token, std::string const& preset) - -> std::string -{ - auto const created = call(rt, "POST", "/_matrix/client/v3/createRoom", token, R"({"preset":")" + preset + R"("})"); - REQUIRE(created.status == 200U); - return *string_member(parse_object(created.body), "room_id"); -} - -auto room_post(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& action, - std::string body = "{}") -> void -{ - REQUIRE(call(rt, "POST", "/_matrix/client/v3/rooms/" + room_id + "/" + action, token, std::move(body)).status == - 200U); -} - -auto say(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& text) -> void -{ - static auto counter = std::uint64_t{0U}; - auto const sent = - call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/send/m.room.message/sa" + std::to_string(++counter), - token, R"({"msgtype":"m.text","body":")" + text + R"("})"); - REQUIRE(sent.status == 200U); -} - -auto set_topic(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& topic) - -> std::string -{ - auto const sent = call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/state/m.room.topic/", token, - R"({"topic":")" + topic + R"("})"); - REQUIRE(sent.status == 200U); - return *string_member(parse_object(sent.body), "event_id"); -} - -auto set_display_name(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, - std::string const& localpart, std::string const& name) -> void -{ - auto const sent = - call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/state/m.room.member/" + user_id(localpart), token, - R"({"membership":"join","displayname":")" + name + R"("})"); - REQUIRE(sent.status == 200U); -} - -struct Synced final -{ - Object body{}; - std::string next_batch{}; -}; - -[[nodiscard]] auto sync(ClientServerRuntime& rt, std::string const& token, std::string_view filter, - std::string const& since = {}, std::string_view extra = {}) -> Synced -{ - auto target = - std::string{"/_matrix/client/v3/sync?filter="} + merovingian::core::percent_encode_path_component(filter); - if (!since.empty()) - { - target += "&since=" + merovingian::core::percent_encode_path_component(since); - } - target += extra; - auto const reply = call(rt, "GET", target, token); - REQUIRE(reply.status == 200U); - auto body = parse_object(reply.body); - auto next = *string_member(body, "next_batch"); - return {std::move(body), std::move(next)}; -} - -// rooms.
., or nullptr. -[[nodiscard]] auto room_section(Synced const& synced, std::string_view section, std::string const& room_id) - -> Object const* -{ - auto const* rooms = object_member_as_object(synced.body, "rooms"); - auto const* group = rooms == nullptr ? nullptr : object_member_as_object(*rooms, section); - return group == nullptr ? nullptr : object_member_as_object(*group, room_id); -} - -[[nodiscard]] auto required_room(Synced const& synced, std::string_view section, std::string const& room_id) - -> Object const& -{ - auto const* room = room_section(synced, section, room_id); - REQUIRE(room != nullptr); - return *room; -} - -// Whether `room` has the member `part`, whatever it holds. -[[nodiscard]] auto has_part(Object const& room, std::string_view part) -> bool -{ - return object_member_as_object(room, part) != nullptr; -} - -// The events of `room.`, which must be present. -[[nodiscard]] auto events_of(Object const& room, std::string_view part) -> std::vector -{ - auto const* section = object_member_as_object(room, part); - REQUIRE(section != nullptr); - auto const* events = object_member_as_array(*section, "events"); - REQUIRE(events != nullptr); - auto out = std::vector{}; - for (auto const& entry : *events) - { - out.push_back(std::get(entry.storage())); - } - return out; -} - -using Keys = std::multiset; - -[[nodiscard]] auto event_types(std::vector const& events) -> Keys -{ - auto types = Keys{}; - for (auto const& event : events) - { - types.insert(*string_member(event, "type")); - } - return types; -} - -// The state_key of every m.room.member event. -[[nodiscard]] auto member_keys(std::vector const& events) -> Keys -{ - auto keys = Keys{}; - for (auto const& event : events) - { - if (*string_member(event, "type") == "m.room.member") - { - keys.insert(*string_member(event, "state_key")); - } - } - return keys; -} - -// `content.` of the event of (type, state_key), or nullopt when `events` has none. -[[nodiscard]] auto content_of(std::vector const& events, std::string_view type, std::string_view state_key, - std::string_view field) -> std::optional -{ - for (auto const& event : events) - { - auto const* key = string_member(event, "state_key"); - if (*string_member(event, "type") == type && key != nullptr && *key == state_key) - { - auto const* content = object_member_as_object(event, "content"); - auto const* value = content == nullptr ? nullptr : string_member(*content, field); - return value == nullptr ? std::optional{} : std::optional{*value}; - } - } - return std::nullopt; -} - -// The `content.topic` of every m.room.topic event in `events`. -[[nodiscard]] auto topics_in(std::vector const& events) -> std::vector -{ - auto topics = std::vector{}; - for (auto const& event : events) - { - if (*string_member(event, "type") == "m.room.topic") - { - auto const* content = object_member_as_object(event, "content"); - topics.push_back(*string_member(*content, "topic")); - } - } - return topics; -} - -auto require_state_after_only(Object const& room) -> void -{ - REQUIRE(has_part(room, "state_after")); - REQUIRE_FALSE(has_part(room, "state")); -} - -auto require_state_only(Object const& room) -> void -{ - REQUIRE(has_part(room, "state")); - REQUIRE_FALSE(has_part(room, "state_after")); -} +using namespace merovingian::tests::state_after_support; } // namespace @@ -447,30 +213,6 @@ SCENARIO("An incremental sync's state_after holds what changed up to the end of REQUIRE(events_of(joined, "state_after").empty()); } } - WHEN("nothing happened") - { - auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); - - THEN("the room is not reported") - { - REQUIRE(room_section(synced, "join", room) == nullptr); - } - } - WHEN("more state changes than one delta chain can hold happened") - { - for (auto index = 0; index < 120; ++index) - { - set_topic(rt, alice, room, "t" + std::to_string(index)); - } - auto const synced = sync(rt, alice, k_one_event, initial.next_batch, k_state_after); - - THEN("state_after still holds only the key that changed, with its last value") - { - auto const after = events_of(required_room(synced, "join", room), "state_after"); - REQUIRE(event_types(after) == Keys{"m.room.topic"}); - REQUIRE(topics_in(after) == std::vector{"t119"}); - } - } } } @@ -584,43 +326,6 @@ SCENARIO("full_state=true returns the whole state in the section use_state_after } } -// Spec: Matrix Client-Server API v1.19 -// Endpoint / Section: GET /_matrix/client/v3/sync, query parameter `use_state_after`; Joined Room `state_after` -// URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync -// -// A room the client had no state for: it joined inside the window, so a delta would tell it nothing. -SCENARIO("state_after of a room joined in the window is the room's whole state", - "[conformance][client-server][sync][state-after]") -{ - GIVEN("a room bob joins after his first sync") - { - auto started = merovingian::homeserver::start_client_server(state_after_config()); - REQUIRE(started.started); - auto& rt = started.runtime; - auto const alice = register_user(rt, "alice"); - auto const bob = register_user(rt, "bob"); - auto const room = create_room(rt, alice, "public_chat"); - set_topic(rt, alice, room, "first"); - auto const initial = sync(rt, bob, k_no_filter, {}, k_state_after); - room_post(rt, bob, room, "join"); - - WHEN("bob syncs with use_state_after=true") - { - auto const synced = sync(rt, bob, k_no_filter, initial.next_batch, k_state_after); - auto const& joined = required_room(synced, "join", room); - - THEN("state_after holds all state, his own join included, and state is omitted") - { - require_state_after_only(joined); - auto const after = events_of(joined, "state_after"); - REQUIRE(event_types(after).contains("m.room.create")); - REQUIRE(topics_in(after) == std::vector{"first"}); - REQUIRE(content_of(after, "m.room.member", user_id("bob"), "membership") == "join"); - } - } - } -} - // Spec: Matrix Client-Server API v1.19 // Endpoint / Section: GET /_matrix/client/v3/sync, Left Room // URL: ../../docs/matrix-v1.19-spec/client-server-api.md#get_matrixclientv3sync @@ -706,10 +411,9 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", auto const synced = sync(rt, dave, k_no_filter, synced_before.next_batch, extra); auto const& left = required_room(synced, "leave", room); - THEN("state_after is present and state is omitted; he is no longer a member, so it stays a delta") + THEN("state_after is present and state is omitted") { require_state_after_only(left); - REQUIRE_FALSE(event_types(events_of(left, "state_after")).contains("m.room.create")); } } WHEN("dave makes an incremental sync with full_state=true and use_state_after=false") @@ -859,20 +563,6 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(member_keys(after).empty()); } } - WHEN("the response that carried carol's membership is lost and the client retries") - { - auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); - say(rt, carol, room, "carol again"); - auto const lost = sync(rt, alice, k_lazy, first.next_batch, k_state_after); - REQUIRE(member_keys(events_of(required_room(lost, "join", room), "state_after")) == Keys{user_id("carol")}); - auto const retried = sync(rt, alice, k_lazy, first.next_batch, k_state_after); - - THEN("it is sent again") - { - REQUIRE(member_keys(events_of(required_room(retried, "join", room), "state_after")) == - Keys{user_id("carol")}); - } - } WHEN("bob speaks and then changes his display name in the same window") { auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); @@ -931,26 +621,6 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE_FALSE(member_keys(after).contains(user_id("carol"))); } } - WHEN("the client acknowledged bob, and bob leaves and rejoins") - { - auto const bob_first = sync(rt, bob, k_lazy, {}, k_state_after); - say(rt, alice, room, "alice"); - auto const bob_second = sync(rt, bob, k_lazy, bob_first.next_batch, k_state_after); - auto const bob_third = sync(rt, bob, k_lazy, bob_second.next_batch, k_state_after); - room_post(rt, bob, room, "leave"); - auto const bob_left = sync(rt, bob, k_lazy, bob_third.next_batch, k_state_after); - room_post(rt, bob, room, "join"); - say(rt, alice, room, "welcome back"); - auto const rejoined = sync(rt, bob, k_lazy, bob_left.next_batch, k_state_after); - auto const after = events_of(required_room(rejoined, "join", room), "state_after"); - - THEN("the tracker is not consulted, and his own join is in state_after though the timeline carries it") - { - REQUIRE(member_keys(after).contains(user_id("alice"))); - REQUIRE(content_of(after, "m.room.member", user_id("bob"), "membership") == "join"); - REQUIRE(event_types(after).contains("m.room.create")); - } - } } GIVEN("a room of alice and dave that dave leaves after bob spoke") { diff --git a/tests/integration/test_sync_state_after_flow.cpp b/tests/integration/test_sync_state_after_flow.cpp new file mode 100644 index 000000000..7ae3b3509 --- /dev/null +++ b/tests/integration/test_sync_state_after_flow.cpp @@ -0,0 +1,430 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// GET /_matrix/client/v3/sync `use_state_after`: what this project decides beyond the spec's rows, driven through the +// client-server stack (ADR-0763 to ADR-0765). What the spec states is in +// tests/conformance/test_sync_state_after_conformance.cpp. +#include "../support/JsonTestSupport.hpp" +#include "../support/SyncStateAfterSupport.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerStartResult.hpp" + +#include + +#include +#include +#include +#include + +namespace +{ + +using namespace merovingian::tests; +using namespace merovingian::tests::state_after_support; + +constexpr auto k_full_state_after = std::string_view{"&full_state=true&use_state_after=true"}; + +} // namespace + +// ADR-0763: a sync that finds nothing new in a joined room does not report it. +SCENARIO("A joined room with nothing new is not reported with use_state_after", + "[integration][sync][state-after][flow]") +{ + GIVEN("a room alice has synced") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + auto const initial = sync(rt, alice, k_no_filter, {}, k_state_after); + + WHEN("nothing happened") + { + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + + THEN("the room is not reported") + { + REQUIRE(room_section(synced, "join", room) == nullptr); + } + } + } +} + +// ADR-0764: a delta chain is bounded (`max_state_group_delta_depth`); a window with more changes than it holds falls +// back to comparing the two states and still sends one event per changed key. +SCENARIO("A state_after delta survives more state changes than one delta chain can hold", + "[integration][sync][state-after][flow]") +{ + GIVEN("a room alice has synced") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + auto const initial = sync(rt, alice, k_no_filter, {}, k_state_after); + + WHEN("more state changes than one delta chain can hold happened") + { + for (auto index = 0; index < 120; ++index) + { + set_topic(rt, alice, room, "t" + std::to_string(index)); + } + auto const synced = sync(rt, alice, k_one_event, initial.next_batch, k_state_after); + + THEN("state_after still holds only the key that changed, with its last value") + { + auto const after = events_of(required_room(synced, "join", room), "state_after"); + REQUIRE(event_types(after) == Keys{"m.room.topic"}); + REQUIRE(topics_in(after) == std::vector{"t119"}); + } + } + } +} + +// ADR-0764: the chain walk (`state_changes_along_chain`) answers a single-extremity room from the state groups alone. +// The fallback reads the room's current state rows, which `state_rows_examined` counts, so a wide state shows which +// path ran. +SCENARIO("A state_after delta of an unforked room is read from the state groups, not the room's state", + "[integration][sync][state-after][flow][state-index]") +{ + GIVEN("a room with sixty state events that alice has synced, and one forward extremity") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + for (auto index = 0; index < 60; ++index) + { + auto const sent = + call(rt, "PUT", "/_matrix/client/v3/rooms/" + room + "/state/org.example.key/k" + std::to_string(index), + alice, R"({"value":"v"})"); + REQUIRE(sent.status == 200U); + } + auto const initial = sync(rt, alice, k_no_filter, {}, k_state_after); + auto const& store = rt.homeserver.database.persistent_store; + REQUIRE(store.forward_extremities.tips_of_room(room).size() == 1U); + + WHEN("a topic change arrives and alice syncs with use_state_after=true") + { + set_topic(rt, alice, room, "second"); + auto const before = store.state_rows_examined.value(); + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + auto const examined = store.state_rows_examined.value() - before; + + THEN("only the topic is sent, and far fewer than sixty state rows were read") + { + auto const after = events_of(required_room(synced, "join", room), "state_after"); + REQUIRE(event_types(after) == Keys{"m.room.topic"}); + REQUIRE(examined < 30U); + } + } + WHEN("only a message arrives and alice syncs with use_state_after=true") + { + say(rt, alice, room, "hello"); + auto const before = store.state_rows_examined.value(); + auto const synced = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + auto const examined = store.state_rows_examined.value() - before; + + THEN("state_after is empty, and far fewer than sixty state rows were read") + { + REQUIRE(events_of(required_room(synced, "join", room), "state_after").empty()); + REQUIRE(examined < 30U); + } + } + } +} + +// The parser accepts only the exact string `true`, as `full_state` does (ADR-0763). +SCENARIO("use_state_after is true only for the exact string true", "[integration][sync][state-after][flow]") +{ + GIVEN("a room alice has synced") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const room = create_room(rt, alice, "public_chat"); + auto const initial = sync(rt, alice, k_no_filter); + set_topic(rt, alice, room, "second"); + + WHEN("the value is True, 1 or empty") + { + auto const capital = sync(rt, alice, k_no_filter, initial.next_batch, "&use_state_after=True"); + auto const one = sync(rt, alice, k_no_filter, initial.next_batch, "&use_state_after=1"); + auto const empty = sync(rt, alice, k_no_filter, initial.next_batch, "&use_state_after="); + + THEN("each is read as false: state is present and state_after is omitted") + { + require_state_only(required_room(capital, "join", room)); + require_state_only(required_room(one, "join", room)); + require_state_only(required_room(empty, "join", room)); + } + } + WHEN("the value is true") + { + auto const exact = sync(rt, alice, k_no_filter, initial.next_batch, k_state_after); + + THEN("state_after is present and state is omitted") + { + require_state_after_only(required_room(exact, "join", room)); + } + } + } +} + +// ADR-0763: a client that joined inside the window held nothing of the room, so it is sent the whole state, with its +// own join, as an initial sync would. (The spec does not state it for an incremental sync.) +SCENARIO("state_after of a room joined in the window is the room's whole state", + "[integration][sync][state-after][flow]") +{ + GIVEN("a room bob joins after his first sync") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const bob = register_user(rt, "bob"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + auto const initial = sync(rt, bob, k_no_filter, {}, k_state_after); + room_post(rt, bob, room, "join"); + + WHEN("bob syncs with use_state_after=true") + { + auto const synced = sync(rt, bob, k_no_filter, initial.next_batch, k_state_after); + auto const& joined = required_room(synced, "join", room); + + THEN("state_after holds all state, his own join included, and state is omitted") + { + require_state_after_only(joined); + auto const after = events_of(joined, "state_after"); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(topics_in(after) == std::vector{"first"}); + REQUIRE(content_of(after, "m.room.member", user_id("bob"), "membership") == "join"); + } + } + } +} + +// ADR-0765: the spec's Left Room `state_after` row does not name `full_state`, so a left room's `state_after` stays a +// delta, as its `state` does (ADR-0453). +SCENARIO("full_state=true leaves a left room's state_after a delta", "[integration][sync][state-after][flow]") +{ + GIVEN("a room dave joined and synced, and whose topic changed before he left") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + set_topic(rt, alice, room, "first"); + room_post(rt, dave, room, "join"); + auto const synced_before = sync(rt, dave, k_no_filter, {}, k_state_after); + set_topic(rt, alice, room, "changed before the leave"); + room_post(rt, dave, room, "leave"); + + WHEN("dave makes an incremental sync with full_state=true and use_state_after=true") + { + auto const synced = sync(rt, dave, k_no_filter, synced_before.next_batch, k_full_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after is the delta: the topic and his leave, not the whole state") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + REQUIRE_FALSE(event_types(after).contains("m.room.create")); + REQUIRE(event_types(after) == Keys{"m.room.member", "m.room.topic"}); + } + } + } +} + +// ADR-0765: a client that was not joined at `since` held nothing of the room, so a left room's `state_after` is the +// whole state as of the leave, not a delta that would hide the create event, power levels and join rules. +SCENARIO("A room joined and left inside one sync window gives state_after its whole readable state", + "[integration][sync][state-after][flow]") +{ + GIVEN("a room dave joins and leaves after his last sync, in which the topic changed before he joined") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + auto const before = sync(rt, dave, k_no_filter, {}, k_state_after); + set_topic(rt, alice, room, "before dave"); + room_post(rt, dave, room, "join"); + say(rt, dave, room, "hello and goodbye"); + room_post(rt, dave, room, "leave"); + set_topic(rt, alice, room, "after dave left"); + + WHEN("dave syncs with use_state_after=true") + { + auto const synced = sync(rt, dave, k_no_filter, before.next_batch, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after is the room as of his leave: create, power levels, join rules, the topic and his leave") + { + require_state_after_only(left); + auto const after = events_of(left, "state_after"); + auto const types = event_types(after); + REQUIRE(types.contains("m.room.create")); + REQUIRE(types.contains("m.room.power_levels")); + REQUIRE(types.contains("m.room.join_rules")); + REQUIRE(topics_in(after) == std::vector{"before dave"}); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + REQUIRE(member_keys(after).count(user_id("dave")) == 1U); + } + } + WHEN("dave syncs with lazy-loading and use_state_after=true") + { + auto const synced = sync(rt, dave, k_lazy, before.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "leave", room), "state_after"); + + THEN("the whole state comes with only his own membership, once, as the leave") + { + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(member_keys(after).count(user_id("dave")) == 1U); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + } + } + } + GIVEN("a room dave left and synced, then rejoined and left again before his next sync") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, dave, room, "join"); + room_post(rt, dave, room, "leave"); + auto const after_first_leave = sync(rt, dave, k_no_filter, {}, k_state_after); + room_post(rt, dave, room, "join"); + room_post(rt, dave, room, "leave"); + + WHEN("dave syncs with use_state_after=true") + { + auto const synced = sync(rt, dave, k_no_filter, after_first_leave.next_batch, k_state_after); + auto const& left = required_room(synced, "leave", room); + + THEN("state_after is the whole readable state as of the second leave") + { + auto const after = events_of(left, "state_after"); + REQUIRE(event_types(after).contains("m.room.create")); + REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); + } + } + } + GIVEN("a room dave was joined in at his last sync and left after its topic changed") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const dave = register_user(rt, "dave"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, dave, room, "join"); + auto const joined = sync(rt, dave, k_no_filter, {}, k_state_after); + set_topic(rt, alice, room, "changed"); + room_post(rt, dave, room, "leave"); + + WHEN("dave syncs with use_state_after=true") + { + auto const synced = sync(rt, dave, k_no_filter, joined.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "leave", room), "state_after"); + + THEN("state_after is still the delta, as the client holds the rest of the state") + { + REQUIRE_FALSE(event_types(after).contains("m.room.create")); + REQUIRE(event_types(after) == Keys{"m.room.member", "m.room.topic"}); + } + } + } + GIVEN("a private room erin was invited to and declined inside one window") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const erin = register_user(rt, "erin"); + auto const room = create_room(rt, alice, "private_chat"); + auto const before = sync(rt, erin, k_no_filter, {}, k_state_after); + room_post(rt, alice, room, "invite", R"({"user_id":")" + user_id("erin") + R"("})"); + room_post(rt, erin, room, "leave"); + + WHEN("erin syncs with use_state_after=true") + { + auto const synced = sync(rt, erin, k_no_filter, before.next_batch, k_state_after); + auto const after = events_of(required_room(synced, "leave", room), "state_after"); + + THEN("she is told only her own leave, as she never could read the room") + { + REQUIRE(event_types(after) == Keys{"m.room.member"}); + REQUIRE(content_of(after, "m.room.member", user_id("erin"), "membership") == "leave"); + } + } + } +} + +// ADR-0450: a lost response must not make the server think the client holds a member it never received. +SCENARIO("A lazy-loaded member whose response was lost is sent again in state_after", + "[integration][sync][state-after][flow]") +{ + GIVEN("a room of alice, bob and carol, in which bob and carol have spoken") + { + auto started = merovingian::homeserver::start_client_server(state_after_config()); + REQUIRE(started.started); + auto& rt = started.runtime; + auto const alice = register_user(rt, "alice"); + auto const bob = register_user(rt, "bob"); + auto const carol = register_user(rt, "carol"); + auto const room = create_room(rt, alice, "public_chat"); + room_post(rt, bob, room, "join"); + room_post(rt, carol, room, "join"); + set_topic(rt, alice, room, "first"); + say(rt, carol, room, "carol"); + say(rt, bob, room, "bob"); + + WHEN("the response that carried carol's membership is lost and the client retries") + { + auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); + say(rt, carol, room, "carol again"); + auto const lost = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + REQUIRE(member_keys(events_of(required_room(lost, "join", room), "state_after")) == Keys{user_id("carol")}); + auto const retried = sync(rt, alice, k_lazy, first.next_batch, k_state_after); + + THEN("it is sent again") + { + REQUIRE(member_keys(events_of(required_room(retried, "join", room), "state_after")) == + Keys{user_id("carol")}); + } + } + WHEN("the client acknowledged bob, and bob leaves and rejoins") + { + auto const bob_first = sync(rt, bob, k_lazy, {}, k_state_after); + say(rt, alice, room, "alice"); + auto const bob_second = sync(rt, bob, k_lazy, bob_first.next_batch, k_state_after); + auto const bob_third = sync(rt, bob, k_lazy, bob_second.next_batch, k_state_after); + room_post(rt, bob, room, "leave"); + auto const bob_left = sync(rt, bob, k_lazy, bob_third.next_batch, k_state_after); + room_post(rt, bob, room, "join"); + say(rt, alice, room, "welcome back"); + auto const rejoined = sync(rt, bob, k_lazy, bob_left.next_batch, k_state_after); + auto const after = events_of(required_room(rejoined, "join", room), "state_after"); + + THEN("the tracker is not consulted, and his own join is in state_after though the timeline carries it") + { + REQUIRE(member_keys(after).contains(user_id("alice"))); + REQUIRE(content_of(after, "m.room.member", user_id("bob"), "membership") == "join"); + REQUIRE(event_types(after).contains("m.room.create")); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index 412633f9a..61ad0bb16 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -1004,6 +1004,7 @@ shell = find_program('sh') 'integration/test_server_signing_key_lifecycle_flow.cpp', 'integration/test_sync_dag_gap_flow.cpp', 'integration/test_sync_handler_flow.cpp', + 'integration/test_sync_state_after_flow.cpp', 'integration/test_sync_stream_position_flow.cpp', 'integration/test_turn_shared_secret_flow.cpp', ), diff --git a/tests/support/AGENTS.md b/tests/support/AGENTS.md index e6ed7cd03..9ff3cd6fb 100644 --- a/tests/support/AGENTS.md +++ b/tests/support/AGENTS.md @@ -20,6 +20,7 @@ Shared test utilities used across unit, conformance, and integration tests. | `IngestedStateFixture.hpp` | `record_ingested_state()` / `record_ingested_state_for_room()`: for a fixture that writes events straight into a `PersistentStore` and then asks for the state AT an event (federation `/state`, `/state_ids`, client `/context`), records the state group ingestion would have recorded for each event (state before = resolution of the prev_events' states, state after = that plus the event, unchanged for a rejected one). Those reads answer from the recorded state, never from a walk of the event graph (ADR-0363). Call it parents first | | `CrossSigningFixture.hpp` | `make_cross_signing_set()`: a master, self-signing and user-signing key with real Ed25519 keys and real signatures (the self-signing and user-signing keys signed by the master key), plus `sign_cross_signing_key_json()`, `cross_signing_upload_body()` and `fresh_cross_signing_upload_body()`. `POST /keys/device_signing/upload` verifies those signatures (ADR-0421), so a test that uploads cross-signing keys uses this, never invented key or signature text | | `ForgedUnsigned.hpp` | `with_forged_unsigned()` / `forged_unsigned_json`: a signed PDU with the `unsigned` a hostile relaying server would write (`m.relations`, `redacted_because`, `prev_content`, `age`, `transaction_id`, `membership`). The signatures and event ID still hold, so it ingests like the genuine PDU (ADR-0402) | +| `SyncStateAfterSupport.hpp` | `state_after_support`: the config, request, `/sync` and response-reading helpers the `use_state_after` conformance and integration scenarios share (`state_after_config`, `call`, `register_user`, `create_room`, `sync`, `required_room`, `events_of`, `event_types`, `member_keys`, ...). Drives the runtime the caller started | | `RegistrationToken.hpp` | Generates registration tokens for test users without going through the full UIAA flow | | `FakeClamdServer.hpp` | `FakeClamdServer`: a clamd stand-in on a temporary AF_UNIX socket. Reads INSTREAM exchanges, records the streamed bytes and answers with a fixed reply (or stays silent for the timeout case). Another I/O exception, shared by the unit and integration suites (G15, ADR-0565); assert on its recorded data from the test thread only | | `TempDirectory.hpp` | `temporary_directory()`: the fallback-aware temporary directory for test scratch files; `unique_temporary_path(stem, extension)`: a name under it unique to the process and call (pid, counter, random), not created. Every scratch file or directory a test creates is named with it, never a fixed name (ADR-0753) | diff --git a/tests/support/SyncStateAfterSupport.hpp b/tests/support/SyncStateAfterSupport.hpp new file mode 100644 index 000000000..518e045a2 --- /dev/null +++ b/tests/support/SyncStateAfterSupport.hpp @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +// Helpers the `use_state_after` conformance and integration scenarios share: a runtime configuration, requests against +// the in-process client-server stack, and readers of a `/sync` response's rooms and state. They drive the runtime the +// caller started and assert with Catch2; they open nothing themselves. + +#include "InMemoryDatabaseConfig.hpp" +#include "JsonTestSupport.hpp" +#include "MasterKey.hpp" +#include "RegistrationToken.hpp" +#include "merovingian/config/ClientRateLimitsConfig.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ListenersConfig.hpp" +#include "merovingian/config/LogModulesConfig.hpp" +#include "merovingian/config/SecurityConfig.hpp" +#include "merovingian/config/ServerConfig.hpp" +#include "merovingian/core/QueryParams.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace merovingian::tests::state_after_support +{ + +using merovingian::canonicaljson::Object; +using merovingian::homeserver::ClientServerRuntime; + +constexpr auto k_state_after = std::string_view{"&use_state_after=true"}; +constexpr auto k_state_before = std::string_view{"&use_state_after=false"}; +constexpr auto k_no_filter = std::string_view{"{}"}; +constexpr auto k_one_event = std::string_view{R"({"room":{"timeline":{"limit":1}}})"}; +constexpr auto k_lazy = std::string_view{R"({"room":{"state":{"lazy_load_members":true}}})"}; +constexpr auto k_lazy_one_event = + std::string_view{R"({"room":{"timeline":{"limit":1},"state":{"lazy_load_members":true}}})"}; +constexpr auto k_include_leave = std::string_view{R"({"room":{"include_leave":true}})"}; + +[[nodiscard]] inline auto state_after_config() -> merovingian::config::Config +{ + auto security = merovingian::config::SecurityConfig{}; + security.secrets.master_key_file = shared_master_key_file(); + enable_token_registration(security); + return { + merovingian::config::ServerConfig{}, + merovingian::config::ListenersConfig{}, + in_memory_database_config(), + security, + merovingian::config::ClientRateLimitsConfig{}, + merovingian::config::LogModulesConfig{}, + }; +} + +struct Reply final +{ + std::uint16_t status{0U}; + std::string body{}; +}; + +[[nodiscard]] inline auto call(ClientServerRuntime& rt, std::string method, std::string target, + std::string const& token, std::string body = {}) -> Reply +{ + auto const result = merovingian::homeserver::handle_client_server_request( + rt, {std::move(method), std::move(target), token, std::move(body)}, /*can_wait=*/false); + return {static_cast(result.response.status), result.response.body}; +} + +[[nodiscard]] inline auto user_id(std::string_view localpart) -> std::string +{ + return "@" + std::string{localpart} + ":example.org"; +} + +[[nodiscard]] inline auto register_user(ClientServerRuntime& rt, std::string const& localpart) -> std::string +{ + auto const registered = + call(rt, "POST", "/_matrix/client/v3/register", {}, registration_json(localpart, "CorrectHorse7!")); + REQUIRE(registered.status == 200U); + return *string_member(parse_object(registered.body), "access_token"); +} + +[[nodiscard]] inline auto create_room(ClientServerRuntime& rt, std::string const& token, std::string const& preset) + -> std::string +{ + auto const created = call(rt, "POST", "/_matrix/client/v3/createRoom", token, R"({"preset":")" + preset + R"("})"); + REQUIRE(created.status == 200U); + return *string_member(parse_object(created.body), "room_id"); +} + +inline auto room_post(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, + std::string const& action, std::string body = "{}") -> void +{ + REQUIRE(call(rt, "POST", "/_matrix/client/v3/rooms/" + room_id + "/" + action, token, std::move(body)).status == + 200U); +} + +inline auto say(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, std::string const& text) + -> void +{ + static auto counter = std::uint64_t{0U}; + auto const sent = + call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/send/m.room.message/sa" + std::to_string(++counter), + token, R"({"msgtype":"m.text","body":")" + text + R"("})"); + REQUIRE(sent.status == 200U); +} + +inline auto set_topic(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, + std::string const& topic) -> std::string +{ + auto const sent = call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/state/m.room.topic/", token, + R"({"topic":")" + topic + R"("})"); + REQUIRE(sent.status == 200U); + return *string_member(parse_object(sent.body), "event_id"); +} + +inline auto set_display_name(ClientServerRuntime& rt, std::string const& token, std::string const& room_id, + std::string const& localpart, std::string const& name) -> void +{ + auto const sent = + call(rt, "PUT", "/_matrix/client/v3/rooms/" + room_id + "/state/m.room.member/" + user_id(localpart), token, + R"({"membership":"join","displayname":")" + name + R"("})"); + REQUIRE(sent.status == 200U); +} + +struct Synced final +{ + Object body{}; + std::string next_batch{}; +}; + +[[nodiscard]] inline auto sync(ClientServerRuntime& rt, std::string const& token, std::string_view filter, + std::string const& since = {}, std::string_view extra = {}) -> Synced +{ + auto target = + std::string{"/_matrix/client/v3/sync?filter="} + merovingian::core::percent_encode_path_component(filter); + if (!since.empty()) + { + target += "&since=" + merovingian::core::percent_encode_path_component(since); + } + target += extra; + auto const reply = call(rt, "GET", target, token); + REQUIRE(reply.status == 200U); + auto body = parse_object(reply.body); + auto next = *string_member(body, "next_batch"); + return {std::move(body), std::move(next)}; +} + +// rooms.
., or nullptr. +[[nodiscard]] inline auto room_section(Synced const& synced, std::string_view section, std::string const& room_id) + -> Object const* +{ + auto const* rooms = object_member_as_object(synced.body, "rooms"); + auto const* group = rooms == nullptr ? nullptr : object_member_as_object(*rooms, section); + return group == nullptr ? nullptr : object_member_as_object(*group, room_id); +} + +[[nodiscard]] inline auto required_room(Synced const& synced, std::string_view section, std::string const& room_id) + -> Object const& +{ + auto const* room = room_section(synced, section, room_id); + REQUIRE(room != nullptr); + return *room; +} + +// Whether `room` has the member `part`, whatever it holds. +[[nodiscard]] inline auto has_part(Object const& room, std::string_view part) -> bool +{ + return object_member_as_object(room, part) != nullptr; +} + +// The events of `room.`, which must be present. +[[nodiscard]] inline auto events_of(Object const& room, std::string_view part) -> std::vector +{ + auto const* section = object_member_as_object(room, part); + REQUIRE(section != nullptr); + auto const* events = object_member_as_array(*section, "events"); + REQUIRE(events != nullptr); + auto out = std::vector{}; + for (auto const& entry : *events) + { + out.push_back(std::get(entry.storage())); + } + return out; +} + +using Keys = std::multiset; + +[[nodiscard]] inline auto event_types(std::vector const& events) -> Keys +{ + auto types = Keys{}; + for (auto const& event : events) + { + types.insert(*string_member(event, "type")); + } + return types; +} + +// The state_key of every m.room.member event. +[[nodiscard]] inline auto member_keys(std::vector const& events) -> Keys +{ + auto keys = Keys{}; + for (auto const& event : events) + { + if (*string_member(event, "type") == "m.room.member") + { + keys.insert(*string_member(event, "state_key")); + } + } + return keys; +} + +// `content.` of the event of (type, state_key), or nullopt when `events` has none. +[[nodiscard]] inline auto content_of(std::vector const& events, std::string_view type, + std::string_view state_key, std::string_view field) -> std::optional +{ + for (auto const& event : events) + { + auto const* key = string_member(event, "state_key"); + if (*string_member(event, "type") == type && key != nullptr && *key == state_key) + { + auto const* content = object_member_as_object(event, "content"); + auto const* value = content == nullptr ? nullptr : string_member(*content, field); + return value == nullptr ? std::optional{} : std::optional{*value}; + } + } + return std::nullopt; +} + +// The `content.topic` of every m.room.topic event in `events`. +[[nodiscard]] inline auto topics_in(std::vector const& events) -> std::vector +{ + auto topics = std::vector{}; + for (auto const& event : events) + { + if (*string_member(event, "type") == "m.room.topic") + { + auto const* content = object_member_as_object(event, "content"); + topics.push_back(*string_member(*content, "topic")); + } + } + return topics; +} + +inline auto require_state_after_only(Object const& room) -> void +{ + REQUIRE(has_part(room, "state_after")); + REQUIRE_FALSE(has_part(room, "state")); +} + +inline auto require_state_only(Object const& room) -> void +{ + REQUIRE(has_part(room, "state")); + REQUIRE_FALSE(has_part(room, "state_after")); +} + +} // namespace merovingian::tests::state_after_support From df7fa640f42f08758e4f12d8d682b2aecccf1dd8 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 18:03:59 +0000 Subject: [PATCH 13/34] Correct the sync row of the capability gaps and track the state_after fork limit A limited incremental sync sends the whole state up to the start of the timeline as `state`, which covers the gap, and `state_after` holds what changed in it, so the row's "Not done" sentence was false. ADR-0764's accepted limit (a key that state resolution flips back to the value recorded at since after a fork is not sent; full_state=true repairs the client) is now a tracked gap. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 ++ docs/todos/capability-gaps.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4b9499b7..6a078bc8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) absence no longer walks every row it missed under the runtime lock. * `use_state_after`: a room joined and left inside one sync window now gets its whole readable state as of the leave in `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0765). +* The sync row of `docs/todos/capability-gaps.md` no longer claims a limited incremental sync gives no gap state, and tracks + the `state_after` limit after a state fork (ADR-0764). ## Release notes diff --git a/docs/todos/capability-gaps.md b/docs/todos/capability-gaps.md index 490c80f2e..727427243 100644 --- a/docs/todos/capability-gaps.md +++ b/docs/todos/capability-gaps.md @@ -67,7 +67,7 @@ When adding a capability, record what is *missing* as explicitly as what works. | `POST /_matrix/client/v3/rooms/{roomId}/receipt/{receiptType}/{eventId}` | `spec-covered` | 0.12.30: accepts `m.read`, `m.read.private`, and `m.fully_read`; rejects unknown types with 400 `M_INVALID_PARAM`, non-members with 403 `M_FORBIDDEN`, and malformed bodies with 400 `M_BAD_JSON`. `m.read` and `m.read.private` are durable receipt rows (ADR-0169), keyed by `(room, type, user, thread_id)`, so restarts preserve read positions. `thread_id` is optional for `m.read`/`m.read.private`, must be a non-empty string naming `main` or a thread root in this room, and is carried by `/sync` `m.receipt` events and federated `m.read` EDUs. `m.read` is public and federated; `m.read.private` is visible only to its sender and is not federated. `m.fully_read` is accepted without `thread_id`, rejected if `thread_id` is supplied, and stored as the owner's `m.fully_read` room account data rather than a receipt; it never appears in `m.receipt` or a federation EDU. Invalid `thread_id` values (including a non-root room event ID) return 400 `M_INVALID_PARAM`. 0.12.60: `/sync` delivers one `m.receipt` event per thread categorisation (unthreaded, `main`, each thread root), so a user's unthreaded and threaded `m.read` for one event both arrive; sliding sync's receipts extension still holds one event per room and keeps the latest receipt on such a clash. **Gap:** the optional 400 for "The event_id is not related to the thread_id" is not checked; the root's existence and main-timeline classification are checked, but the acknowledged event's relationship to it is not. | | `POST /_matrix/client/v3/user_directory/search` | `spec-covered` | 0.12.60: searches only the users visible to the requester (a shared joined room, or joined to a `public` or `world_readable` room), as Complement expects; the spec allows searching more. Remote users are still not queried (the spec says SHOULD). | | `POST /_matrix/client/v3/delete_devices` | `spec-covered` | | -| `GET /_matrix/client/v3/sync` | `spec-covered` | Unread-notification counts are relative to the user's read receipts, partitioned by thread (an unthreaded receipt covers every earlier event, a threaded one only its thread; 0.12.30), and bound to the response snapshot. 0.12.30: the `unread_thread_notifications` filter flag (on `room.timeline`) returns per-thread counts and limits `unread_notifications` to the main timeline, and a non-state event the user sends reads its own thread up to itself (ADR-0170). The `summary` object (`m.joined_member_count`, `m.invited_member_count`, `m.heroes`) is now emitted for every joined room and covered by conformance tests. `filter_id` parameter now wired and conformance-covered; sync-only stream tokens now bound to the response snapshot for to-device delivery so room keys queued during response construction remain pending for the next sync. 0.12.59: a joined room's `state` on an initial sync (or a newly joined room) is the state up to the start of the timeline, not current state. Not done: an incremental sync with a limited timeline gives no state updates for the gap between `since` and the timeline's start. 0.12.60: `rooms.leave` lists rooms the user left **or was banned from**; each holds the filtered timeline (types, senders, `contains_url` now apply to a `/sync` timeline, and a `limit` of 0 means none) up to the leave or ban, as the user may see it, and an initial sync's `state` as of the start of that timeline (as of the leave when it is empty). An incremental sync reports a leave once, after `since`; `include_leave` lists left rooms on an initial sync only. A forgotten room is never listed (the spec says so, although Complement `TestRoomForget` expects its leave in the next incremental sync). Timeline events carry `unsigned.membership` (the requesting user's membership at the event, `leave` before they had one) and, for the device that sent them, `unsigned.transaction_id`; the same two on `GET /rooms/{roomId}/event/{eventId}`. 0.12.71: sliding sync's timeline and `required_state` events carry the same two fields, and no read path serves a stored event's `unsigned` (a received PDU is stored without it; ADR-0402). **Not done:** both fields are absent from `/messages`, `/context`, `/relations` and `/search`, and transaction-ID idempotency is still scoped to the user, not the device (the spec scopes it to a device). 0.12.60: an initial `/sync` always carries a global `m.push_rules` event with the effective ruleset (defaults plus changes), as Complement expects; an incremental one only after a change. **Lazy-loading of room members (0.12.73, ADR-0450 to ADR-0454).** With `room.state.lazy_load_members`, `state` holds the non-member state as before and only these member events: the senders of the delivered timeline (as of its start, skipped when the client has acknowledged that very event through its `since`), every member change in the gap of a limited sync, and the user's own on joining or `full_state`. Redundancy is tracked in memory per (user, device, filter, room) and lost on restart; `include_redundant_members` and `full_state=true` bypass it. The same flag on `room.timeline` does nothing, and the state filter's `types`/`senders` apply to the members. `full_state=true` is now honoured for joined rooms. Not done: sliding sync. 1.1.0: `use_state_after` answers `state_after` and omits `state` for joined and left rooms, on initial and incremental sync, with `full_state` and with `lazy_load_members` (ADR-0763 to ADR-0765); invited and knocked rooms have no such field. | +| `GET /_matrix/client/v3/sync` | `spec-covered` | Unread-notification counts are relative to the user's read receipts, partitioned by thread (an unthreaded receipt covers every earlier event, a threaded one only its thread; 0.12.30), and bound to the response snapshot. 0.12.30: the `unread_thread_notifications` filter flag (on `room.timeline`) returns per-thread counts and limits `unread_notifications` to the main timeline, and a non-state event the user sends reads its own thread up to itself (ADR-0170). The `summary` object (`m.joined_member_count`, `m.invited_member_count`, `m.heroes`) is now emitted for every joined room and covered by conformance tests. `filter_id` parameter now wired and conformance-covered; sync-only stream tokens now bound to the response snapshot for to-device delivery so room keys queued during response construction remain pending for the next sync. 0.12.59: a joined room's `state` on an initial sync (or a newly joined room) is the state up to the start of the timeline, not current state. An incremental sync with a limited timeline sends the whole state up to the start of the timeline as `state`, which covers the gap between `since` and the timeline's start (not as a delta), and `state_after` holds what changed in the gap. 0.12.60: `rooms.leave` lists rooms the user left **or was banned from**; each holds the filtered timeline (types, senders, `contains_url` now apply to a `/sync` timeline, and a `limit` of 0 means none) up to the leave or ban, as the user may see it, and an initial sync's `state` as of the start of that timeline (as of the leave when it is empty). An incremental sync reports a leave once, after `since`; `include_leave` lists left rooms on an initial sync only. A forgotten room is never listed (the spec says so, although Complement `TestRoomForget` expects its leave in the next incremental sync). Timeline events carry `unsigned.membership` (the requesting user's membership at the event, `leave` before they had one) and, for the device that sent them, `unsigned.transaction_id`; the same two on `GET /rooms/{roomId}/event/{eventId}`. 0.12.71: sliding sync's timeline and `required_state` events carry the same two fields, and no read path serves a stored event's `unsigned` (a received PDU is stored without it; ADR-0402). **Not done:** both fields are absent from `/messages`, `/context`, `/relations` and `/search`, and transaction-ID idempotency is still scoped to the user, not the device (the spec scopes it to a device). 0.12.60: an initial `/sync` always carries a global `m.push_rules` event with the effective ruleset (defaults plus changes), as Complement expects; an incremental one only after a change. **Lazy-loading of room members (0.12.73, ADR-0450 to ADR-0454).** With `room.state.lazy_load_members`, `state` holds the non-member state as before and only these member events: the senders of the delivered timeline (as of its start, skipped when the client has acknowledged that very event through its `since`), every member change in the gap of a limited sync, and the user's own on joining or `full_state`. Redundancy is tracked in memory per (user, device, filter, room) and lost on restart; `include_redundant_members` and `full_state=true` bypass it. The same flag on `room.timeline` does nothing, and the state filter's `types`/`senders` apply to the members. `full_state=true` is now honoured for joined rooms. Not done: sliding sync. 1.1.0: `use_state_after` answers `state_after` and omits `state` for joined and left rooms, on initial and incremental sync, with `full_state` and with `lazy_load_members` (ADR-0763 to ADR-0765); invited and knocked rooms have no such field. **Not done (accepted limit, ADR-0764):** after a state fork at `since`, a key that state resolution later flips back to the value recorded at `since` is not sent in `state_after`, because the delta compares against the state recorded for the event at `since`, not the state the client was told; `full_state=true` repairs the client. Storing each response's state per token would close it and was refused for its cost. | | `GET /_matrix/client/v3/keys/changes` | `spec-covered` | Requires and validates `from` and `to` sync tokens, decodes them to `sync_stream_id` boundaries, and filters `changed`/`left` device-list changes to that half-open range. Conformance coverage includes missing/invalid parameters and an upper-bound test proving `to` excludes later changes. | | Remote device-list cache | `spec-covered` | Server-Server API v1.19 "Device Management". 0.12.81 (G14, ADR-0560, ADR-0561): the list of a remote user who shares a joined room with a local user is filled from `GET /user/devices/{userId}` (its `stream_id` places later updates) and kept in memory, bounded per user and in all. `m.device_list_update` EDUs apply in `stream_id` order; a `prev_id` above the snapshot that was never applied, or an empty one, marks the list stale and the next `/keys/query` fetches it again; `m.signing_key_update` does the same. An update that changes nothing is no longer reported in `device_lists.changed`. Not persisted: a restart refills on demand. Complement `TestDeviceListsUpdateOverFederation` (three subtests) stays skip-listed until it is run against this change. | | `POST /_matrix/client/v4/sync` and `POST /_matrix/client/unstable/org.matrix.msc4186/sync` (MSC4186 Simplified Sliding Sync) | `spec-covered` | Advertised via `unstable_features["org.matrix.msc4186"] = true` and the compatibility alias `unstable_features["org.matrix.simplified_msc3575"] = true`; the stable `v4` endpoint is now also routed and accepts the same request shape. Implements named lists with windowed `ranges` or singular `range`, SYNC/INVALIDATE/INSERT/DELETE/UPDATE ops, per-room state/timeline, room subscriptions, and all five extensions (to_device, e2ee, account_data, receipts, typing). Long-poll parks in the notifier's registry and holds no sync-pool thread while it waits (ADR-0569); `pos` and `timeout` may be supplied in query parameters or in the JSON body. List + `room_subscriptions` merge: when a room is in both a list window and a `room_subscriptions` entry, `required_state` is the superset (dedup + wildcard-aware), `timeline_limit` is the max, and `include_heroes` is OR'd (previously the subscription overrode the list wholesale) — landed 0.11.9. Multi-list combination: a room in MORE THAN ONE list window now has ALL matching list windows combined (left-fold over `combine_room_configs`, associative+commutative) with each other and the `room_subscriptions` entry, not just the first matching list — landed 0.11.10. Access control (CSAZ-1): a `room_subscriptions` entry is served only for a room the caller has joined; an invited room yields `invite_state` only (no timeline or `required_state`); any other membership omits the room, and the receipts/typing extensions drop rooms the caller has not joined. `timeline_limit` is clamped to 100, and more than 256 subscriptions or 256 `required_state` pairs is a 400 `M_INVALID_PARAM` (`sync::room_access_for`, `sliding_sync_request_limit_violation`). Truncated timelines (0.12.45): a room carries `limited` when its timeline holds fewer events than it could, `prev_batch` (a `/messages` `from`) on every non-empty timeline, `num_live`, and on a grown `timeline_limit` resends its latest events with `expanded_timeline`, answering without long-polling once per request `pos` (ADR-0221). Remaining: broader MSC4186 conformance fixtures, subscription timeline pruning, and `bump_stamp`, which is never sent (Element X orders its room list by it). | From 8982bba761f552caabca02ee382e8f414299a0f1 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 18:04:09 +0000 Subject: [PATCH 14/34] Keep the capability-gaps changelog bullet within the line limit Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a078bc8e..e72bea967 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,8 +24,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) absence no longer walks every row it missed under the runtime lock. * `use_state_after`: a room joined and left inside one sync window now gets its whole readable state as of the leave in `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0765). -* The sync row of `docs/todos/capability-gaps.md` no longer claims a limited incremental sync gives no gap state, and tracks - the `state_after` limit after a state fork (ADR-0764). +* The sync row of `docs/todos/capability-gaps.md` no longer says a limited sync gives no gap state, and tracks the + `state_after` limit after a state fork (ADR-0764). ## Release notes From 5e2dda7f9deb8dfd7ffe70a65b4497dae7d85587 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 19:27:15 +0000 Subject: [PATCH 15/34] Drop the 1.1 series-start ADR and move the use_state_after ADRs above the OAuth 2.0 ones ADR-0762 said the 1.1 series starts with the MatrixRTC work, which is false on top of the OAuth 2.0 branch, and it had not landed. ADR-0763 to ADR-0765 sat below numbers that branch already uses, so they become ADR-0818 to ADR-0820 and every reference follows. The sync documentation names 1.1.1 as the version that adds use_state_after. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 10 +++--- ...t-1-1-0-with-the-matrixrtc-support-work.md | 32 ------------------- ...nce-since-as-of-the-responses-position.md} | 2 +- ...te-after-delta-compares-the-two-states.md} | 0 ...-after-stops-at-what-the-user-may-read.md} | 0 docs/adr/index.md | 7 ++-- docs/matrix-v1.19-client-server-api.md | 2 +- docs/todos/capability-gaps.md | 2 +- src/homeserver/ClientServerRuntime.cpp | 12 +++---- src/sync/AGENTS.md | 4 +-- .../test_sync_state_after_conformance.cpp | 4 +-- .../test_sync_state_after_flow.cpp | 16 +++++----- 12 files changed, 29 insertions(+), 62 deletions(-) delete mode 100644 docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md rename docs/adr/{0763-state-after-is-the-change-since-since-as-of-the-responses-position.md => 0818-state-after-is-the-change-since-since-as-of-the-responses-position.md} (97%) rename docs/adr/{0764-a-state-after-delta-compares-the-two-states.md => 0819-a-state-after-delta-compares-the-two-states.md} (100%) rename docs/adr/{0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md => 0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e72bea967..5e625848d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Features * `GET /_matrix/client/v3/sync` honours `use_state_after` (Matrix v1.16): with it true each joined and left room - carries `state_after` and no `state` (ADR-0763, ADR-0764, ADR-0765). -* The 1.1 series has its cross-release upgrade fixture: `tests/fixtures/upgrade/1.0.3/`, written by the 1.0.3 - release, opened, migrated and served on SQLite and PostgreSQL by the `[fixture]` scenarios (ADR-0216). + carries `state_after` and no `state` (ADR-0818, ADR-0819, ADR-0820). +* The cross-release upgrade fixture `tests/fixtures/upgrade/1.0.3/`, written by the 1.0.3 release, is opened, taken + through migration 002 and served on SQLite and PostgreSQL by the `[fixture]` scenarios (ADR-0216). * The `use_state_after` scenarios asserting project choices, not spec rows, moved from the conformance suite to `tests/integration/test_sync_state_after_flow.cpp`. @@ -23,9 +23,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) * `use_state_after`: the state anchor at `since` is a binary search of the room's index, so a client back after a long absence no longer walks every row it missed under the runtime lock. * `use_state_after`: a room joined and left inside one sync window now gets its whole readable state as of the leave in - `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0765). + `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0820). * The sync row of `docs/todos/capability-gaps.md` no longer says a limited sync gives no gap state, and tracks the - `state_after` limit after a state fork (ADR-0764). + `state_after` limit after a state fork (ADR-0819). ## Release notes diff --git a/docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md b/docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md deleted file mode 100644 index d10a95e43..000000000 --- a/docs/adr/0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md +++ /dev/null @@ -1,32 +0,0 @@ -# The 1.1 series starts at 1.1.0 with the MatrixRTC support work - -* Status: accepted -* Date: 2026-11-16 - -## Context and Problem Statement - -[versioning.md](../versioning.md) says `BUILD` is the working counter and that `MINOR` changes "only when the owner -decides". The branch that adds MatrixRTC support is a run of slices that add features and, later, the first schema -migration since 1.0.0. Its first slice is `use_state_after` on `GET /sync`. Which version does that work carry? - -## Considered Options - -* Continue with `BUILD` bumps from 1.0.3 (1.0.4 onward) (rejected) -* Stay on 1.0.x until the first schema migration lands (rejected) -* Start the 1.1 series at 1.1.0 with this work - -## Decision Outcome - -Chosen option: start the 1.1 series at 1.1.0. The owner decided this on 2026-11-16, and this branch, whose first slice -is `use_state_after`, carries 1.1.0. - -A 1.0.x number means a fix or a small change to the 1.0 server. This work adds new client-visible features and will add a -schema migration, which an operator upgrading from 1.0.x must be able to see in the version before reading a change log. -Bumping `BUILD` would hide that, and waiting for the migration would number the early slices as fixes and move the -series start to a commit that is not the first of the series. - -### Consequences - -* The upgrade fixture that [tests/fixtures/upgrade/README.md](../../tests/fixtures/upgrade/README.md) ties to the start of - the 1.1 series is generated from `main` commit `5f458d57` (1.0.3) in a later commit on this branch, not by this slice. -* Every location in the [versioning table](../versioning.md#every-location-that-must-be-updated) moves to 1.1.0 together. diff --git a/docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md b/docs/adr/0818-state-after-is-the-change-since-since-as-of-the-responses-position.md similarity index 97% rename from docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md rename to docs/adr/0818-state-after-is-the-change-since-since-as-of-the-responses-position.md index 610634305..fe8c5da4c 100644 --- a/docs/adr/0763-state-after-is-the-change-since-since-as-of-the-responses-position.md +++ b/docs/adr/0818-state-after-is-the-change-since-since-as-of-the-responses-position.md @@ -27,7 +27,7 @@ client takes no state from the timeline, so a `state_after` as of that message w filter the two points hold the same state, so this reading never contradicts the spec. `state` sends the whole state on a `limited` sync because a client cannot otherwise tell what the gap changed. `state_after` -has no such gap: the delta ([ADR-0764](0764-a-state-after-delta-compares-the-two-states.md)) covers `since` to the position +has no such gap: the delta ([ADR-0819](0819-a-state-after-delta-compares-the-two-states.md)) covers `since` to the position whatever the timeline's size, so whole state would only repeat what the client has, on the requests that are already largest. Whole state remains for what the spec names (no `since`, `full_state=true`) and for a room joined in the window, where the client holds nothing to apply a delta to. diff --git a/docs/adr/0764-a-state-after-delta-compares-the-two-states.md b/docs/adr/0819-a-state-after-delta-compares-the-two-states.md similarity index 100% rename from docs/adr/0764-a-state-after-delta-compares-the-two-states.md rename to docs/adr/0819-a-state-after-delta-compares-the-two-states.md diff --git a/docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md b/docs/adr/0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md similarity index 100% rename from docs/adr/0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md rename to docs/adr/0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md diff --git a/docs/adr/index.md b/docs/adr/index.md index 4fdf8578c..b971fdf0e 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -696,7 +696,6 @@ * [0815 - The authentication pages are built into the binary, without JavaScript](0815-the-authentication-pages-are-built-into-the-binary-without-javascript.md) * [0816 - `server.oauth.*` replaces `server.oidc.*`, and the old keys are refused](0816-server-oauth-replaces-server-oidc-and-the-old-keys-are-refused.md) * [0817 - A form body is read strictly and refused whole](0817-a-form-body-is-read-strictly-and-refused-whole.md) -* [0762 - The 1.1 series starts at 1.1.0 with the MatrixRTC support work](0762-the-1-1-series-starts-at-1-1-0-with-the-matrixrtc-support-work.md) -* [0763 - state_after is the change since since, as of the response's position](0763-state-after-is-the-change-since-since-as-of-the-responses-position.md) -* [0764 - A state_after delta compares the two states, and every change it finds is sent under lazy-loading](0764-a-state-after-delta-compares-the-two-states.md) -* [0765 - A left room's state_after stops at what the user may read, and always holds their own membership](0765-a-left-rooms-state-after-stops-at-what-the-user-may-read.md) +* [0818 - state_after is the change since since, as of the response's position](0818-state-after-is-the-change-since-since-as-of-the-responses-position.md) +* [0819 - A state_after delta compares the two states, and every change it finds is sent under lazy-loading](0819-a-state-after-delta-compares-the-two-states.md) +* [0820 - A left room's state_after stops at what the user may read, and always holds their own membership](0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md) diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index e91fcb081..d72f16080 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -514,7 +514,7 @@ In each case below the spec leaves the choice to the server; the owner decided t - **An identifier over 255 bytes is refused (ADR-0613).** `GET /profile/{userId}` and `GET /directory/room/{roomAlias}` answer `400 M_INVALID_PARAM` before any lookup. The spec limits a user ID and a room alias to 255 bytes. - **Unset profile fields are absent.** `GET /profile/{userId}`, the user directory and `/search` `profile_info` leave out a `displayname` or `avatar_url` that is not set, instead of sending an empty string. - **`GET /rooms/{roomId}/messages` with `lazy_load_members`.** `state` holds only the `m.room.member` events of the senders of the returned `chunk` (no other state). Redundant membership events are not tracked across calls, so a sender's membership is sent on every page (the spec's `include_redundant_members` default permits it). -- **`use_state_after` on `/sync` (1.1.0, ADR-0763 to ADR-0765).** With `use_state_after=true`, joined and left rooms answer `state_after`, present even when empty, and omit `state`; with `false` or absent, the reverse. It is the state as of the response's position: all of it with no `since`, `full_state=true` or a join in the window, otherwise what differs from the state at `since`, including what the timeline carries, what the timeline filter hid and a key state resolution moved back, also on a `limited` sync. A left room's stops at the user's leave and always holds their own membership; `full_state` does not make it whole, but a room the user joined and left inside one window (or rejoined and left again) gets its whole readable state as of the leave, as the client held none of it. Invited and knocked rooms have no `state_after` in the spec and get none. With `lazy_load_members`, every member change is sent and each timeline sender's member is as of the end of the timeline. Sliding sync is unchanged. +- **`use_state_after` on `/sync` (1.1.1, ADR-0818 to ADR-0820).** With `use_state_after=true`, joined and left rooms answer `state_after`, present even when empty, and omit `state`; with `false` or absent, the reverse. It is the state as of the response's position: all of it with no `since`, `full_state=true` or a join in the window, otherwise what differs from the state at `since`, including what the timeline carries, what the timeline filter hid and a key state resolution moved back, also on a `limited` sync. A left room's stops at the user's leave and always holds their own membership; `full_state` does not make it whole, but a room the user joined and left inside one window (or rejoined and left again) gets its whole readable state as of the leave, as the client held none of it. Invited and knocked rooms have no `state_after` in the spec and get none. With `lazy_load_members`, every member change is sent and each timeline sender's member is as of the end of the timeline. Sliding sync is unchanged. - **Lazy-loading on `/context` and `/sync` (0.12.73).** `GET /context` with `lazy_load_members` returns as `state` only the member events of the senders of `events_before`, `event` and `events_after` (stateless, redundancy allowed). `GET /sync` with `room.state.lazy_load_members` (the `timeline` flag does nothing) sends, besides the non-member state the room would carry without lazy-loading (the state filter still applies, and an incremental sync is a delta), the member events of the delivered timeline's senders, every member change in the gap of a limited sync, and the user's own membership on joining or `full_state=true`. A member the client acknowledged through its `since` is skipped (best effort, in memory, per user, device and `filter`); `include_redundant_members=true` and `full_state=true` skip nothing. `full_state=true` is now honoured for joined rooms. `use_state_after` is implemented (1.1.0, below). - **Initial `/sync` carries `m.push_rules`.** An initial `/sync` (no usable `since`) always holds a global `m.push_rules` account-data event with the user's effective ruleset (the server defaults plus their changes), if the filter's `account_data` admits it. An incremental `/sync` carries it only after a change. - **A hole in the event graph makes a joined room's `/sync` timeline `limited` (ADR-0289).** The spec defines `limited` by the filter `limit` and leaves events the server never received over federation unspecified; an event in the window whose `prev_events` are not all held (no row, or an outlier with no state group; rejected, soft-failed and backfilled events count as held, ADR-0290) starts a limited timeline at that event, with `prev_batch` for `/messages` to reach what lies before. A limited incremental sync carries the state at the start of the timeline. Sliding sync is unchanged. diff --git a/docs/todos/capability-gaps.md b/docs/todos/capability-gaps.md index 727427243..c7e16859c 100644 --- a/docs/todos/capability-gaps.md +++ b/docs/todos/capability-gaps.md @@ -67,7 +67,7 @@ When adding a capability, record what is *missing* as explicitly as what works. | `POST /_matrix/client/v3/rooms/{roomId}/receipt/{receiptType}/{eventId}` | `spec-covered` | 0.12.30: accepts `m.read`, `m.read.private`, and `m.fully_read`; rejects unknown types with 400 `M_INVALID_PARAM`, non-members with 403 `M_FORBIDDEN`, and malformed bodies with 400 `M_BAD_JSON`. `m.read` and `m.read.private` are durable receipt rows (ADR-0169), keyed by `(room, type, user, thread_id)`, so restarts preserve read positions. `thread_id` is optional for `m.read`/`m.read.private`, must be a non-empty string naming `main` or a thread root in this room, and is carried by `/sync` `m.receipt` events and federated `m.read` EDUs. `m.read` is public and federated; `m.read.private` is visible only to its sender and is not federated. `m.fully_read` is accepted without `thread_id`, rejected if `thread_id` is supplied, and stored as the owner's `m.fully_read` room account data rather than a receipt; it never appears in `m.receipt` or a federation EDU. Invalid `thread_id` values (including a non-root room event ID) return 400 `M_INVALID_PARAM`. 0.12.60: `/sync` delivers one `m.receipt` event per thread categorisation (unthreaded, `main`, each thread root), so a user's unthreaded and threaded `m.read` for one event both arrive; sliding sync's receipts extension still holds one event per room and keeps the latest receipt on such a clash. **Gap:** the optional 400 for "The event_id is not related to the thread_id" is not checked; the root's existence and main-timeline classification are checked, but the acknowledged event's relationship to it is not. | | `POST /_matrix/client/v3/user_directory/search` | `spec-covered` | 0.12.60: searches only the users visible to the requester (a shared joined room, or joined to a `public` or `world_readable` room), as Complement expects; the spec allows searching more. Remote users are still not queried (the spec says SHOULD). | | `POST /_matrix/client/v3/delete_devices` | `spec-covered` | | -| `GET /_matrix/client/v3/sync` | `spec-covered` | Unread-notification counts are relative to the user's read receipts, partitioned by thread (an unthreaded receipt covers every earlier event, a threaded one only its thread; 0.12.30), and bound to the response snapshot. 0.12.30: the `unread_thread_notifications` filter flag (on `room.timeline`) returns per-thread counts and limits `unread_notifications` to the main timeline, and a non-state event the user sends reads its own thread up to itself (ADR-0170). The `summary` object (`m.joined_member_count`, `m.invited_member_count`, `m.heroes`) is now emitted for every joined room and covered by conformance tests. `filter_id` parameter now wired and conformance-covered; sync-only stream tokens now bound to the response snapshot for to-device delivery so room keys queued during response construction remain pending for the next sync. 0.12.59: a joined room's `state` on an initial sync (or a newly joined room) is the state up to the start of the timeline, not current state. An incremental sync with a limited timeline sends the whole state up to the start of the timeline as `state`, which covers the gap between `since` and the timeline's start (not as a delta), and `state_after` holds what changed in the gap. 0.12.60: `rooms.leave` lists rooms the user left **or was banned from**; each holds the filtered timeline (types, senders, `contains_url` now apply to a `/sync` timeline, and a `limit` of 0 means none) up to the leave or ban, as the user may see it, and an initial sync's `state` as of the start of that timeline (as of the leave when it is empty). An incremental sync reports a leave once, after `since`; `include_leave` lists left rooms on an initial sync only. A forgotten room is never listed (the spec says so, although Complement `TestRoomForget` expects its leave in the next incremental sync). Timeline events carry `unsigned.membership` (the requesting user's membership at the event, `leave` before they had one) and, for the device that sent them, `unsigned.transaction_id`; the same two on `GET /rooms/{roomId}/event/{eventId}`. 0.12.71: sliding sync's timeline and `required_state` events carry the same two fields, and no read path serves a stored event's `unsigned` (a received PDU is stored without it; ADR-0402). **Not done:** both fields are absent from `/messages`, `/context`, `/relations` and `/search`, and transaction-ID idempotency is still scoped to the user, not the device (the spec scopes it to a device). 0.12.60: an initial `/sync` always carries a global `m.push_rules` event with the effective ruleset (defaults plus changes), as Complement expects; an incremental one only after a change. **Lazy-loading of room members (0.12.73, ADR-0450 to ADR-0454).** With `room.state.lazy_load_members`, `state` holds the non-member state as before and only these member events: the senders of the delivered timeline (as of its start, skipped when the client has acknowledged that very event through its `since`), every member change in the gap of a limited sync, and the user's own on joining or `full_state`. Redundancy is tracked in memory per (user, device, filter, room) and lost on restart; `include_redundant_members` and `full_state=true` bypass it. The same flag on `room.timeline` does nothing, and the state filter's `types`/`senders` apply to the members. `full_state=true` is now honoured for joined rooms. Not done: sliding sync. 1.1.0: `use_state_after` answers `state_after` and omits `state` for joined and left rooms, on initial and incremental sync, with `full_state` and with `lazy_load_members` (ADR-0763 to ADR-0765); invited and knocked rooms have no such field. **Not done (accepted limit, ADR-0764):** after a state fork at `since`, a key that state resolution later flips back to the value recorded at `since` is not sent in `state_after`, because the delta compares against the state recorded for the event at `since`, not the state the client was told; `full_state=true` repairs the client. Storing each response's state per token would close it and was refused for its cost. | +| `GET /_matrix/client/v3/sync` | `spec-covered` | Unread-notification counts are relative to the user's read receipts, partitioned by thread (an unthreaded receipt covers every earlier event, a threaded one only its thread; 0.12.30), and bound to the response snapshot. 0.12.30: the `unread_thread_notifications` filter flag (on `room.timeline`) returns per-thread counts and limits `unread_notifications` to the main timeline, and a non-state event the user sends reads its own thread up to itself (ADR-0170). The `summary` object (`m.joined_member_count`, `m.invited_member_count`, `m.heroes`) is now emitted for every joined room and covered by conformance tests. `filter_id` parameter now wired and conformance-covered; sync-only stream tokens now bound to the response snapshot for to-device delivery so room keys queued during response construction remain pending for the next sync. 0.12.59: a joined room's `state` on an initial sync (or a newly joined room) is the state up to the start of the timeline, not current state. An incremental sync with a limited timeline sends the whole state up to the start of the timeline as `state`, which covers the gap between `since` and the timeline's start (not as a delta), and `state_after` holds what changed in the gap. 0.12.60: `rooms.leave` lists rooms the user left **or was banned from**; each holds the filtered timeline (types, senders, `contains_url` now apply to a `/sync` timeline, and a `limit` of 0 means none) up to the leave or ban, as the user may see it, and an initial sync's `state` as of the start of that timeline (as of the leave when it is empty). An incremental sync reports a leave once, after `since`; `include_leave` lists left rooms on an initial sync only. A forgotten room is never listed (the spec says so, although Complement `TestRoomForget` expects its leave in the next incremental sync). Timeline events carry `unsigned.membership` (the requesting user's membership at the event, `leave` before they had one) and, for the device that sent them, `unsigned.transaction_id`; the same two on `GET /rooms/{roomId}/event/{eventId}`. 0.12.71: sliding sync's timeline and `required_state` events carry the same two fields, and no read path serves a stored event's `unsigned` (a received PDU is stored without it; ADR-0402). **Not done:** both fields are absent from `/messages`, `/context`, `/relations` and `/search`, and transaction-ID idempotency is still scoped to the user, not the device (the spec scopes it to a device). 0.12.60: an initial `/sync` always carries a global `m.push_rules` event with the effective ruleset (defaults plus changes), as Complement expects; an incremental one only after a change. **Lazy-loading of room members (0.12.73, ADR-0450 to ADR-0454).** With `room.state.lazy_load_members`, `state` holds the non-member state as before and only these member events: the senders of the delivered timeline (as of its start, skipped when the client has acknowledged that very event through its `since`), every member change in the gap of a limited sync, and the user's own on joining or `full_state`. Redundancy is tracked in memory per (user, device, filter, room) and lost on restart; `include_redundant_members` and `full_state=true` bypass it. The same flag on `room.timeline` does nothing, and the state filter's `types`/`senders` apply to the members. `full_state=true` is now honoured for joined rooms. Not done: sliding sync. 1.1.1: `use_state_after` answers `state_after` and omits `state` for joined and left rooms, on initial and incremental sync, with `full_state` and with `lazy_load_members` (ADR-0818 to ADR-0820); invited and knocked rooms have no such field. **Not done (accepted limit, ADR-0819):** after a state fork at `since`, a key that state resolution later flips back to the value recorded at `since` is not sent in `state_after`, because the delta compares against the state recorded for the event at `since`, not the state the client was told; `full_state=true` repairs the client. Storing each response's state per token would close it and was refused for its cost. | | `GET /_matrix/client/v3/keys/changes` | `spec-covered` | Requires and validates `from` and `to` sync tokens, decodes them to `sync_stream_id` boundaries, and filters `changed`/`left` device-list changes to that half-open range. Conformance coverage includes missing/invalid parameters and an upper-bound test proving `to` excludes later changes. | | Remote device-list cache | `spec-covered` | Server-Server API v1.19 "Device Management". 0.12.81 (G14, ADR-0560, ADR-0561): the list of a remote user who shares a joined room with a local user is filled from `GET /user/devices/{userId}` (its `stream_id` places later updates) and kept in memory, bounded per user and in all. `m.device_list_update` EDUs apply in `stream_id` order; a `prev_id` above the snapshot that was never applied, or an empty one, marks the list stale and the next `/keys/query` fetches it again; `m.signing_key_update` does the same. An update that changes nothing is no longer reported in `device_lists.changed`. Not persisted: a restart refills on demand. Complement `TestDeviceListsUpdateOverFederation` (three subtests) stays skip-listed until it is run against this change. | | `POST /_matrix/client/v4/sync` and `POST /_matrix/client/unstable/org.matrix.msc4186/sync` (MSC4186 Simplified Sliding Sync) | `spec-covered` | Advertised via `unstable_features["org.matrix.msc4186"] = true` and the compatibility alias `unstable_features["org.matrix.simplified_msc3575"] = true`; the stable `v4` endpoint is now also routed and accepts the same request shape. Implements named lists with windowed `ranges` or singular `range`, SYNC/INVALIDATE/INSERT/DELETE/UPDATE ops, per-room state/timeline, room subscriptions, and all five extensions (to_device, e2ee, account_data, receipts, typing). Long-poll parks in the notifier's registry and holds no sync-pool thread while it waits (ADR-0569); `pos` and `timeout` may be supplied in query parameters or in the JSON body. List + `room_subscriptions` merge: when a room is in both a list window and a `room_subscriptions` entry, `required_state` is the superset (dedup + wildcard-aware), `timeline_limit` is the max, and `include_heroes` is OR'd (previously the subscription overrode the list wholesale) — landed 0.11.9. Multi-list combination: a room in MORE THAN ONE list window now has ALL matching list windows combined (left-fold over `combine_room_configs`, associative+commutative) with each other and the `room_subscriptions` entry, not just the first matching list — landed 0.11.10. Access control (CSAZ-1): a `room_subscriptions` entry is served only for a room the caller has joined; an invited room yields `invite_state` only (no timeline or `required_state`); any other membership omits the room, and the receipts/typing extensions drop rooms the caller has not joined. `timeline_limit` is clamped to 100, and more than 256 subscriptions or 256 `required_state` pairs is a 400 `M_INVALID_PARAM` (`sync::room_access_for`, `sliding_sync_request_limit_violation`). Truncated timelines (0.12.45): a room carries `limited` when its timeline holds fewer events than it could, `prev_batch` (a `/messages` `from`) on every non-empty timeline, `num_live`, and on a grown `timeline_limit` resends its latest events with `expanded_timeline`, answering without long-polling once per request `pos` (ADR-0221). Remaining: broader MSC4186 conformance fixtures, subscription timeline pruning, and `bump_stamp`, which is never sent (Element X orders its room list by it). | diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 1ce39e7f0..c163c4641 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -5431,7 +5431,7 @@ namespace // A member event above this stream ordering is a change since `since`: in the gap when `state` is built, // and in the timeline as well for `state_after`, whose client takes no state from the timeline. std::optional gap_after{}; - // The events of a `state_after` delta (ADR-0764): each is a change the client must apply, whatever its + // The events of a `state_after` delta (ADR-0819): each is a change the client must apply, whatever its // stream ordering (state resolution can move a key back to an older event). std::unordered_set changed_event_ids{}; // Drop every state event that is not a member event (an incremental left room's state is a delta). @@ -5833,7 +5833,7 @@ namespace }); } - // ADR-0764: the IDs of the state events that make a joined room's state at `position` differ from its state at + // ADR-0819: the IDs of the state events that make a joined room's state at `position` differ from its state at // `since_ordering`, for `state_after`. A key can change with no event of the window about it (state resolution can // move it back), so this compares the two states; it does not list the window's state events. // @@ -6501,7 +6501,7 @@ namespace // What room state the user may read (ADR-0357). `none` for a user who never joined: a retracted // knock, a declined invite, a ban before any join. sync::RoomReadAccess access{}; - // GET /sync `use_state_after`: answer `state_after` instead of `state` (ADR-0765). + // GET /sync `use_state_after`: answer `state_after` instead of `state` (ADR-0820). bool use_state_after{false}; }; @@ -6532,7 +6532,7 @@ namespace }); } - // ADR-0765: a left room's `state_after`: "The state updates for the room up to the end of the timeline", which ends + // ADR-0820: a left room's `state_after`: "The state updates for the room up to the end of the timeline", which ends // at the leave. The room's state is read only as far as the user may read it (ADR-0357, room_read_access_for), so // nothing after their leave is told, however late the ban that ended their membership came. The user's own // membership is the event in `leave_event` (the leave, or the ban that followed it); the client takes no state from @@ -7079,7 +7079,7 @@ namespace since_token.has_value() && joined_membership_changed_since(store, room.room_id, user, since_ordering); auto state_events = canonicaljson::Array{}; // Spec (GET /sync, `full_state`): "all state events will be returned, even if since is non-empty". - // ADR-0763: `state_after` is the changes since `since`, on a limited sync too: they hold everything the gap + // ADR-0818: `state_after` is the changes since `since`, on a limited sync too: they hold everything the gap // changed, and whole state would only repeat what the client has. auto const state_path = !since_token.has_value() || newly_joined || full_state || (limited && !use_state_after); @@ -7111,7 +7111,7 @@ namespace auto const lazy_ref = selector.has_value() ? LazySelectorRef{std::ref(*selector)} : LazySelectorRef{}; if (use_state_after) { - // ADR-0763: as of the response's position, not the last delivered event: a state event the timeline + // ADR-0818: as of the response's position, not the last delivered event: a state event the timeline // filter hides lies before `next_batch`, and no later sync would report it. if (state_path) { diff --git a/src/sync/AGENTS.md b/src/sync/AGENTS.md index 28696a633..546cc3a66 100644 --- a/src/sync/AGENTS.md +++ b/src/sync/AGENTS.md @@ -359,7 +359,7 @@ room and, when the user is local, for them. The membership acceptor (`send_join` `device_lists.left` for remote users. A user who joins over federation is also `changed` for themselves (spec: a server may add a user to `changed` when they share a new room). -## `use_state_after` on `/sync` (ADR-0763 to ADR-0765) +## `use_state_after` on `/sync` (ADR-0818 to ADR-0820) `state_after` is as of `sync_position`, not the last delivered event, or a state event the timeline filter hides is lost. A delta is `joined_state_changes`: it compares the state at `since` with the state at the position. Do not replace it with a @@ -369,7 +369,7 @@ single forward extremity with `ForwardExtremityTable::tips_of_room`, never a wal the first row at or below its position through `EventTable::room_in_stream_order_up_to` (binary search) and steps only over withheld rows (`state_anchor_rows_examined`); never walk the rows above the position. A left room's delta is taken against `since` only when the user was joined then (`RoomReadAccess::join_ordering` at or below `since`); a join in the window gets the -whole readable state (ADR-0765). A left room goes through +whole readable state (ADR-0820). A left room goes through `build_left_room_state_after`, which reads only as far as `room_read_access_for` allows: never a joined room's delta, which would tell a departed user what came after their leave. With lazy-loading the delta's member events are always kept (`Plan::changed_event_ids`), and the senders' come by key from `member_event_at_position`. A timeline event is not state diff --git a/tests/conformance/test_sync_state_after_conformance.cpp b/tests/conformance/test_sync_state_after_conformance.cpp index b42cb196e..7b6f1538b 100644 --- a/tests/conformance/test_sync_state_after_conformance.cpp +++ b/tests/conformance/test_sync_state_after_conformance.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later // // GET /_matrix/client/v3/sync `use_state_after` and the `state_after` section of joined and left rooms -// (client-server-api.md, GET /sync, "Joined Room", "Left Room"; ADR-0763 to ADR-0765). +// (client-server-api.md, GET /sync, "Joined Room", "Left Room"; ADR-0818 to ADR-0820). #include "../support/JsonTestSupport.hpp" #include "../support/SyncStateAfterSupport.hpp" #include "merovingian/homeserver/ClientServerRuntime.hpp" @@ -503,7 +503,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", // `state_after`: "State updates for m.room.member events will be incomplete if lazy_load_members is enabled in the // /sync filter, and only return the member events required to display the senders of the timeline events in this // response." Lazy-loading room members: "servers MUST include the syncing user's own membership event ... when the full -// state of rooms is requested". Redundancy follows ADR-0450; a timeline event is not state here, so ADR-0764 forces +// state of rooms is requested". Redundancy follows ADR-0450; a timeline event is not state here, so ADR-0819 forces // every change. SCENARIO("Lazy-loading selects the members of state_after", "[conformance][client-server][sync][state-after]") { diff --git a/tests/integration/test_sync_state_after_flow.cpp b/tests/integration/test_sync_state_after_flow.cpp index 7ae3b3509..8748cb20d 100644 --- a/tests/integration/test_sync_state_after_flow.cpp +++ b/tests/integration/test_sync_state_after_flow.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later // // GET /_matrix/client/v3/sync `use_state_after`: what this project decides beyond the spec's rows, driven through the -// client-server stack (ADR-0763 to ADR-0765). What the spec states is in +// client-server stack (ADR-0818 to ADR-0820). What the spec states is in // tests/conformance/test_sync_state_after_conformance.cpp. #include "../support/JsonTestSupport.hpp" #include "../support/SyncStateAfterSupport.hpp" @@ -26,7 +26,7 @@ constexpr auto k_full_state_after = std::string_view{"&full_state=true&use_state } // namespace -// ADR-0763: a sync that finds nothing new in a joined room does not report it. +// ADR-0818: a sync that finds nothing new in a joined room does not report it. SCENARIO("A joined room with nothing new is not reported with use_state_after", "[integration][sync][state-after][flow]") { @@ -51,7 +51,7 @@ SCENARIO("A joined room with nothing new is not reported with use_state_after", } } -// ADR-0764: a delta chain is bounded (`max_state_group_delta_depth`); a window with more changes than it holds falls +// ADR-0819: a delta chain is bounded (`max_state_group_delta_depth`); a window with more changes than it holds falls // back to comparing the two states and still sends one event per changed key. SCENARIO("A state_after delta survives more state changes than one delta chain can hold", "[integration][sync][state-after][flow]") @@ -84,7 +84,7 @@ SCENARIO("A state_after delta survives more state changes than one delta chain c } } -// ADR-0764: the chain walk (`state_changes_along_chain`) answers a single-extremity room from the state groups alone. +// ADR-0819: the chain walk (`state_changes_along_chain`) answers a single-extremity room from the state groups alone. // The fallback reads the room's current state rows, which `state_rows_examined` counts, so a wide state shows which // path ran. SCENARIO("A state_after delta of an unforked room is read from the state groups, not the room's state", @@ -138,7 +138,7 @@ SCENARIO("A state_after delta of an unforked room is read from the state groups, } } -// The parser accepts only the exact string `true`, as `full_state` does (ADR-0763). +// The parser accepts only the exact string `true`, as `full_state` does (ADR-0818). SCENARIO("use_state_after is true only for the exact string true", "[integration][sync][state-after][flow]") { GIVEN("a room alice has synced") @@ -176,7 +176,7 @@ SCENARIO("use_state_after is true only for the exact string true", "[integration } } -// ADR-0763: a client that joined inside the window held nothing of the room, so it is sent the whole state, with its +// ADR-0818: a client that joined inside the window held nothing of the room, so it is sent the whole state, with its // own join, as an initial sync would. (The spec does not state it for an incremental sync.) SCENARIO("state_after of a room joined in the window is the room's whole state", "[integration][sync][state-after][flow]") @@ -210,7 +210,7 @@ SCENARIO("state_after of a room joined in the window is the room's whole state", } } -// ADR-0765: the spec's Left Room `state_after` row does not name `full_state`, so a left room's `state_after` stays a +// ADR-0820: the spec's Left Room `state_after` row does not name `full_state`, so a left room's `state_after` stays a // delta, as its `state` does (ADR-0453). SCENARIO("full_state=true leaves a left room's state_after a delta", "[integration][sync][state-after][flow]") { @@ -244,7 +244,7 @@ SCENARIO("full_state=true leaves a left room's state_after a delta", "[integrati } } -// ADR-0765: a client that was not joined at `since` held nothing of the room, so a left room's `state_after` is the +// ADR-0820: a client that was not joined at `since` held nothing of the room, so a left room's `state_after` is the // whole state as of the leave, not a delta that would hide the create event, power levels and join rules. SCENARIO("A room joined and left inside one sync window gives state_after its whole readable state", "[integration][sync][state-after][flow]") From a29d73d1f633d8d3dca81f44bbb51023c4961d3e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 19:27:15 +0000 Subject: [PATCH 16/34] Make the 1.0.3 upgrade fixture scenarios prove the v1 to v2 migration The OAuth 2.0 branch adds migration 002, so opening the 1.0.3 database applies a real step. The scenarios now assert the schema version, the new ledger row and that the OAuth tables take a row; the docs and README say the same. Co-Authored-By: Claude Sonnet 5.5 --- docs/database-persistence.md | 5 +- docs/testing-standards.md | 5 +- .../test_database_upgrade_flow.cpp | 47 +++++++++++++++++-- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/docs/database-persistence.md b/docs/database-persistence.md index cbc62a886..f404d2cf0 100644 --- a/docs/database-persistence.md +++ b/docs/database-persistence.md @@ -914,8 +914,9 @@ value. Test fixtures that do not exercise persistence select it through fixture was removed when the migrations collapsed, 0.12.47, ADR-0224). Two scenarios (`[fixture]`, one per backend) open it, run `migrate_*_schema_to`, start the runtime on it and read back the users, access tokens, rooms, events, media, account data, pusher and device keys the manifest lists, then send a new - event signed with the old release's key. 1.1.0 changes no schema, so that migration is a no-op at v1: the - scenarios prove the open-and-migrate path serves a 1.0.3 database, not a v1 to v2 step. + event signed with the old release's key. Opening the 1.0.3 database applies migration 002 (the OAuth 2.0 tables), so + the scenarios first assert the v1 to v2 step ran: the schema version is `current_schema_version()`, the ledger gained the + `oauth_tables` row, and `oauth_clients` and `oauth_sessions` exist and take a row. They then assert everything the manifest lists is still served. The PostgreSQL scenario needs `MEROVINGIAN_TEST_POSTGRESQL_URI` and reports itself skipped without it. - Both stores read the migration ledger in numeric order (`ORDER BY CAST(version AS INTEGER)`): `schema_migrations.version` is `TEXT`, and diff --git a/docs/testing-standards.md b/docs/testing-standards.md index 88f82bf94..5fd179780 100644 --- a/docs/testing-standards.md +++ b/docs/testing-standards.md @@ -96,8 +96,9 @@ checks for it, but it reads the scenario's own text, so it cannot see a helper t `tests/fixtures/upgrade//` is a database the previous series' last release wrote through its own HTTP API (ADR-0216), and `test_database_upgrade_flow.cpp` opens, migrates and serves it on SQLite and on PostgreSQL (the `[fixture]` scenarios). Those scenarios read every ID, token and key from `manifest.json`, never -name one, and never regenerate the fixture with current code. While the schema has not changed since that -release, the migration is a no-op, and the scenarios must not claim to prove a step that does not exist. The PostgreSQL +name one, and never regenerate the fixture with current code. The scenarios assert what opening the database does: +when the schema has changed since that release (1.1.0 adds migration 002), they assert the step was applied, and while it has +not, the migration is a no-op and they must not claim to prove a step that does not exist. The PostgreSQL scenario carries `[postgresql]` and skips without `MEROVINGIAN_TEST_POSTGRESQL_URI`. ### Scenarios that need a quiet store diff --git a/tests/integration/test_database_upgrade_flow.cpp b/tests/integration/test_database_upgrade_flow.cpp index 08b0206ba..c88c012d7 100644 --- a/tests/integration/test_database_upgrade_flow.cpp +++ b/tests/integration/test_database_upgrade_flow.cpp @@ -1325,22 +1325,41 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this std::filesystem::permissions(path, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); auto const master_key = write_fixture_master_key(expected.master_key_hex); auto const signing_key = sqlite_fixture_signing_key(path, expected.server_name); + // The release wrote schema v1: one ledger row, and none of version 2's tables. + auto const ledger_rows_before = sqlite_count(path, "SELECT count(*) FROM schema_migrations"); + auto const oauth_objects_before = sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE name LIKE 'oauth%'"); WHEN("the migration path runs on it and the current runtime starts on the result") { auto const migrated = merovingian::database::migrate_sqlite_schema_to( path.string(), merovingian::database::current_schema_version()); + // Version 2's tables take a row and give it back; the runtime is not started yet, so nothing else holds the file. + auto const oauth_tables_usable = + sqlite_execute(path, "INSERT INTO oauth_clients VALUES ('upgrade-probe', '{}', '0', '0')") && + sqlite_count(path, "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == 1 && + sqlite_execute(path, "DELETE FROM oauth_clients WHERE client_id = 'upgrade-probe'"); auto started = merovingian::homeserver::start_client_server( sqlite_fixture_config(path, master_key, expected.server_name)); REQUIRE(started.started); auto const view = view_fixture(started.runtime, expected); - THEN("the schema is current and everything the old release stored is served") + THEN("the v1 to v2 step ran, version 2's tables are usable, and everything the old release stored is served") { REQUIRE(migrated.ok); - // Today the current version is the one the old release wrote: a no-op, not a v1 to v2 step. + REQUIRE(expected.schema_version == 1); + REQUIRE(ledger_rows_before == 1); + REQUIRE(oauth_objects_before == 0); REQUIRE(migrated.state.version == merovingian::database::current_schema_version()); - REQUIRE(static_cast(migrated.state.version) >= expected.schema_version); + REQUIRE(static_cast(migrated.state.version) > expected.schema_version); + REQUIRE(static_cast(migrated.state.applied_migrations.size()) > ledger_rows_before); + REQUIRE(migrated.state.applied_migrations.back().name == "oauth_tables"); + REQUIRE(sqlite_count(path, "SELECT count(*) FROM schema_migrations") == + static_cast(migrated.state.applied_migrations.size())); + REQUIRE(sqlite_count(path, "SELECT count(*) FROM oauth_clients") == 0); + REQUIRE(sqlite_count(path, "SELECT count(*) FROM oauth_sessions") == 0); + REQUIRE(sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND " + "name = 'idx_oauth_sessions_client_id'") == 1); + REQUIRE(oauth_tables_usable); REQUIRE(started.runtime.homeserver.database.schema_version == merovingian::database::current_schema_version()); require_fixture_served(view, expected); @@ -1370,21 +1389,39 @@ SCENARIO("The 1.0.3 release's own PostgreSQL database is migrated and served by auto const uri_file = write_uri_file(scratch.conninfo()); auto const master_key = write_fixture_master_key(expected.master_key_hex); auto const signing_key = postgresql_fixture_signing_key(scratch.conninfo(), expected.server_name); + // The release wrote schema v1: one ledger row, and none of version 2's tables. + auto const ledger_rows_before = pg_count(scratch.conninfo(), "SELECT count(*) FROM schema_migrations"); + auto const oauth_tables_before = + pg_count(scratch.conninfo(), "SELECT count(*) FROM information_schema.tables WHERE " + "table_schema = 'public' AND table_name LIKE 'oauth%'"); WHEN("the migration path runs on it and the current runtime starts on the result") { auto const migrated = merovingian::database::migrate_postgresql_schema_to( scratch.conninfo(), merovingian::database::current_schema_version()); + // Version 2's tables take a row and give it back; the runtime is not started yet. + auto const oauth_tables_usable = + pg_execute(scratch.conninfo(), "INSERT INTO oauth_clients VALUES ('upgrade-probe', '{}', '0', '0')") && + pg_count(scratch.conninfo(), "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == 1 && + pg_execute(scratch.conninfo(), "DELETE FROM oauth_clients WHERE client_id = 'upgrade-probe'"); auto started = merovingian::homeserver::start_client_server( postgresql_fixture_config(uri_file, master_key, expected.server_name)); REQUIRE(started.started); auto const view = view_fixture(started.runtime, expected); - THEN("the schema is current and everything the old release stored is served") + THEN("the v1 to v2 step ran, version 2's tables are usable, and everything the old release stored is served") { REQUIRE(migrated.ok); + REQUIRE(expected.schema_version == 1); + REQUIRE(ledger_rows_before == 1); + REQUIRE(oauth_tables_before == 0); REQUIRE(migrated.state.version == merovingian::database::current_schema_version()); - REQUIRE(static_cast(migrated.state.version) >= expected.schema_version); + REQUIRE(static_cast(migrated.state.version) > expected.schema_version); + REQUIRE(static_cast(migrated.state.applied_migrations.size()) > ledger_rows_before); + REQUIRE(migrated.state.applied_migrations.back().name == "oauth_tables"); + REQUIRE(pg_count(scratch.conninfo(), "SELECT count(*) FROM schema_migrations") == + static_cast(migrated.state.applied_migrations.size())); + REQUIRE(oauth_tables_usable); REQUIRE(started.runtime.homeserver.database.schema_version == merovingian::database::current_schema_version()); require_fixture_served(view, expected); From ea9b9764926db90f8a80b8e9ae05e917787f3cbe Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 19:28:10 +0000 Subject: [PATCH 17/34] Format the upgrade fixture scenarios Co-Authored-By: Claude Sonnet 5.5 --- tests/integration/test_database_upgrade_flow.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_database_upgrade_flow.cpp b/tests/integration/test_database_upgrade_flow.cpp index c88c012d7..07b221afd 100644 --- a/tests/integration/test_database_upgrade_flow.cpp +++ b/tests/integration/test_database_upgrade_flow.cpp @@ -1327,13 +1327,15 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this auto const signing_key = sqlite_fixture_signing_key(path, expected.server_name); // The release wrote schema v1: one ledger row, and none of version 2's tables. auto const ledger_rows_before = sqlite_count(path, "SELECT count(*) FROM schema_migrations"); - auto const oauth_objects_before = sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE name LIKE 'oauth%'"); + auto const oauth_objects_before = + sqlite_count(path, "SELECT count(*) FROM sqlite_master WHERE name LIKE 'oauth%'"); WHEN("the migration path runs on it and the current runtime starts on the result") { auto const migrated = merovingian::database::migrate_sqlite_schema_to( path.string(), merovingian::database::current_schema_version()); - // Version 2's tables take a row and give it back; the runtime is not started yet, so nothing else holds the file. + // Version 2's tables take a row and give it back; the runtime is not started yet, so nothing else holds the + // file. auto const oauth_tables_usable = sqlite_execute(path, "INSERT INTO oauth_clients VALUES ('upgrade-probe', '{}', '0', '0')") && sqlite_count(path, "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == 1 && @@ -1343,7 +1345,8 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this REQUIRE(started.started); auto const view = view_fixture(started.runtime, expected); - THEN("the v1 to v2 step ran, version 2's tables are usable, and everything the old release stored is served") + THEN( + "the v1 to v2 step ran, version 2's tables are usable, and everything the old release stored is served") { REQUIRE(migrated.ok); REQUIRE(expected.schema_version == 1); @@ -1402,14 +1405,16 @@ SCENARIO("The 1.0.3 release's own PostgreSQL database is migrated and served by // Version 2's tables take a row and give it back; the runtime is not started yet. auto const oauth_tables_usable = pg_execute(scratch.conninfo(), "INSERT INTO oauth_clients VALUES ('upgrade-probe', '{}', '0', '0')") && - pg_count(scratch.conninfo(), "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == 1 && + pg_count(scratch.conninfo(), "SELECT count(*) FROM oauth_clients WHERE client_id = 'upgrade-probe'") == + 1 && pg_execute(scratch.conninfo(), "DELETE FROM oauth_clients WHERE client_id = 'upgrade-probe'"); auto started = merovingian::homeserver::start_client_server( postgresql_fixture_config(uri_file, master_key, expected.server_name)); REQUIRE(started.started); auto const view = view_fixture(started.runtime, expected); - THEN("the v1 to v2 step ran, version 2's tables are usable, and everything the old release stored is served") + THEN( + "the v1 to v2 step ran, version 2's tables are usable, and everything the old release stored is served") { REQUIRE(migrated.ok); REQUIRE(expected.schema_version == 1); From 433ad83f07b2aaa197a3084b80cdf5149306c187 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 20:33:37 +0000 Subject: [PATCH 18/34] Add server.turn.uris so voip/turnServer can return several TURN URIs A UDP and a TCP or TLS relay need two entries in the response's uris array. The new list key sits beside server.turn.server, which is kept so a 1.0.x config still parses; setting both is refused. Entries are validated and refused, never repaired (ADR-0821). Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + config/merovingian.conf.bsd.example | 4 +- config/merovingian.conf.example | 4 +- ...beside-server-turn-server-which-is-kept.md | 39 +++ docs/adr/index.md | 1 + docs/matrix-v1.19-client-server-api.md | 1 + docs/todos/capability-gaps.md | 2 +- docs/user-manual.md | 5 +- .../merovingian/config/TurnServerConfig.hpp | 5 + src/config/Config.cpp | 62 ++++- src/config/ConfigParser.cpp | 4 + src/config/ReloadPlan.cpp | 4 + src/homeserver/ClientServerRuntime.cpp | 16 +- .../test_turn_shared_secret_flow.cpp | 37 +++ .../integration/test_voip_discovery_flow.cpp | 164 +++++++++++++ tests/meson.build | 2 + tests/unit/test_config_turn_uris.cpp | 231 ++++++++++++++++++ 17 files changed, 574 insertions(+), 9 deletions(-) create mode 100644 docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md create mode 100644 tests/integration/test_voip_discovery_flow.cpp create mode 100644 tests/unit/test_config_turn_uris.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e625848d..485d0f013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) carries `state_after` and no `state` (ADR-0818, ADR-0819, ADR-0820). * The cross-release upgrade fixture `tests/fixtures/upgrade/1.0.3/`, written by the 1.0.3 release, is opened, taken through migration 002 and served on SQLite and PostgreSQL by the `[fixture]` scenarios (ADR-0216). +* `server.turn.uris` lists several TURN URIs for `GET /voip/turnServer`, in order; `server.turn.server` still works and + both together are refused (ADR-0821). * The `use_state_after` scenarios asserting project choices, not spec rows, moved from the conformance suite to `tests/integration/test_sync_state_after_flow.cpp`. diff --git a/config/merovingian.conf.bsd.example b/config/merovingian.conf.bsd.example index 072879541..51b478832 100644 --- a/config/merovingian.conf.bsd.example +++ b/config/merovingian.conf.bsd.example @@ -131,13 +131,15 @@ server.http.ipv6_client_prefix_length=64 # server.identity_server.total_timeout_seconds=30 # TURN server for VoIP relay (optional) -# When server.turn.server is empty, GET /_matrix/client/v3/voip/turnServer +# When server.turn.server and server.turn.uris are both empty, GET /_matrix/client/v3/voip/turnServer # returns an empty object so clients gracefully disable relay support. # Either static credentials, issued to every user (only supply credentials that # are acceptable for all users of this homeserver), or a coturn shared secret # (use-auth-secret): each user then gets time-limited credentials of their own. # Set one or the other; both is refused. # server.turn.server=turn:turn.example.org:3478?transport=udp +# Or several URIs, in the order clients should try them (instead of server.turn.server; both is refused): +# server.turn.uris=turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp # server.turn.username=merovingian # server.turn.password=set-in-owner-only-secret-file # server.turn.shared_secret_file=@SYSCONFDIR@/merovingian/turn-secret diff --git a/config/merovingian.conf.example b/config/merovingian.conf.example index 598867b3d..105d162bf 100644 --- a/config/merovingian.conf.example +++ b/config/merovingian.conf.example @@ -104,13 +104,15 @@ server.http.ipv6_client_prefix_length=64 # server.identity_server.total_timeout_seconds=30 # TURN server for VoIP relay (optional) -# When server.turn.server is empty, GET /_matrix/client/v3/voip/turnServer +# When server.turn.server and server.turn.uris are both empty, GET /_matrix/client/v3/voip/turnServer # returns an empty object so clients gracefully disable relay support. # Either static credentials, issued to every user (only supply credentials that # are acceptable for all users of this homeserver), or a coturn shared secret # (use-auth-secret): each user then gets time-limited credentials of their own. # Set one or the other; both is refused. # server.turn.server=turn:turn.example.org:3478?transport=udp +# Or several URIs, in the order clients should try them (instead of server.turn.server; both is refused): +# server.turn.uris=turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp # server.turn.username=merovingian # server.turn.password=set-in-owner-only-secret-file # server.turn.shared_secret_file=/etc/merovingian/turn-secret diff --git a/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md b/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md new file mode 100644 index 000000000..db0a4a6cc --- /dev/null +++ b/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md @@ -0,0 +1,39 @@ +# server.turn.uris is added beside server.turn.server, which is kept + +* Status: accepted +* Date: 2026-11-16 + +## Context and Problem Statement + +`GET /voip/turnServer` answers `uris`, "a list of TURN URIs" (Matrix v1.19, Voice over IP), but the configuration held +one URI, `server.turn.server`. An operator normally runs a UDP and a TCP or TLS relay, and a client behind a firewall +that blocks UDP needs the second one. How does the configuration name several URIs without breaking a 1.0.x file? + +## Decision Outcome + +Chosen option: "a new list key beside the old one", because a 1.0.x configuration must still parse in 1.1 and the old +value keeps its meaning. + +* `server.turn.uris` is a comma-separated list (as `server.identity_server.trusted_servers` is). The response's `uris` + array carries the entries in the configured order. `server.turn.server` stays, and still gives a one-element array. +* TURN is enabled when either key is set. Setting both is refused at parse time, like `shared_secret_file` with + `username` and `password`, so it is never unclear which one clients get. +* Each `server.turn.uris` entry is refused, never repaired, unless it starts with `turn:` or `turns:` followed by + something, is at most 512 bytes of printable ASCII (no whitespace or control character), is not a repeat of an + earlier entry, and the list holds at most 16 entries. `server.turn.server` has never been validated, and is not + tightened here: a value an existing configuration holds must not start to be refused by an upgrade. +* The key is reloadable and diffed by the reload plan like the other `server.turn.*` keys. + +## Pros and Cons of the Options + +### Replace server.turn.server with the list + +* Bad, because every 1.0.x configuration that sets it would stop working. + +### Make server.turn.server accept a list + +* Bad, because it changes the meaning of an existing value: a URI containing a comma would be split. + +## Links + +* Builds on [ADR-0480](0480-turn-credentials-come-from-a-coturn-shared-secret-file-through-an-openssl-hmac-sha1.md) diff --git a/docs/adr/index.md b/docs/adr/index.md index b971fdf0e..89e066e0c 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -699,3 +699,4 @@ * [0818 - state_after is the change since since, as of the response's position](0818-state-after-is-the-change-since-since-as-of-the-responses-position.md) * [0819 - A state_after delta compares the two states, and every change it finds is sent under lazy-loading](0819-a-state-after-delta-compares-the-two-states.md) * [0820 - A left room's state_after stops at what the user may read, and always holds their own membership](0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md) +* [0821 - server.turn.uris is added beside server.turn.server, which is kept](0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md) diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index d72f16080..2b42ae235 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -539,6 +539,7 @@ In each case below the spec leaves the choice to the server; the owner decided t - **`POST /users/{userId}/report` (v1.14, ADR-0482).** Needs an access token and is rate-limited. The body is a JSON object whose `reason`, when present, is a string (`400 M_BAD_JSON` otherwise). A local user ID no account holds is `404 M_NOT_FOUND` and stores nothing; a user of another server is accepted without a lookup. It is stored as an audit row `trust_safety.account.accept_report` (reporter, reported user), one per pair, listed by `GET /admin/safety/reports`; a repeat answers `200` without a second row. The reporter need share no room with the reported user. - **Password reset by email (ADR-0481).** `POST /account/password/email/requestToken` (no token) answers `200` with `sid` and `submit_url` for every well-formed request, whether or not an account holds the address, and emails an 8-character code only to an active account's address; it never answers `M_THREEPID_NOT_FOUND` or `M_THREEPID_IN_USE`. A repeat with the same or a lower `send_attempt` returns the same `sid` and sends nothing. With no relay (`server.email.smtp_url`) it is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`. The client posts `{sid, client_secret, token}` to `submit_url` (`/_matrix/client/unstable/org.merovingian/account/password/email/submitToken`, this server's own path, the Identity Service `submitToken` shape): `{"success":true}`, or `400 M_TOKEN_INCORRECT`, `M_INVALID_PARAM` or `M_SESSION_EXPIRED` (five wrong codes end the session; sessions last 15 minutes). `POST /account/password` with no access token and `auth: {type: "m.login.email.identity", threepid_creds: {sid, client_secret}}` then sets the password of the account that holds the address (`401` with the flows for a missing or unvalidated stage, `400 M_WEAK_PASSWORD` for a weak password and the session is kept for a retry) and, with `logout_devices` (the default), revokes every token of the account. `POST /account/password/msisdn/requestToken` is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED` (no SMS). +- **`GET /voip/turnServer` with `server.turn.uris` (ADR-0821)** answers one `uris` entry per configured URI, in the configured order; `server.turn.server` still gives one. - **`GET /voip/turnServer` with `server.turn.shared_secret_file` (ADR-0480)** answers per-user credentials in the coturn REST API format: `username` `:`, `password` the base64 HMAC-SHA1 of it under the secret, `ttl` `server.turn.ttl_seconds`. - **`GET /_merovingian/health` (ADR-0479)** is not a Matrix endpoint: an unauthenticated probe, `200 ok` or `503 unavailable`. diff --git a/docs/todos/capability-gaps.md b/docs/todos/capability-gaps.md index c7e16859c..8e408520c 100644 --- a/docs/todos/capability-gaps.md +++ b/docs/todos/capability-gaps.md @@ -48,7 +48,7 @@ When adding a capability, record what is *missing* as explicitly as what works. | Endpoint | Status | Needs | | --- | --- | --- | | `GET /_matrix/client/v1/auth_metadata` and the MSC2965 unstable path | `spec-covered` | The OAuth 2.0 server metadata, served while `server.oauth.enabled` is true (ADR-0773); `404 M_UNRECOGNIZED` while it is off. Registration, the authorisation pages, the token endpoint (code and refresh grants), revocation, the account pages at `/_merovingian/account` (all six account management actions), the `m.oauth` UIA stage, the `legacy_api_enabled=false` switch and the hand-off for OAuth 2.0 aware clients (`m.login.sso` with `oauth_aware_preferred`) are served (ADR-0779 to ADR-0802), and so is the device authorization grant with `server.oauth.device_grant_enabled=true` (ADR-0804 to ADR-0806). An OAuth session still cannot add an email address or mint a login token (no page; `403 M_FORBIDDEN`). Element X's QR-code login needs the MSC4108 rendezvous API, which is not in spec v1.19 and not part of 1.1.0 (owner decision), so it stays unavailable although the device authorization grant is served. Nothing here has yet been tried with Element X or in a real browser; see the open items in `docs/todos/oauth-2-0-api.md`. | -| `GET /_matrix/client/v3/voip/turnServer` | `spec-covered` | Static TURN credentials from `server.turn.*` configuration, or per-user time-limited coturn shared-secret credentials when `server.turn.shared_secret_file` is set (`src/homeserver/TurnCredentials.cpp`, ADR-0480). | +| `GET /_matrix/client/v3/voip/turnServer` | `spec-covered` | Static TURN credentials from `server.turn.*` configuration, or per-user time-limited coturn shared-secret credentials when `server.turn.shared_secret_file` is set (`src/homeserver/TurnCredentials.cpp`, ADR-0480). `server.turn.uris` advertises several URIs (ADR-0821). | | Unrouted v1.19 operations | `not-started` | These answer `404 M_UNRECOGNIZED` (most are asserted as `(implementation gap)` scenarios in `tests/conformance/test_client_server_conformance.cpp`): legacy `GET /v3/events`, `GET /v3/events/{eventId}`, `GET /v3/initialSync`; `GET /.well-known/matrix/policy_server` and `GET /.well-known/matrix/support`. `POST /v1/login/get_token` (off unless `security.login_token.enabled`; UIA password every call; single-use in-memory tokens, ADR-0548) and `GET /v3/admin/whois/{userId}` (devices only: the server records no IP, user agent or last-seen time, so `connections` is always empty) are implemented (CS8). The two `/account/password/*/requestToken` routes are routed (email: ADR-0481; msisdn answers `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`). User reporting, redaction, the appservice ping and the appservice network directory have their own rows. | | Room summary over federation | `runtime-wired` | `GET /v1/room_summary/{roomIdOrAlias}?via=` asks up to 3 `via` servers over federation `GET /hierarchy/{roomId}` (the `room` chunk; v1.19 defines no federation summary route) for a room this server holds no state for (ADR-0556): one ADR-0079 slot (429 at once without one, also without a token), one deadline, unverified answer, shown only by its own `join_rule` / `world_readable` / `allowed_room_ids`. Local state always wins; a malformed `via` is `400 M_INVALID_PARAM`. Not implemented: resolving an alias that lives on another server (404 / 401), and `membership` for a room not held locally. | | `GET /_matrix/media/v3/thumbnail/*`, `GET /_matrix/client/v1/media/thumbnail/*` | `spec-covered` | Real image resampling (PNG/JPEG, `scale`/`crop`) via the sandboxed worker landed in 0.8.10. Remote thumbnails fetch via the authenticated federation media endpoint and resample locally instead of serving the full-size original (v0.11.5). Conformance coverage now verifies that a remote thumbnail request never returns 200 with the full-size original bytes when federation infrastructure is unavailable (v0.11.5). | diff --git a/docs/user-manual.md b/docs/user-manual.md index f9cf9dcb4..390f88cfe 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -807,8 +807,9 @@ support. | Key | Default | When to change | |---|---|---| | `server.turn.server` | (empty) | TURN server URI advertised to clients, e.g. `turn:turn.example.org:3478?transport=udp`. Leave empty to keep the endpoint returning `{}`. | -| `server.turn.username` | (empty) | Static username issued to authenticated clients. Required when `server.turn.server` is set. | -| `server.turn.password` | (empty) | Static password issued alongside the username. Required when `server.turn.server` is set. | +| `server.turn.uris` | (empty) | Several TURN URIs, comma-separated, advertised in this order, e.g. `turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp`. An alternative to `server.turn.server`, which stays for existing files; setting both is refused. Each entry must start with `turn:` or `turns:`, be at most 512 bytes of printable ASCII with no whitespace, and appear once; at most 16 entries; a bad entry refuses the configuration. TURN is on when either key is set (ADR-0821). | +| `server.turn.username` | (empty) | Static username issued to authenticated clients. Required when `server.turn.server` or `server.turn.uris` is set (unless `shared_secret_file` is used). | +| `server.turn.password` | (empty) | Static password issued alongside the username. Required when `server.turn.server` or `server.turn.uris` is set (unless `shared_secret_file` is used). | | `server.turn.shared_secret_file` | (empty) | Path to an owner-read-only file (mode `0400`, as every secret file) holding the TURN server's `static-auth-secret`, for coturn run with `use-auth-secret`. Each client then gets its own credentials: username `:`, password the base64 HMAC-SHA1 of that username under the secret (the coturn REST API), valid for `ttl_seconds`. Cannot be combined with `username` and `password`: the configuration is refused if both are set. | | `server.turn.ttl_seconds` | `86400` | Credential lifetime advertised in the response, and with `shared_secret_file` the time until the username's expiry. | diff --git a/include/merovingian/config/TurnServerConfig.hpp b/include/merovingian/config/TurnServerConfig.hpp index 3e1184be8..dd22bfd60 100644 --- a/include/merovingian/config/TurnServerConfig.hpp +++ b/include/merovingian/config/TurnServerConfig.hpp @@ -4,6 +4,7 @@ #include #include +#include namespace merovingian::config { @@ -14,6 +15,10 @@ struct TurnServerConfig final // When empty the TURN endpoint returns an empty object so VoIP clients // gracefully disable relay support. std::string server{}; + // Several TURN URIs advertised to clients in the configured order, e.g. a UDP and a TLS relay (ADR-0821). An + // alternative to `server`, which stays for 1.0.x config files: setting both is refused. Each entry is a `turn:` or + // `turns:` URI, validated by validate(). TURN is enabled when either `server` or `uris` is set. + std::vector uris{}; // Static credentials issued to every authenticated client, for a TURN server that takes one shared account // or none. Mutually exclusive with shared_secret_file. std::string username{}; diff --git a/src/config/Config.cpp b/src/config/Config.cpp index 73c2ec7e1..98062d84d 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -339,6 +340,51 @@ auto is_valid_https_url(std::string_view url) noexcept -> bool return !remainder.empty() && remainder.find(' ') == std::string_view::npos; } +namespace +{ + +// The longest server.turn.uris entry and the most entries one list holds. +constexpr auto max_turn_uri_bytes = std::size_t{512U}; +constexpr auto max_turn_uris = std::size_t{16U}; + +// Why `uris` is not an acceptable server.turn.uris list, or nullopt. Fixed text, never the input. +auto turn_uris_error(std::vector const& uris) -> std::optional +{ + if (uris.size() > max_turn_uris) + { + return "at most 16 TURN URIs may be listed"; + } + for (auto index = std::size_t{0U}; index < uris.size(); ++index) + { + auto const& uri = uris[index]; + auto const rest = starts_with(uri, "turn:") ? std::string_view{uri}.substr(5U) + : starts_with(uri, "turns:") ? std::string_view{uri}.substr(6U) + : std::string_view{}; + if (rest.empty()) + { + return "each TURN URI must start with turn: or turns: and name a host"; + } + if (uri.size() > max_turn_uri_bytes) + { + return "a TURN URI may be at most 512 bytes"; + } + if (!std::ranges::all_of(uri, [](char c) { + return static_cast(c) >= 0x21U && static_cast(c) <= 0x7EU; + })) + { + return "a TURN URI must be printable ASCII without whitespace"; + } + auto const earlier = uris.begin() + static_cast(index); + if (std::ranges::find(uris.begin(), earlier, uri) != earlier) + { + return "a TURN URI may be listed only once"; + } + } + return std::nullopt; +} + +} // namespace + auto is_valid_federation_policy(std::string_view policy) noexcept -> bool { return policy == "allow" || policy == "deny"; @@ -1151,13 +1197,25 @@ auto validate(Config const& config) -> std::vector // A partially populated turn block (e.g. only a username) is rejected // because it would advertise a broken relay to clients. auto const& turn = config.server().turn; + // server.turn.uris is the list form of server.turn.server (ADR-0821). Two ways to name the relays would leave it + // unclear which one clients get, so both together is refused; each entry is refused, never repaired, when it is + // not a plain turn: or turns: URI. + if (!turn.server.empty() && !turn.uris.empty()) + { + findings.push_back({"server.turn.uris", "set either server.turn.server or server.turn.uris, not both"}); + } + if (auto const uris_error = turn_uris_error(turn.uris); uris_error.has_value()) + { + findings.push_back({"server.turn.uris", std::string{*uris_error}}); + } + auto const turn_configured = !turn.server.empty() || !turn.uris.empty(); if (!turn.shared_secret_file.empty() && (!turn.username.empty() || !turn.password.empty())) { // Two ways to issue credentials would leave it unclear which one clients get (ADR-0480). findings.push_back( {"server.turn.shared_secret_file", "set either shared_secret_file or username and password, not both"}); } - else if (!turn.server.empty() && turn.shared_secret_file.empty()) + else if (turn_configured && turn.shared_secret_file.empty()) { if (turn.username.empty()) { @@ -1168,7 +1226,7 @@ auto validate(Config const& config) -> std::vector findings.push_back({"server.turn.password", "TURN server configured but password is missing"}); } } - else if (turn.server.empty() && + else if (!turn_configured && (!turn.username.empty() || !turn.password.empty() || !turn.shared_secret_file.empty())) { findings.push_back({"server.turn.server", "TURN credentials supplied but no server URI is configured"}); diff --git a/src/config/ConfigParser.cpp b/src/config/ConfigParser.cpp index c564b78c3..6c96d492f 100644 --- a/src/config/ConfigParser.cpp +++ b/src/config/ConfigParser.cpp @@ -984,6 +984,10 @@ namespace { server.turn.server = std::string{value}; } + else if (key == "server.turn.uris") + { + server.turn.uris = parse_string_list(value); + } else if (key == "server.turn.username") { server.turn.username = std::string{value}; diff --git a/src/config/ReloadPlan.cpp b/src/config/ReloadPlan.cpp index aa97be467..5729f040f 100644 --- a/src/config/ReloadPlan.cpp +++ b/src/config/ReloadPlan.cpp @@ -560,6 +560,10 @@ auto build_reload_plan(Config const& current, Config const& next) -> ReloadPlan { add_change(plan, "server.turn.server"); } + if (current.server().turn.uris != next.server().turn.uris) + { + add_change(plan, "server.turn.uris"); + } if (current.server().turn.username != next.server().turn.username) { add_change(plan, "server.turn.username"); diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index c163c4641..8694e4198 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -19569,7 +19569,7 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp if (req.method == "GET" && request_path == "/_matrix/client/v3/voip/turnServer") { auto const& turn = rt.homeserver.config.server().turn; - if (turn.server.empty()) + if (turn.server.empty() && turn.uris.empty()) { return dispatch_resp(req, rt, 200U, "{}"); } @@ -19600,8 +19600,20 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp username = std::move(credentials->username); password = std::move(credentials->password); } + // server.turn.server (one URI, kept for 1.0.x configs) or server.turn.uris (ADR-0821), in configured order. + // The two are never both set: validate() refuses it. auto uris = canonicaljson::Array{}; - uris.push_back(json_str(turn.server)); + if (!turn.server.empty()) + { + uris.push_back(json_str(turn.server)); + } + else + { + for (auto const& uri : turn.uris) + { + uris.push_back(json_str(uri)); + } + } auto const body = json_serialize(json_obj({ json_member("username", json_str(username)), json_member("password", json_str(password)), diff --git a/tests/integration/test_turn_shared_secret_flow.cpp b/tests/integration/test_turn_shared_secret_flow.cpp index e64722b68..30cd75c24 100644 --- a/tests/integration/test_turn_shared_secret_flow.cpp +++ b/tests/integration/test_turn_shared_secret_flow.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include namespace @@ -194,3 +195,39 @@ SCENARIO("GET /voip/turnServer issues per-user time-limited credentials from a s } } } + +SCENARIO("GET /voip/turnServer lists every server.turn.uris entry in order with shared-secret credentials", + "[homeserver][turn][turn-uris][client-server]") +{ + GIVEN("a server with two TURN URIs in server.turn.uris and a shared secret file") + { + auto const secret = SecretFile{"north\n"}; + auto config = secret_turn_config(secret.path(), 3600U); + config.server().turn.server.clear(); + config.server().turn.uris = {"turn:turn.example.org:3478?transport=udp", "turns:turn.example.org:5349"}; + auto started = merovingian::homeserver::start_client_server(std::move(config)); + REQUIRE(started.started); + auto& runtime = started.runtime; + auto const alice = login(runtime, "alice"); + + WHEN("a client asks for credentials") + { + auto const response = merovingian::homeserver::handle_client_server_request( + runtime, {"GET", "/_matrix/client/v3/voip/turnServer", alice, {}}); + + THEN("the credentials are per-user and uris holds both entries in the configured order") + { + REQUIRE(response.response.status == 200U); + auto const body = parse_object(response.response.body); + auto const* username = string_member(body, "username"); + REQUIRE(username != nullptr); + REQUIRE(username->ends_with(":@alice:example.org")); + auto const* uris = object_member_as_array(body, "uris"); + REQUIRE(uris != nullptr); + REQUIRE(uris->size() == 2U); + REQUIRE(*std::get_if(&(*uris)[0U].storage()) == "turn:turn.example.org:3478?transport=udp"); + REQUIRE(*std::get_if(&(*uris)[1U].storage()) == "turns:turn.example.org:5349"); + } + } + } +} diff --git a/tests/integration/test_voip_discovery_flow.cpp b/tests/integration/test_voip_discovery_flow.cpp new file mode 100644 index 000000000..252db2b13 --- /dev/null +++ b/tests/integration/test_voip_discovery_flow.cpp @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// Voice and video call discovery at request level: several TURN URIs in GET /voip/turnServer (ADR-0821) and the +// MatrixRTC transports (MSC4143, ADR-0822). The transports routes are not in the Matrix v1.19 specification, so they +// have no conformance scenario. + +#include "../support/InMemoryDatabaseConfig.hpp" +#include "../support/JsonTestSupport.hpp" +#include "../support/MasterKey.hpp" +#include "../support/RegistrationToken.hpp" +#include "merovingian/canonicaljson/Value.hpp" +#include "merovingian/config/ClientRateLimitsConfig.hpp" +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ListenersConfig.hpp" +#include "merovingian/config/LogModulesConfig.hpp" +#include "merovingian/config/SecurityConfig.hpp" +#include "merovingian/config/ServerConfig.hpp" +#include "merovingian/homeserver/ClientServerRuntime.hpp" +#include "merovingian/homeserver/ClientServerStartResult.hpp" +#include "merovingian/homeserver/LocalHttpResponse.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace +{ + +using namespace merovingian::tests; + +[[nodiscard]] auto voip_config(merovingian::config::ServerConfig server) -> merovingian::config::Config +{ + auto security = merovingian::config::SecurityConfig{}; + security.secrets.master_key_file = merovingian::tests::shared_master_key_file(); + merovingian::tests::enable_token_registration(security); + return { + std::move(server), + merovingian::config::ListenersConfig{}, + merovingian::tests::in_memory_database_config(), + std::move(security), + merovingian::config::ClientRateLimitsConfig{}, + merovingian::config::LogModulesConfig{}, + }; +} + +[[nodiscard]] auto login(merovingian::homeserver::ClientServerRuntime& runtime) -> std::string +{ + REQUIRE(merovingian::homeserver::handle_client_server_request( + runtime, + {"POST", "/_matrix/client/v3/register", {}, merovingian::tests::registration_json("alice", "CorrectHorse7!")}) + .response.status == 200U); + auto const response = merovingian::homeserver::handle_client_server_request( + runtime, {"POST", + "/_matrix/client/v3/login", + {}, + R"({"type":"m.login.password","identifier":{"type":"m.id.user","user":"@alice:example.org"},)" + R"("password":"CorrectHorse7!","device_id":"D1"})"}); + REQUIRE(response.response.status == 200U); + auto const body = parse_object(response.response.body); + auto const* token = string_member(body, "access_token"); + REQUIRE(token != nullptr); + return *token; +} + +[[nodiscard]] auto text_at(merovingian::canonicaljson::Array const& array, std::size_t index) -> std::string +{ + auto const* text = std::get_if(&array.at(index).storage()); + return text == nullptr ? std::string{""} : *text; +} + +[[nodiscard]] auto get(merovingian::homeserver::ClientServerRuntime& runtime, std::string const& target, + std::string const& token = {}) -> merovingian::homeserver::LocalHttpResponse +{ + return merovingian::homeserver::handle_client_server_request(runtime, {"GET", target, token, {}}).response; +} + +} // namespace + +SCENARIO("GET /voip/turnServer returns every server.turn.uris entry in order with static credentials", + "[homeserver][turn][turn-uris][client-server]") +{ + GIVEN("a server with three TURN URIs and static credentials") + { + auto server = merovingian::config::ServerConfig{}; + server.turn.uris = {"turn:turn.example.org:3478?transport=udp", "turn:turn.example.org:3478?transport=tcp", + "turns:turn.example.org:5349?transport=tcp"}; + server.turn.username = "turnuser"; + server.turn.password = "turnpass"; + server.turn.ttl_seconds = 600U; + auto started = merovingian::homeserver::start_client_server(voip_config(std::move(server))); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("a client asks for TURN credentials") + { + auto const response = get(started.runtime, "/_matrix/client/v3/voip/turnServer", token); + + THEN("the uris array holds the three entries in the configured order") + { + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + REQUIRE(string_member(body, "username") != nullptr); + CHECK(*string_member(body, "username") == "turnuser"); + CHECK(*string_member(body, "password") == "turnpass"); + CHECK(*int_member(body, "ttl") == 600); + auto const* uris = object_member_as_array(body, "uris"); + REQUIRE(uris != nullptr); + REQUIRE(uris->size() == 3U); + CHECK(text_at(*uris, 0U) == "turn:turn.example.org:3478?transport=udp"); + CHECK(text_at(*uris, 1U) == "turn:turn.example.org:3478?transport=tcp"); + CHECK(text_at(*uris, 2U) == "turns:turn.example.org:5349?transport=tcp"); + } + } + } + + GIVEN("a server configured the 1.0.x way with the single server.turn.server") + { + auto server = merovingian::config::ServerConfig{}; + server.turn.server = "turn:turn.example.org:3478"; + server.turn.username = "turnuser"; + server.turn.password = "turnpass"; + auto started = merovingian::homeserver::start_client_server(voip_config(std::move(server))); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("a client asks for TURN credentials") + { + auto const response = get(started.runtime, "/_matrix/client/v3/voip/turnServer", token); + + THEN("uris holds that one entry, as before") + { + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + auto const* uris = object_member_as_array(body, "uris"); + REQUIRE(uris != nullptr); + REQUIRE(uris->size() == 1U); + CHECK(text_at(*uris, 0U) == "turn:turn.example.org:3478"); + } + } + } + + GIVEN("a server with neither TURN key") + { + auto started = merovingian::homeserver::start_client_server(voip_config({})); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("a client asks for TURN credentials") + { + auto const response = get(started.runtime, "/_matrix/client/v3/voip/turnServer", token); + + THEN("the answer is the empty object") + { + REQUIRE(response.status == 200U); + CHECK(response.body == "{}"); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index 61ad0bb16..ebab03c98 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -335,6 +335,7 @@ shell = find_program('sh') 'unit/test_config_oauth.cpp', 'unit/test_config_sso.cpp', 'unit/test_config_identity_server.cpp', + 'unit/test_config_turn_uris.cpp', 'unit/test_identity_bind_domains.cpp', 'unit/test_config_parser.cpp', 'unit/test_config_reload_plan.cpp', @@ -1007,6 +1008,7 @@ shell = find_program('sh') 'integration/test_sync_state_after_flow.cpp', 'integration/test_sync_stream_position_flow.cpp', 'integration/test_turn_shared_secret_flow.cpp', + 'integration/test_voip_discovery_flow.cpp', ), include_directories: inc, link_args: password_hash_cost_link_args, diff --git a/tests/unit/test_config_turn_uris.cpp b/tests/unit/test_config_turn_uris.cpp new file mode 100644 index 000000000..972d3b0dd --- /dev/null +++ b/tests/unit/test_config_turn_uris.cpp @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// server.turn.uris: several TURN URIs beside the older single server.turn.server (ADR-0821). +// Spec: docs/matrix-v1.19-spec/client-server-api.md, "Voice over IP": `uris` is a list of TURN URIs. + +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ConfigParseResult.hpp" +#include "merovingian/config/ConfigParser.hpp" +#include "merovingian/config/ConfigValidationFinding.hpp" +#include "merovingian/config/ReloadPlan.hpp" +#include "merovingian/config/ReloadPolicy.hpp" +#include "merovingian/config/ServerConfig.hpp" + +#include + +#include +#include +#include + +namespace +{ + +[[nodiscard]] auto has_finding(std::vector const& findings, + std::string_view field) -> bool +{ + for (auto const& finding : findings) + { + if (finding.field == field) + { + return true; + } + } + return false; +} + +[[nodiscard]] auto parse_uris(std::string const& uris) -> merovingian::config::ConfigParseResult +{ + return merovingian::config::parse_key_value_config("server.turn.uris=" + uris + + "\nserver.turn.username=u\nserver.turn.password=p\n"); +} + +} // namespace + +SCENARIO("server.turn.uris lists several TURN URIs in order", "[config][turn][turn-uris]") +{ + GIVEN("a UDP, a TCP and a TLS relay and static credentials") + { + auto const parsed = + parse_uris("turn:turn.example.org:3478?transport=udp, turn:turn.example.org:3478?transport=tcp," + "turns:turn.example.org:5349?transport=tcp"); + + WHEN("validated") + { + THEN("the entries are kept in the configured order and nothing is reported") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + REQUIRE(parsed.findings.empty()); + auto const& uris = parsed.config.server().turn.uris; + REQUIRE(uris.size() == 3U); + CHECK(uris[0U] == "turn:turn.example.org:3478?transport=udp"); + CHECK(uris[1U] == "turn:turn.example.org:3478?transport=tcp"); + CHECK(uris[2U] == "turns:turn.example.org:5349?transport=tcp"); + CHECK(parsed.config.server().turn.server.empty()); + } + } + } + + GIVEN("only the older server.turn.server key, as a 1.0.x config sets it") + { + auto const parsed = merovingian::config::parse_key_value_config( + "server.turn.server=turn:turn.example.org:3478\nserver.turn.username=u\nserver.turn.password=p\n"); + + WHEN("validated") + { + THEN("it is accepted unchanged and server.turn.uris stays empty") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + CHECK(parsed.config.server().turn.server == "turn:turn.example.org:3478"); + CHECK(parsed.config.server().turn.uris.empty()); + } + } + } + + GIVEN("server.turn.uris with a shared secret file and no username or password") + { + auto const parsed = merovingian::config::parse_key_value_config( + "server.turn.uris=turn:t.example.org\nserver.turn.shared_secret_file=/run/secrets/turn\n"); + + WHEN("validated") + { + THEN("the list alone enables TURN, so nothing is reported") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + } + } + } +} + +SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][turn][turn-uris]") +{ + GIVEN("both server.turn.server and server.turn.uris") + { + auto const parsed = merovingian::config::parse_key_value_config( + "server.turn.server=turn:a.example.org\nserver.turn.uris=turn:b.example.org\n" + "server.turn.username=u\nserver.turn.password=p\n"); + + THEN("the configuration is refused naming server.turn.uris") + { + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } + } + + GIVEN("an entry with a scheme other than turn: or turns:") + { + THEN("each is refused") + { + for (auto const* bad : {"https://t.example.org", "stun:t.example.org", "TURN:t.example.org", + "turn.example.org:3478", "turn:"}) + { + INFO(bad); + auto const parsed = parse_uris(std::string{"turn:ok.example.org,"} + bad); + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } + } + } + + GIVEN("an entry holding whitespace or a control character") + { + auto const inner_space = parse_uris("turn:a b.example.org"); + auto const tab = parse_uris("turn:a\tb.example.org"); + auto const bell = parse_uris("turn:a\x07.example.org"); + auto const high = parse_uris("turn:caf\xC3\xA9.example.org"); + + THEN("each is refused") + { + for (auto const* parsed : {&inner_space, &tab, &bell, &high}) + { + REQUIRE_FALSE(merovingian::config::is_valid(parsed->config)); + REQUIRE(has_finding(parsed->findings, "server.turn.uris")); + } + } + } + + GIVEN("an entry longer than 512 bytes") + { + auto const longest = "turn:" + std::string(507U, 'a'); + auto const too_long = "turn:" + std::string(508U, 'a'); + + THEN("512 bytes is accepted and 513 is refused") + { + REQUIRE(merovingian::config::is_valid(parse_uris(longest).config)); + auto const parsed = parse_uris(too_long); + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } + } + + GIVEN("more than 16 entries") + { + auto list = std::string{}; + auto const build = [&list](int count) { + list.clear(); + for (int i = 0; i < count; ++i) + { + list += (i == 0 ? "" : ","); + list += "turn:t" + std::to_string(i) + ".example.org"; + } + }; + + THEN("16 are accepted and 17 are refused") + { + build(16); + REQUIRE(merovingian::config::is_valid(parse_uris(list).config)); + build(17); + auto const parsed = parse_uris(list); + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } + } + + GIVEN("the same entry twice") + { + auto const parsed = parse_uris("turn:t.example.org,turns:t.example.org,turn:t.example.org"); + + THEN("the configuration is refused") + { + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } + } + + GIVEN("server.turn.uris set but credentials missing") + { + auto const parsed = merovingian::config::parse_key_value_config("server.turn.uris=turn:t.example.org\n"); + + THEN("the missing username and password are reported, as for server.turn.server") + { + REQUIRE(has_finding(parsed.findings, "server.turn.username")); + REQUIRE(has_finding(parsed.findings, "server.turn.password")); + } + } +} + +SCENARIO("server.turn.uris is reloadable and diffed like the other server.turn keys", "[config][turn][turn-uris]") +{ + GIVEN("two configurations that differ only in server.turn.uris") + { + auto const before = parse_uris("turn:a.example.org"); + auto const after = parse_uris("turn:a.example.org,turns:a.example.org"); + + WHEN("a reload is planned") + { + auto const plan = merovingian::config::build_reload_plan(before.config, after.config); + + THEN("the key is reported and needs no restart") + { + bool found = false; + for (auto const& change : plan.changes()) + { + found = found || change.key == "server.turn.uris"; + } + CHECK(found); + CHECK(merovingian::config::reload_policy_for_key("server.turn.uris") == + merovingian::config::ReloadPolicy::reloadable); + } + } + } +} From acf25cb2b6f98eac0dc2a8c663440dc941f68a6c Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 20:44:01 +0000 Subject: [PATCH 19/34] Advertise MatrixRTC LiveKit transports from server.rtc.livekit.service_url Adds the config key (strict https URL, refused when bad), GET /_matrix/client/v1/rtc/transports and its MSC4143 unstable path, the two /versions flags and the well-known rtc_foci member. The type is livekit, not m.livekit, because released clients parse only the former and m.livekit implies a homeserver token endpoint that does not exist (ADR-0822). Also runs clang-format over the Part A code. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 +- config/merovingian.conf.bsd.example | 6 + config/merovingian.conf.example | 6 + ...s-type-livekit-with-livekit-service-url.md | 51 ++++ docs/adr/index.md | 1 + docs/matrix-v1.19-client-server-api.md | 1 + docs/todos/capability-gaps.md | 1 + docs/user-manual.md | 36 +++ include/merovingian/config/RtcConfig.hpp | 21 ++ include/merovingian/config/ServerConfig.hpp | 3 + src/config/AGENTS.md | 1 + src/config/Config.cpp | 84 ++++--- src/config/ConfigParser.cpp | 5 + src/config/ReloadPlan.cpp | 6 + src/homeserver/AGENTS.md | 1 + src/homeserver/ClientServerRuntime.cpp | 57 ++++- .../test_route_query_conformance.cpp | 4 +- .../test_turn_shared_secret_flow.cpp | 5 +- .../integration/test_voip_discovery_flow.cpp | 228 +++++++++++++++++- tests/meson.build | 1 + tests/unit/test_config_rtc.cpp | 129 ++++++++++ 21 files changed, 608 insertions(+), 45 deletions(-) create mode 100644 docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md create mode 100644 include/merovingian/config/RtcConfig.hpp create mode 100644 tests/unit/test_config_rtc.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 485d0f013..796b51fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) carries `state_after` and no `state` (ADR-0818, ADR-0819, ADR-0820). * The cross-release upgrade fixture `tests/fixtures/upgrade/1.0.3/`, written by the 1.0.3 release, is opened, taken through migration 002 and served on SQLite and PostgreSQL by the `[fixture]` scenarios (ADR-0216). +* MatrixRTC transport discovery (MSC4143): `server.rtc.livekit.service_url` is advertised as a `livekit` transport + on `/rtc/transports`, `/versions` and the well-known document (ADR-0822). * `server.turn.uris` lists several TURN URIs for `GET /voip/turnServer`, in order; `server.turn.server` still works and both together are refused (ADR-0821). * The `use_state_after` scenarios asserting project choices, not spec rows, moved from the conformance suite to @@ -31,7 +33,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) ## Release notes -1.1.1 adds `use_state_after` to `/sync`. No configuration change is needed. +1.1.1 adds `use_state_after` to `/sync`, `server.turn.uris` and MatrixRTC transport discovery. The new keys +(`server.turn.uris`, `server.rtc.livekit.service_url`) are optional; no configuration change is needed. Group calls +still need your own LiveKit SFU and `lk-jwt-service`, and delayed events (MSC4140) are not implemented. # Version 1.1.0 diff --git a/config/merovingian.conf.bsd.example b/config/merovingian.conf.bsd.example index 51b478832..105c5c086 100644 --- a/config/merovingian.conf.bsd.example +++ b/config/merovingian.conf.bsd.example @@ -145,6 +145,12 @@ server.http.ipv6_client_prefix_length=64 # server.turn.shared_secret_file=@SYSCONFDIR@/merovingian/turn-secret # server.turn.ttl_seconds=86400 +# MatrixRTC group calls (optional) +# The HTTPS URL of an lk-jwt-service (Element's LiveKit authorisation service) that you run beside a LiveKit SFU. +# This server never contacts it: it only tells authenticated clients, and the /.well-known/matrix/client +# document, where it is. Empty means no group-call transport is advertised. Reloadable. +# server.rtc.livekit.service_url=https://livekit-jwt.example.org/lk-jwt-service + # Listener exposure # Keep TLS disabled only on loopback binds behind a declared reverse proxy. # If binding to a public interface, set tls=true, reverse_proxy=false, and diff --git a/config/merovingian.conf.example b/config/merovingian.conf.example index 105d162bf..082e1e84d 100644 --- a/config/merovingian.conf.example +++ b/config/merovingian.conf.example @@ -118,6 +118,12 @@ server.http.ipv6_client_prefix_length=64 # server.turn.shared_secret_file=/etc/merovingian/turn-secret # server.turn.ttl_seconds=86400 +# MatrixRTC group calls (optional) +# The HTTPS URL of an lk-jwt-service (Element's LiveKit authorisation service) that you run beside a LiveKit SFU. +# This server never contacts it: it only tells authenticated clients, and the /.well-known/matrix/client +# document, where it is. Empty means no group-call transport is advertised. Reloadable. +# server.rtc.livekit.service_url=https://livekit-jwt.example.org/lk-jwt-service + # Listener exposure # Keep TLS disabled only on loopback binds behind a declared reverse proxy. # If binding to a public interface, set tls=true, reverse_proxy=false, and diff --git a/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md b/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md new file mode 100644 index 000000000..d6aaaa28a --- /dev/null +++ b/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md @@ -0,0 +1,51 @@ +# The RTC transport is advertised as type livekit with livekit_service_url + +* Status: accepted +* Date: 2026-11-16 + +## Context and Problem Statement + +MatrixRTC (MSC4143, with the LiveKit transport of MSC4195) is not in the Matrix v1.19 specification. Clients learn where +to get a LiveKit token from the homeserver, in three places: `GET /_matrix/client/v1/rtc/transports`, the older +`GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports`, and the `org.matrix.msc4143.rtc_foci` member of +`/.well-known/matrix/client`. This server has no token service of its own; the operator runs Element's +`lk-jwt-service` beside a LiveKit SFU. What does the server advertise, and how? + +## Decision Outcome + +Chosen option: "advertise the operator's service as type `livekit`", because released Element Web, Element X and +Element Call understand only that type. + +* `server.rtc.livekit.service_url` (an HTTPS URL, refused when not one; never fetched by this server) is advertised as + `{"type":"livekit","livekit_service_url":""}`. The entry has no `url` member. +* Both routes need an access token and are rate limited like any client route. The stable path answers + `{"transports":[...]}` and, with nothing configured, `{"transports":[]}`. The unstable path answers + `{"rtc_transports":[...]}` and, with nothing configured, `{}`. Both mirror the reference server. The transport + entry is identical on both, since one call can mix clients. +* `/versions` always carries `org.matrix.msc4143` and `org.matrix.msc4143.stable` as `true`: the routes exist + whether or not a service is configured. Released clients call the unstable path only when the first is present; + newer clients call the stable path when the second is. +* `/.well-known/matrix/client` carries `org.matrix.msc4143.rtc_foci` (the same entry in a list) only when a service + URL is configured, and is otherwise byte for byte what it was. Clients built on the Rust SDK fall back to it. + +A future reader must not "fix" the type to the MSC's stable name `m.livekit` without also adding the token +endpoints. + +## Pros and Cons of the Options + +### Type `m.livekit` + +* Bad, because released clients do not parse it. +* Bad, because it tells a client to fetch tokens from the homeserver, a `/rtc/livekit/*` endpoint that does not exist. + +### A different type on each path + +* Bad, because a call can mix clients that use different paths. + +### Drop the well-known member because MSC4143 rejected well-known discovery + +* Bad, because Rust SDK based clients still fall back to it. + +## Links + +* Related: [ADR-0821](0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md) diff --git a/docs/adr/index.md b/docs/adr/index.md index 89e066e0c..346534385 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -700,3 +700,4 @@ * [0819 - A state_after delta compares the two states, and every change it finds is sent under lazy-loading](0819-a-state-after-delta-compares-the-two-states.md) * [0820 - A left room's state_after stops at what the user may read, and always holds their own membership](0820-a-left-rooms-state-after-stops-at-what-the-user-may-read.md) * [0821 - server.turn.uris is added beside server.turn.server, which is kept](0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md) +* [0822 - The RTC transport is advertised as type livekit with livekit_service_url](0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md) diff --git a/docs/matrix-v1.19-client-server-api.md b/docs/matrix-v1.19-client-server-api.md index 2b42ae235..f54740812 100644 --- a/docs/matrix-v1.19-client-server-api.md +++ b/docs/matrix-v1.19-client-server-api.md @@ -539,6 +539,7 @@ In each case below the spec leaves the choice to the server; the owner decided t - **`POST /users/{userId}/report` (v1.14, ADR-0482).** Needs an access token and is rate-limited. The body is a JSON object whose `reason`, when present, is a string (`400 M_BAD_JSON` otherwise). A local user ID no account holds is `404 M_NOT_FOUND` and stores nothing; a user of another server is accepted without a lookup. It is stored as an audit row `trust_safety.account.accept_report` (reporter, reported user), one per pair, listed by `GET /admin/safety/reports`; a repeat answers `200` without a second row. The reporter need share no room with the reported user. - **Password reset by email (ADR-0481).** `POST /account/password/email/requestToken` (no token) answers `200` with `sid` and `submit_url` for every well-formed request, whether or not an account holds the address, and emails an 8-character code only to an active account's address; it never answers `M_THREEPID_NOT_FOUND` or `M_THREEPID_IN_USE`. A repeat with the same or a lower `send_attempt` returns the same `sid` and sends nothing. With no relay (`server.email.smtp_url`) it is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`. The client posts `{sid, client_secret, token}` to `submit_url` (`/_matrix/client/unstable/org.merovingian/account/password/email/submitToken`, this server's own path, the Identity Service `submitToken` shape): `{"success":true}`, or `400 M_TOKEN_INCORRECT`, `M_INVALID_PARAM` or `M_SESSION_EXPIRED` (five wrong codes end the session; sessions last 15 minutes). `POST /account/password` with no access token and `auth: {type: "m.login.email.identity", threepid_creds: {sid, client_secret}}` then sets the password of the account that holds the address (`401` with the flows for a missing or unvalidated stage, `400 M_WEAK_PASSWORD` for a weak password and the session is kept for a retry) and, with `logout_devices` (the default), revokes every token of the account. `POST /account/password/msisdn/requestToken` is `400 M_THREEPID_MEDIUM_NOT_SUPPORTED` (no SMS). +- **MatrixRTC transport discovery (MSC4143, not in v1.19; ADR-0822).** `GET /_matrix/client/v1/rtc/transports` answers `{"transports":[{"type":"livekit","livekit_service_url":""}]}` and `GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports` answers `{"rtc_transports":[...]}`, both needing an access token; with `server.rtc.livekit.service_url` empty they answer `{"transports":[]}` and `{}`. `/versions` always lists `org.matrix.msc4143` and `org.matrix.msc4143.stable`, and the well-known document gains `org.matrix.msc4143.rtc_foci` when a URL is set. There is no `m.livekit` transport and no `/rtc/livekit/*` route. - **`GET /voip/turnServer` with `server.turn.uris` (ADR-0821)** answers one `uris` entry per configured URI, in the configured order; `server.turn.server` still gives one. - **`GET /voip/turnServer` with `server.turn.shared_secret_file` (ADR-0480)** answers per-user credentials in the coturn REST API format: `username` `:`, `password` the base64 HMAC-SHA1 of it under the secret, `ttl` `server.turn.ttl_seconds`. - **`GET /_merovingian/health` (ADR-0479)** is not a Matrix endpoint: an unauthenticated probe, `200 ok` or `503 unavailable`. diff --git a/docs/todos/capability-gaps.md b/docs/todos/capability-gaps.md index 8e408520c..3d582cb33 100644 --- a/docs/todos/capability-gaps.md +++ b/docs/todos/capability-gaps.md @@ -48,6 +48,7 @@ When adding a capability, record what is *missing* as explicitly as what works. | Endpoint | Status | Needs | | --- | --- | --- | | `GET /_matrix/client/v1/auth_metadata` and the MSC2965 unstable path | `spec-covered` | The OAuth 2.0 server metadata, served while `server.oauth.enabled` is true (ADR-0773); `404 M_UNRECOGNIZED` while it is off. Registration, the authorisation pages, the token endpoint (code and refresh grants), revocation, the account pages at `/_merovingian/account` (all six account management actions), the `m.oauth` UIA stage, the `legacy_api_enabled=false` switch and the hand-off for OAuth 2.0 aware clients (`m.login.sso` with `oauth_aware_preferred`) are served (ADR-0779 to ADR-0802), and so is the device authorization grant with `server.oauth.device_grant_enabled=true` (ADR-0804 to ADR-0806). An OAuth session still cannot add an email address or mint a login token (no page; `403 M_FORBIDDEN`). Element X's QR-code login needs the MSC4108 rendezvous API, which is not in spec v1.19 and not part of 1.1.0 (owner decision), so it stays unavailable although the device authorization grant is served. Nothing here has yet been tried with Element X or in a real browser; see the open items in `docs/todos/oauth-2-0-api.md`. | +| `GET /_matrix/client/v1/rtc/transports` and the MSC4143 unstable path | `runtime-wired` | MatrixRTC transport discovery (MSC4143, not in the v1.19 spec): advertises the operator's `lk-jwt-service` (`server.rtc.livekit.service_url`) as type `livekit` (ADR-0822). Gaps: no delayed events (MSC4140), so a vanished participant can stay listed in a call; no built-in token service (`m.livekit`, `/rtc/livekit/*`). | | `GET /_matrix/client/v3/voip/turnServer` | `spec-covered` | Static TURN credentials from `server.turn.*` configuration, or per-user time-limited coturn shared-secret credentials when `server.turn.shared_secret_file` is set (`src/homeserver/TurnCredentials.cpp`, ADR-0480). `server.turn.uris` advertises several URIs (ADR-0821). | | Unrouted v1.19 operations | `not-started` | These answer `404 M_UNRECOGNIZED` (most are asserted as `(implementation gap)` scenarios in `tests/conformance/test_client_server_conformance.cpp`): legacy `GET /v3/events`, `GET /v3/events/{eventId}`, `GET /v3/initialSync`; `GET /.well-known/matrix/policy_server` and `GET /.well-known/matrix/support`. `POST /v1/login/get_token` (off unless `security.login_token.enabled`; UIA password every call; single-use in-memory tokens, ADR-0548) and `GET /v3/admin/whois/{userId}` (devices only: the server records no IP, user agent or last-seen time, so `connections` is always empty) are implemented (CS8). The two `/account/password/*/requestToken` routes are routed (email: ADR-0481; msisdn answers `400 M_THREEPID_MEDIUM_NOT_SUPPORTED`). User reporting, redaction, the appservice ping and the appservice network directory have their own rows. | | Room summary over federation | `runtime-wired` | `GET /v1/room_summary/{roomIdOrAlias}?via=` asks up to 3 `via` servers over federation `GET /hierarchy/{roomId}` (the `room` chunk; v1.19 defines no federation summary route) for a room this server holds no state for (ADR-0556): one ADR-0079 slot (429 at once without one, also without a token), one deadline, unverified answer, shown only by its own `join_rule` / `world_readable` / `allowed_room_ids`. Local state always wins; a malformed `via` is `400 M_INVALID_PARAM`. Not implemented: resolving an alias that lives on another server (404 / 401), and `membership` for a room not held locally. | diff --git a/docs/user-manual.md b/docs/user-manual.md index 390f88cfe..5239d1151 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -823,6 +823,34 @@ support. > Set the same value as coturn's `static-auth-secret`, and keep > `server.turn.ttl_seconds` at or below coturn's own limits. +#### Voice and video calls — `server.rtc.*` and `server.turn.*` + +Merovingian does not carry call media. A call needs a service you run beside it: + +- **1:1 calls** need a TURN server such as coturn. Configure it with `server.turn.*` (above). +- **Group calls (MatrixRTC)** need a LiveKit SFU plus Element's `lk-jwt-service`, the authorisation service that + hands a caller a LiveKit token. `lk-jwt-service` checks who is calling through this server's existing OpenID + endpoints (`/_matrix/client/v3/user/{userId}/openid/request_token` and the federation `openid/userinfo`), so it must + be able to reach this server over HTTPS. Merovingian only tells clients where it is. + +| Key | Default | When to change | +|---|---|---| +| `server.rtc.livekit.service_url` | (empty) | The HTTPS URL of your `lk-jwt-service`, for example `https://livekit-jwt.example.org/lk-jwt-service`. Empty means no group-call transport is advertised. Must be an absolute `https` URL with a host, no user name or password, no `#fragment`, no whitespace and at most 2048 bytes; anything else refuses the configuration. The server never contacts this URL. | + +With the key set, an authenticated client reading `GET /_matrix/client/v1/rtc/transports` (or the older +`GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports`) learns a transport of type `livekit` with that +`livekit_service_url`, and `GET /.well-known/matrix/client` gains an `org.matrix.msc4143.rtc_foci` member with the same +entry, which clients built on the Rust SDK fall back to. `/_matrix/client/versions` always lists `org.matrix.msc4143` +and `org.matrix.msc4143.stable` (ADR-0822). The URL is shown to anyone who can read the well-known document, so give +it only an address you are happy to publish. + +Not implemented yet: + +- **Delayed events (MSC4140).** A participant whose client vanishes (a crash or lost network) is not removed from the + call's membership state automatically and can stay listed until they rejoin or someone removes them. +- **A built-in token service.** There is no `/_matrix/client/v1/rtc/livekit/*` endpoint and the transport type + `m.livekit` is not advertised; you must run `lk-jwt-service` yourself. + #### OAuth 2.0 API — `server.oauth.*` Merovingian has a built-in OAuth 2.0 API (Matrix v1.19 "OAuth 2.0 API"), which @@ -1944,6 +1972,7 @@ take effect. | `presence.*` | Restart required | | `database.pool_size` | Restart required | | `server.turn.*` | Reloadable | +| `server.rtc.*` | Reloadable | | `security.trust_safety.*` | Reloadable | | `security.access_token_lifetime_ms` / `security.refresh_token_lifetime_ms` / `security.audit_log_retention_days` | Reloadable | | `security.login_throttle.*` | Reloadable | @@ -2255,6 +2284,9 @@ server { default_type application/json; add_header Access-Control-Allow-Origin "*" always; return 200 '{"m.homeserver":{"base_url":"https://matrix.example.org"}}'; + # With server.rtc.livekit.service_url set, a document served here instead of by Merovingian must carry the + # "org.matrix.msc4143.rtc_foci" member itself: + # {"m.homeserver":{...},"org.matrix.msc4143.rtc_foci":[{"type":"livekit","livekit_service_url":"https://..."}]} } location = /.well-known/matrix/server { @@ -2345,6 +2377,10 @@ printf '{"m.server":"matrix.example.org:443"}' \ > /var/www/merovingian/.well-known/matrix/server ``` +A static `client` file is not produced by Merovingian: with `server.rtc.livekit.service_url` set, add the +`org.matrix.msc4143.rtc_foci` member to it yourself (see Voice and video calls above), or let Merovingian answer +`/.well-known/matrix/client`. + ### Apache httpd example This example assumes `mod_ssl`, `mod_headers`, `mod_proxy`, diff --git a/include/merovingian/config/RtcConfig.hpp b/include/merovingian/config/RtcConfig.hpp new file mode 100644 index 000000000..803c26c3c --- /dev/null +++ b/include/merovingian/config/RtcConfig.hpp @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include + +namespace merovingian::config +{ + +// MatrixRTC transport discovery (MSC4143, MSC4195; ADR-0822). This server never fetches or proxies anything here: it +// only tells authenticated clients where the operator's own MatrixRTC authorisation service is. +struct RtcConfig final +{ + // The HTTPS URL of an external MatrixRTC authorisation service (Element's `lk-jwt-service`) that the operator + // runs beside a LiveKit SFU. Advertised as a `livekit` transport. Empty means no transport is configured. + // validate() refuses a value that is not an absolute `https` URL with a host, no userinfo, no fragment, no + // whitespace or control character, and at most 2048 bytes. + std::string livekit_service_url{}; +}; + +} // namespace merovingian::config diff --git a/include/merovingian/config/ServerConfig.hpp b/include/merovingian/config/ServerConfig.hpp index 38f6b1176..84dc1455a 100644 --- a/include/merovingian/config/ServerConfig.hpp +++ b/include/merovingian/config/ServerConfig.hpp @@ -9,6 +9,7 @@ #include "merovingian/config/IdentityServerConfig.hpp" #include "merovingian/config/OAuthConfig.hpp" #include "merovingian/config/PushConfig.hpp" +#include "merovingian/config/RtcConfig.hpp" #include "merovingian/config/SsoConfig.hpp" #include "merovingian/config/TurnServerConfig.hpp" @@ -37,6 +38,8 @@ struct ServerConfig final // TURN relay configuration for GET /_matrix/client/v3/voip/turnServer. // Empty by default; when populated the endpoint returns real credentials. TurnServerConfig turn{}; + // MatrixRTC transport discovery (MSC4143). Empty by default: no transport is advertised (see RtcConfig). + RtcConfig rtc{}; // The built-in OAuth 2.0 API. Off by default (see OAuthConfig). OAuthConfig oauth{}; // Identity Service API client config for 3PID invites/bind/unbind/ diff --git a/src/config/AGENTS.md b/src/config/AGENTS.md index 3f6eb3f18..566a37554 100644 --- a/src/config/AGENTS.md +++ b/src/config/AGENTS.md @@ -9,6 +9,7 @@ Parses and validates the server configuration and plans (but does not yet apply) | `Config.cpp` | `Config` class (its section structs, such as `ServerConfig`, `ListenersConfig`, `DatabaseConfig` and `SecurityConfig`, each have their own `.hpp`) — typed representation of every config option; `validate_config()`, `parse_size_limit()` | | `ConfigParser.cpp` | Parses the key-value `.conf` file into `Config`; validates ranges and required fields | | `RuntimeConfigSnapshot.cpp` | `RuntimeConfigSnapshot`: holds a `Config` and can swap it for a reload-only candidate. Only the unit tests construct it today; the server does not | +| `RtcConfig.hpp`; TURN and RTC rules in `Config.cpp` | The `server.turn.uris` list rules (`turn:`/`turns:` entries, 512 bytes, 16 entries, no repeats, never both with `server.turn.server`; ADR-0821) and the `server.rtc.livekit.service_url` rule (an `oauth::ParsedUri` with scheme `https` and a host; ADR-0822). `RtcConfig.hpp` is that section's struct. | | `ReloadPlan.cpp` | Diffs old vs new config and classifies each change; used by `--plan-config-reload`. No live reload path (SIGHUP/admin socket) exists yet | | `ReloadPolicy.cpp` | `reload_policy_for_key()`: which keys need a restart. Keys are reloadable by default. Restart is required for `server.name`, the `database.*` URI and role keys, listener TLS certificate/key and `reverse_proxy` keys, `security.registration.token_file`, `database.pool_size`, `security.media.thumbnail_cache_size`, `security.media.thumbnail_max_concurrent_per_user`, `security.secrets.master_key_file`, `security.federation.key_resolution_*`, `security.federation.join_response_max_size`, `client_rate_limits.*`, `log_modules.*`, `federation.worker.*`, `server.cors.*`, `server.http.*`, `server.identity_server.*`, `server.push.*`, `server.oauth.*`, `appservice.*` and `presence.*`. The rate-limit engine and per-module log map are built once at start-up, so they are not hot-reloadable. | diff --git a/src/config/Config.cpp b/src/config/Config.cpp index 98062d84d..c986866ca 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -29,6 +29,7 @@ #include "merovingian/http/RateLimit.hpp" #include "merovingian/oauth/HandOffAllowlist.hpp" #include "merovingian/oauth/OAuthBaseUrl.hpp" +#include "merovingian/oauth/ParsedUri.hpp" #include #include @@ -343,45 +344,45 @@ auto is_valid_https_url(std::string_view url) noexcept -> bool namespace { -// The longest server.turn.uris entry and the most entries one list holds. -constexpr auto max_turn_uri_bytes = std::size_t{512U}; -constexpr auto max_turn_uris = std::size_t{16U}; + // The longest server.turn.uris entry and the most entries one list holds. + constexpr auto max_turn_uri_bytes = std::size_t{512U}; + constexpr auto max_turn_uris = std::size_t{16U}; -// Why `uris` is not an acceptable server.turn.uris list, or nullopt. Fixed text, never the input. -auto turn_uris_error(std::vector const& uris) -> std::optional -{ - if (uris.size() > max_turn_uris) - { - return "at most 16 TURN URIs may be listed"; - } - for (auto index = std::size_t{0U}; index < uris.size(); ++index) + // Why `uris` is not an acceptable server.turn.uris list, or nullopt. Fixed text, never the input. + auto turn_uris_error(std::vector const& uris) -> std::optional { - auto const& uri = uris[index]; - auto const rest = starts_with(uri, "turn:") ? std::string_view{uri}.substr(5U) - : starts_with(uri, "turns:") ? std::string_view{uri}.substr(6U) - : std::string_view{}; - if (rest.empty()) - { - return "each TURN URI must start with turn: or turns: and name a host"; - } - if (uri.size() > max_turn_uri_bytes) - { - return "a TURN URI may be at most 512 bytes"; - } - if (!std::ranges::all_of(uri, [](char c) { - return static_cast(c) >= 0x21U && static_cast(c) <= 0x7EU; - })) + if (uris.size() > max_turn_uris) { - return "a TURN URI must be printable ASCII without whitespace"; + return "at most 16 TURN URIs may be listed"; } - auto const earlier = uris.begin() + static_cast(index); - if (std::ranges::find(uris.begin(), earlier, uri) != earlier) + for (auto index = std::size_t{0U}; index < uris.size(); ++index) { - return "a TURN URI may be listed only once"; + auto const& uri = uris[index]; + auto const rest = starts_with(uri, "turn:") ? std::string_view{uri}.substr(5U) + : starts_with(uri, "turns:") ? std::string_view{uri}.substr(6U) + : std::string_view{}; + if (rest.empty()) + { + return "each TURN URI must start with turn: or turns: and name a host"; + } + if (uri.size() > max_turn_uri_bytes) + { + return "a TURN URI may be at most 512 bytes"; + } + if (!std::ranges::all_of(uri, [](char c) { + return static_cast(c) >= 0x21U && static_cast(c) <= 0x7EU; + })) + { + return "a TURN URI must be printable ASCII without whitespace"; + } + auto const earlier = uris.begin() + static_cast(index); + if (std::ranges::find(uris.begin(), earlier, uri) != earlier) + { + return "a TURN URI may be listed only once"; + } } + return std::nullopt; } - return std::nullopt; -} } // namespace @@ -1226,12 +1227,27 @@ auto validate(Config const& config) -> std::vector findings.push_back({"server.turn.password", "TURN server configured but password is missing"}); } } - else if (!turn_configured && - (!turn.username.empty() || !turn.password.empty() || !turn.shared_secret_file.empty())) + else if (!turn_configured && (!turn.username.empty() || !turn.password.empty() || !turn.shared_secret_file.empty())) { findings.push_back({"server.turn.server", "TURN credentials supplied but no server URI is configured"}); } + // MatrixRTC transport discovery (ADR-0822): the service URL is sent to every authenticated client and, in the + // well-known document, to anyone, so it is refused unless it is an absolute https URL with a host. oauth::ParsedUri + // is the project's strict reader: printable ASCII only, no userinfo, no fragment, at most 2048 bytes. Nothing is + // repaired, and this server never fetches the URL. + if (auto const& service_url = config.server().rtc.livekit_service_url; !service_url.empty()) + { + auto const uri = oauth::ParsedUri::read(service_url); + auto const* parsed = std::get_if(&uri); + if (parsed == nullptr || parsed->scheme() != "https" || !parsed->has_authority() || parsed->host().empty()) + { + findings.push_back({"server.rtc.livekit.service_url", + "must be an https URL with a host, no userinfo, no fragment, no whitespace and at most " + "2048 bytes"}); + } + } + // The built-in OAuth 2.0 API (ADR-0812, ADR-0816). Every OAuth address is derived from public_baseurl, so with the // API on it must be a base URL the OAuth reader accepts (ADR-0773). The other server.oauth.* keys mean nothing // with the API off, and are refused rather than silently ignored. diff --git a/src/config/ConfigParser.cpp b/src/config/ConfigParser.cpp index 6c96d492f..0aace1949 100644 --- a/src/config/ConfigParser.cpp +++ b/src/config/ConfigParser.cpp @@ -988,6 +988,11 @@ namespace { server.turn.uris = parse_string_list(value); } + // MatrixRTC transport discovery (ADR-0822). validate() judges the URL. + else if (key == "server.rtc.livekit.service_url") + { + server.rtc.livekit_service_url = std::string{value}; + } else if (key == "server.turn.username") { server.turn.username = std::string{value}; diff --git a/src/config/ReloadPlan.cpp b/src/config/ReloadPlan.cpp index 5729f040f..b15152cb4 100644 --- a/src/config/ReloadPlan.cpp +++ b/src/config/ReloadPlan.cpp @@ -581,6 +581,12 @@ auto build_reload_plan(Config const& current, Config const& next) -> ReloadPlan add_change(plan, "server.turn.ttl_seconds"); } + // MatrixRTC transport discovery. + if (current.server().rtc.livekit_service_url != next.server().rtc.livekit_service_url) + { + add_change(plan, "server.rtc.livekit.service_url"); + } + // The OAuth 2.0 API: every field is diffed, and every change is restart_required (see ReloadPolicy.cpp). if (current.server().oauth.enabled != next.server().oauth.enabled) { diff --git a/src/homeserver/AGENTS.md b/src/homeserver/AGENTS.md index 2080e8791..ca39f3565 100644 --- a/src/homeserver/AGENTS.md +++ b/src/homeserver/AGENTS.md @@ -71,6 +71,7 @@ This is the largest and most complex module — read this file carefully before | `PresenceService.cpp` | Presence (CS API "Presence", SS API `m.presence`): `read_presence` (GET), `apply_sync_presence` (the `/sync` `set_presence` parameter), `build_sync_presence_events` (`presence.events`), `accept_inbound_presence` (the EDU sink's presence case) and `record_presence`, the one place a presence row is stored. `expire_idle_presence` (the idle sweep, ADR-0492). `presence.enabled=false` turns every entry point into a no-op. Sending the EDU stays in `ClientServerRuntime.cpp` (`broadcast_presence_edu`), beside the other outbound EDUs | | `PushRetry.cpp` | `attempt_push_delivery` (every push attempt: circuit breaker, rejected pushkeys, scheduling retries) and `PushRetryJob`, the one thread that sends retries (R14, ADR-0206); a run is bounded by a time budget, takes retries one recipient at a time in turn and sends with a shorter timeout (ADR-0410), and a slow answer counts against the gateway's circuit (ADR-0409) | | `PasswordReset.cpp` | Password reset by email (ADR-0481): `make_password_reset_token` (8 characters, 40 bits), `normalize_password_reset_token`, `send_password_reset_email` (a background task parked in `orphan_futures_`, at most 8 in flight, no lock held while it sends). The endpoints are in `ClientServerRuntime.cpp` before the auth gate; `reset_local_user_password` is in `AuthService.cpp` | +| `ClientServerRuntime.cpp` (MatrixRTC) | `GET /_matrix/client/v1/rtc/transports` and `GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports` advertise `server.rtc.livekit.service_url` as `{"type":"livekit","livekit_service_url":...}` (`livekit_transport_json`): `transports` and `[]` when none on the stable path, `rtc_transports` and `{}` on the unstable one; both flags `org.matrix.msc4143` and `org.matrix.msc4143.stable` are always in `/versions`, and the well-known document gains `org.matrix.msc4143.rtc_foci` only with a URL (ADR-0822). The type is never `m.livekit` and there is no `/rtc/livekit/*` route. `voip/turnServer` returns `server.turn.server` or, when that is empty, each `server.turn.uris` entry in order (ADR-0821) | | `TurnCredentials.cpp` | `make_turn_credentials`: the coturn REST API username and password for `GET /voip/turnServer` (ADR-0480) | | `EmailPush.cpp` | Kind `"email"` pushers (ADR-0219, ADR-0220): `owned_email_pusher_address` (the pushkey must be the account's own email 3PID), `remove_email_pushers_for_address`, and `EmailPushJob`, the one thread that sends due digests through `send_due_email_notifications`. It re-reads the pusher, the 3PID, the account and the badge count under `runtime.mutex` (one hold, one `PushBadgeBudget`: rows and glob steps bounded together, ADR-0722), then sends with the mutex released | | `RuntimeSigningKeyStore.cpp` | Production `SigningKeyStore` over the persisted server signing-key rows | diff --git a/src/homeserver/ClientServerRuntime.cpp b/src/homeserver/ClientServerRuntime.cpp index 8694e4198..9345930f6 100644 --- a/src/homeserver/ClientServerRuntime.cpp +++ b/src/homeserver/ClientServerRuntime.cpp @@ -856,6 +856,18 @@ namespace return canonicaljson::make_member(std::move(key), std::move(value)); } + // The MatrixRTC transport entry for the LiveKit authorisation service at `service_url` (MSC4143, MSC4195; + // ADR-0822). The type is `livekit`, never `m.livekit`, and there is no `url` member. + [[nodiscard]] auto livekit_transport_json(std::string const& service_url) -> canonicaljson::Value + { + return canonicaljson::Value{ + canonicaljson::Object{ + canonicaljson::make_member("type", canonicaljson::Value{std::string{"livekit"}}), + canonicaljson::make_member("livekit_service_url", canonicaljson::Value{service_url}), + } + }; + } + [[nodiscard]] auto json_serialize(canonicaljson::Value const& value) -> std::string { auto const result = canonicaljson::serialize_canonical(value); @@ -4104,6 +4116,8 @@ namespace "/_matrix/client/v1/media/upload", "/_matrix/client/v1/media/preview_url", "/_matrix/client/v3/voip/turnServer", + "/_matrix/client/v1/rtc/transports", + "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports", "/_matrix/client/v3/room_keys/version", "/_matrix/client/v3/room_keys/keys", "/_matrix/client/v3/devices", @@ -4207,6 +4221,8 @@ namespace {"/_matrix/client/v1/media/upload", "POST" }, {"/_matrix/client/v1/media/preview_url", "GET" }, {"/_matrix/client/v3/voip/turnServer", "GET" }, + {"/_matrix/client/v1/rtc/transports", "GET" }, + {"/_matrix/client/unstable/org.matrix.msc4143/rtc/transports", "GET" }, {"/_matrix/client/v3/room_keys/version", "GET, POST" }, {"/_matrix/client/v3/room_keys/keys", "DELETE, GET, PUT"}, {"/_matrix/client/v3/devices", "GET" }, @@ -15318,10 +15334,18 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp if (req.method == "GET" && request_path == "/.well-known/matrix/client") { auto const& base_url = rt.homeserver.config.server().public_baseurl; - return dispatch_resp(req, rt, 200U, - json_serialize(json_obj({ - json_member("m.homeserver", json_obj({json_member("base_url", json_str(base_url))})), - }))); + auto document = canonicaljson::Object{ + json_member("m.homeserver", json_obj({json_member("base_url", json_str(base_url))})), + }; + // MSC4143: the Rust SDK falls back to this member when it cannot read the transports route (ADR-0822). Only + // with a service configured; without one the body is what it always was. + if (auto const& service_url = rt.homeserver.config.server().rtc.livekit_service_url; !service_url.empty()) + { + auto foci = canonicaljson::Array{}; + foci.push_back(livekit_transport_json(service_url)); + document.push_back(json_member("org.matrix.msc4143.rtc_foci", json_arr(std::move(foci)))); + } + return dispatch_resp(req, rt, 200U, json_serialize(json_obj(std::move(document)))); } // GET /_matrix/client/versions is the unauthenticated discovery endpoint @@ -15341,6 +15365,8 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp json_member("versions", json_arr(std::move(versions))), json_member("unstable_features", json_obj({ json_member("org.matrix.msc4186", json_bool(true)), + json_member("org.matrix.msc4143", json_bool(true)), + json_member("org.matrix.msc4143.stable", json_bool(true)), json_member("org.matrix.simplified_msc3575", json_bool(true)), })), }))); @@ -19623,6 +19649,29 @@ static auto handle_client_server_request_impl(ClientServerRuntime& rt, LocalHttp return dispatch_resp(req, rt, 200U, body); } + // GET /_matrix/client/v1/rtc/transports and its unstable MSC4143 path: where the operator's MatrixRTC + // authorisation service is (ADR-0822). The two differ in the property name and in what they answer when no + // service is configured, as the reference server's do. + if (req.method == "GET" && (request_path == "/_matrix/client/v1/rtc/transports" || + request_path == "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports")) + { + auto const& service_url = rt.homeserver.config.server().rtc.livekit_service_url; + auto const stable = request_path == "/_matrix/client/v1/rtc/transports"; + auto transports = canonicaljson::Array{}; + if (!service_url.empty()) + { + transports.push_back(livekit_transport_json(service_url)); + } + if (!stable && transports.empty()) + { + return dispatch_resp(req, rt, 200U, "{}"); + } + return dispatch_resp(req, rt, 200U, + json_serialize(json_obj({ + json_member(stable ? "transports" : "rtc_transports", json_arr(std::move(transports))), + }))); + } + auto const key_route = auth::match_key_api_route(req.method, req.target); if (key_route.matched) { diff --git a/tests/conformance/test_route_query_conformance.cpp b/tests/conformance/test_route_query_conformance.cpp index b247f599a..a8f589c65 100644 --- a/tests/conformance/test_route_query_conformance.cpp +++ b/tests/conformance/test_route_query_conformance.cpp @@ -111,7 +111,7 @@ SCENARIO("every client route that matched its whole target answers a request wit // Every route that was compared with `req.target == "..."`. A handler may answer anything // (a UIA challenge, a 400 for the empty body); what it may not do is answer as though the // route does not exist. /logout is last: it ends the session. - constexpr auto routes = std::array{ + constexpr auto routes = std::array{ { {"GET", "/.well-known/matrix/client", ""}, {"GET", "/_matrix/client/versions", ""}, @@ -150,6 +150,8 @@ SCENARIO("every client route that matched its whole target answers a request wit {"GET", "/_matrix/media/v3/config", ""}, {"GET", "/_matrix/client/v1/media/config", ""}, {"GET", "/_matrix/client/v3/voip/turnServer", ""}, + {"GET", "/_matrix/client/v1/rtc/transports", ""}, + {"GET", "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports", ""}, {"GET", "/_matrix/client/v3/devices", ""}, {"POST", "/_matrix/client/v3/delete_devices", "{}"}, {"GET", "/_matrix/client/v3/joined_rooms", ""}, diff --git a/tests/integration/test_turn_shared_secret_flow.cpp b/tests/integration/test_turn_shared_secret_flow.cpp index 30cd75c24..21296dd28 100644 --- a/tests/integration/test_turn_shared_secret_flow.cpp +++ b/tests/integration/test_turn_shared_secret_flow.cpp @@ -11,12 +11,12 @@ #include "../support/TempDirectory.hpp" #include "merovingian/config/ClientRateLimitsConfig.hpp" #include "merovingian/config/Config.hpp" +#include "merovingian/config/ConfigParser.hpp" #include "merovingian/config/ListenersConfig.hpp" #include "merovingian/config/LogModulesConfig.hpp" #include "merovingian/config/ReloadPlan.hpp" #include "merovingian/config/SecurityConfig.hpp" #include "merovingian/config/ServerConfig.hpp" -#include "merovingian/config/ConfigParser.hpp" #include "merovingian/crypto/HmacSha1.hpp" #include "merovingian/homeserver/ClientServerClock.hpp" #include "merovingian/homeserver/ClientServerRuntime.hpp" @@ -225,7 +225,8 @@ SCENARIO("GET /voip/turnServer lists every server.turn.uris entry in order with auto const* uris = object_member_as_array(body, "uris"); REQUIRE(uris != nullptr); REQUIRE(uris->size() == 2U); - REQUIRE(*std::get_if(&(*uris)[0U].storage()) == "turn:turn.example.org:3478?transport=udp"); + REQUIRE(*std::get_if(&(*uris)[0U].storage()) == + "turn:turn.example.org:3478?transport=udp"); REQUIRE(*std::get_if(&(*uris)[1U].storage()) == "turns:turn.example.org:5349"); } } diff --git a/tests/integration/test_voip_discovery_flow.cpp b/tests/integration/test_voip_discovery_flow.cpp index 252db2b13..f73310245 100644 --- a/tests/integration/test_voip_discovery_flow.cpp +++ b/tests/integration/test_voip_discovery_flow.cpp @@ -24,8 +24,9 @@ #include #include -#include +#include #include +#include #include namespace @@ -51,8 +52,10 @@ using namespace merovingian::tests; [[nodiscard]] auto login(merovingian::homeserver::ClientServerRuntime& runtime) -> std::string { REQUIRE(merovingian::homeserver::handle_client_server_request( - runtime, - {"POST", "/_matrix/client/v3/register", {}, merovingian::tests::registration_json("alice", "CorrectHorse7!")}) + runtime, {"POST", + "/_matrix/client/v3/register", + {}, + merovingian::tests::registration_json("alice", "CorrectHorse7!")}) .response.status == 200U); auto const response = merovingian::homeserver::handle_client_server_request( runtime, {"POST", @@ -162,3 +165,222 @@ SCENARIO("GET /voip/turnServer returns every server.turn.uris entry in order wit } } } + +namespace +{ + +constexpr auto livekit_url = "https://livekit-jwt.example.org/lk-jwt-service"; + +[[nodiscard]] auto rtc_server(std::string url) -> merovingian::config::ServerConfig +{ + auto server = merovingian::config::ServerConfig{}; + server.rtc.livekit_service_url = std::move(url); + return server; +} + +} // namespace + +SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", "[homeserver][rtc][client-server]") +{ + GIVEN("a server with a LiveKit service URL") + { + auto started = merovingian::homeserver::start_client_server(voip_config(rtc_server(livekit_url))); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("a client reads the stable transports route") + { + auto const response = get(started.runtime, "/_matrix/client/v1/rtc/transports", token); + + THEN("it gets a livekit transport under `transports`") + { + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + auto const* transports = object_member_as_array(body, "transports"); + REQUIRE(transports != nullptr); + REQUIRE(transports->size() == 1U); + CHECK( + response.body == + R"({"transports":[{"livekit_service_url":"https://livekit-jwt.example.org/lk-jwt-service","type":"livekit"}]})"); + auto const* entry = std::get_if(&(*transports)[0U].storage()); + REQUIRE(entry != nullptr); + CHECK(entry->size() == 2U); + CHECK(*string_member(*entry, "type") == "livekit"); + CHECK(*string_member(*entry, "livekit_service_url") == livekit_url); + } + } + + WHEN("a client reads the unstable transports route, with a query string") + { + auto const response = + get(started.runtime, "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports?unused=1", token); + + THEN("it gets the same entry under `rtc_transports`") + { + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + CHECK( + response.body == + R"({"rtc_transports":[{"livekit_service_url":"https://livekit-jwt.example.org/lk-jwt-service","type":"livekit"}]})"); + auto const* transports = object_member_as_array(body, "rtc_transports"); + REQUIRE(transports != nullptr); + REQUIRE(transports->size() == 1U); + auto const* entry = std::get_if(&(*transports)[0U].storage()); + REQUIRE(entry != nullptr); + CHECK(entry->size() == 2U); + CHECK(*string_member(*entry, "type") == "livekit"); + CHECK(*string_member(*entry, "livekit_service_url") == livekit_url); + } + } + + WHEN("a client reads the stable route with a query string") + { + auto const response = get(started.runtime, "/_matrix/client/v1/rtc/transports?unused=1", token); + + THEN("it still routes") + { + CHECK(response.status == 200U); + } + } + } + + GIVEN("a server with no LiveKit service URL") + { + auto started = merovingian::homeserver::start_client_server(voip_config({})); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("a client reads each route") + { + auto const stable = get(started.runtime, "/_matrix/client/v1/rtc/transports", token); + auto const unstable = + get(started.runtime, "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports", token); + + THEN("the stable path answers an empty list and the unstable path an empty object") + { + REQUIRE(stable.status == 200U); + CHECK(stable.body == R"({"transports":[]})"); + REQUIRE(unstable.status == 200U); + CHECK(unstable.body == "{}"); + } + } + } +} + +SCENARIO("MatrixRTC transports need an access token", "[homeserver][rtc][client-server]") +{ + GIVEN("a server with a LiveKit service URL") + { + auto started = merovingian::homeserver::start_client_server(voip_config(rtc_server(livekit_url))); + REQUIRE(started.started); + std::ignore = login(started.runtime); + + WHEN("each route is read without a token, and with an unknown one") + { + THEN("each answers 401 and names no URL") + { + for (auto const* path : {"/_matrix/client/v1/rtc/transports", + "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports"}) + { + INFO(path); + auto const none = get(started.runtime, path); + auto const unknown = get(started.runtime, path, "not-a-token"); + CHECK(none.status == 401U); + CHECK(unknown.status == 401U); + CHECK(none.body.find("livekit") == std::string::npos); + CHECK(unknown.body.find("livekit") == std::string::npos); + } + } + } + } +} + +SCENARIO("/versions announces MatrixRTC discovery whether or not a service is configured", + "[homeserver][rtc][client-server]") +{ + GIVEN("servers with and without a LiveKit service URL") + { + auto with_url = merovingian::homeserver::start_client_server(voip_config(rtc_server(livekit_url))); + auto without = merovingian::homeserver::start_client_server(voip_config({})); + REQUIRE(with_url.started); + REQUIRE(without.started); + + WHEN("/versions is read on each") + { + auto const a = get(with_url.runtime, "/_matrix/client/versions"); + auto const b = get(without.runtime, "/_matrix/client/versions"); + + THEN("both carry org.matrix.msc4143 and org.matrix.msc4143.stable as true") + { + for (auto const* response : {&a, &b}) + { + REQUIRE(response->status == 200U); + auto const body = parse_object(response->body); + auto const* features = object_member_as_object(body, "unstable_features"); + REQUIRE(features != nullptr); + for (auto const* flag : {"org.matrix.msc4143", "org.matrix.msc4143.stable"}) + { + auto const* value = bool_member(*features, flag); + REQUIRE(value != nullptr); + CHECK(*value); + } + CHECK(bool_member(*features, "org.matrix.msc4186") != nullptr); + } + } + } + } +} + +SCENARIO("/.well-known/matrix/client carries rtc_foci only when a service is configured", + "[homeserver][rtc][client-server]") +{ + GIVEN("a server with a LiveKit service URL and a public base URL") + { + auto server = rtc_server(livekit_url); + server.public_baseurl = "https://matrix.example.org"; + auto started = merovingian::homeserver::start_client_server(voip_config(std::move(server))); + REQUIRE(started.started); + + WHEN("the well-known document is read") + { + auto const response = get(started.runtime, "/.well-known/matrix/client"); + + THEN("it adds org.matrix.msc4143.rtc_foci beside m.homeserver") + { + REQUIRE(response.status == 200U); + auto const body = parse_object(response.body); + CHECK( + response.body == + R"({"m.homeserver":{"base_url":"https://matrix.example.org"},"org.matrix.msc4143.rtc_foci":[{"livekit_service_url":"https://livekit-jwt.example.org/lk-jwt-service","type":"livekit"}]})"); + REQUIRE(object_member_as_object(body, "m.homeserver") != nullptr); + auto const* foci = object_member_as_array(body, "org.matrix.msc4143.rtc_foci"); + REQUIRE(foci != nullptr); + REQUIRE(foci->size() == 1U); + auto const* entry = std::get_if(&(*foci)[0U].storage()); + REQUIRE(entry != nullptr); + CHECK(entry->size() == 2U); + CHECK(*string_member(*entry, "type") == "livekit"); + CHECK(*string_member(*entry, "livekit_service_url") == livekit_url); + } + } + } + + GIVEN("a server with no LiveKit service URL") + { + auto server = merovingian::config::ServerConfig{}; + server.public_baseurl = "https://matrix.example.org"; + auto started = merovingian::homeserver::start_client_server(voip_config(std::move(server))); + REQUIRE(started.started); + + WHEN("the well-known document is read") + { + auto const response = get(started.runtime, "/.well-known/matrix/client"); + + THEN("the body is exactly what it was before MatrixRTC") + { + REQUIRE(response.status == 200U); + CHECK(response.body == R"({"m.homeserver":{"base_url":"https://matrix.example.org"}})"); + } + } + } +} diff --git a/tests/meson.build b/tests/meson.build index ebab03c98..ecd15dfcd 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -336,6 +336,7 @@ shell = find_program('sh') 'unit/test_config_sso.cpp', 'unit/test_config_identity_server.cpp', 'unit/test_config_turn_uris.cpp', + 'unit/test_config_rtc.cpp', 'unit/test_identity_bind_domains.cpp', 'unit/test_config_parser.cpp', 'unit/test_config_reload_plan.cpp', diff --git a/tests/unit/test_config_rtc.cpp b/tests/unit/test_config_rtc.cpp new file mode 100644 index 000000000..d128ca9a1 --- /dev/null +++ b/tests/unit/test_config_rtc.cpp @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: 2026 James Chapman +// SPDX-License-Identifier: GPL-3.0-or-later +// +// server.rtc.livekit.service_url: where MatrixRTC clients get a LiveKit token (MSC4143, MSC4195; ADR-0822). The +// routes that advertise it are not in Matrix v1.19, so these are project-level checks. + +#include "merovingian/config/Config.hpp" +#include "merovingian/config/ConfigParseResult.hpp" +#include "merovingian/config/ConfigParser.hpp" +#include "merovingian/config/ConfigValidationFinding.hpp" +#include "merovingian/config/ReloadPlan.hpp" +#include "merovingian/config/ReloadPolicy.hpp" +#include "merovingian/config/ServerConfig.hpp" + +#include + +#include +#include +#include + +namespace +{ + +[[nodiscard]] auto has_finding(std::vector const& findings, + std::string_view field) -> bool +{ + for (auto const& finding : findings) + { + if (finding.field == field) + { + return true; + } + } + return false; +} + +[[nodiscard]] auto parse_url(std::string const& url) -> merovingian::config::ConfigParseResult +{ + return merovingian::config::parse_key_value_config("server.rtc.livekit.service_url=" + url + "\n"); +} + +} // namespace + +SCENARIO("server.rtc.livekit.service_url accepts an https URL and defaults to none", "[config][rtc]") +{ + GIVEN("a configuration that does not mention the key") + { + auto const parsed = merovingian::config::parse_key_value_config(""); + + THEN("no transport is configured and nothing is reported") + { + CHECK(parsed.config.server().rtc.livekit_service_url.empty()); + CHECK_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + } + } + + GIVEN("a valid https URL, with a port and a path") + { + auto const parsed = parse_url("https://livekit-jwt.example.org:8443/lk-jwt-service"); + + THEN("it is kept exactly as written") + { + CHECK_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + CHECK(parsed.config.server().rtc.livekit_service_url == + "https://livekit-jwt.example.org:8443/lk-jwt-service"); + } + } +} + +SCENARIO("server.rtc.livekit.service_url refuses anything but a plain https URL", "[config][rtc]") +{ + GIVEN("values that are not acceptable") + { + auto const long_url = "https://example.org/" + std::string(2100U, 'a'); + auto const bad = std::vector{ + "http://livekit.example.org", + "HTTPS://livekit.example.org", + "ftp://livekit.example.org", + "livekit.example.org", + "//livekit.example.org", + "https://", + "https:///path", + "https://user@livekit.example.org", + "https://user:pw@livekit.example.org", + "https://livekit.example.org/#frag", + "https://livekit.example.org/a b", + "https://livekit.example.org/\x01", + "https://livekit.example.org/caf\xC3\xA9", + long_url, + }; + + THEN("each is refused with a finding on the key, and is never repaired") + { + for (auto const& url : bad) + { + INFO(url.substr(0U, 60U)); + auto const parsed = parse_url(url); + REQUIRE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + } + } + } +} + +SCENARIO("server.rtc.livekit.service_url is reloadable and diffed by the reload plan", "[config][rtc]") +{ + GIVEN("two configurations that differ only in the service URL") + { + auto const before = parse_url("https://a.example.org"); + auto const after = parse_url("https://b.example.org"); + + WHEN("a reload is planned") + { + auto const plan = merovingian::config::build_reload_plan(before.config, after.config); + + THEN("the key is named and needs no restart") + { + auto found = false; + for (auto const& change : plan.changes()) + { + found = found || change.key == "server.rtc.livekit.service_url"; + } + CHECK(found); + CHECK(merovingian::config::reload_policy_for_key("server.rtc.livekit.service_url") == + merovingian::config::ReloadPolicy::reloadable); + } + } + } +} From df6419b3eafd82a9595c2aa5affc8f35fc88f396 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 21:25:35 +0000 Subject: [PATCH 20/34] Hold server.turn.uris to the RFC 7065 grammar and refuse a service_url a client cannot append to A TURN entry with no host (turn:?transport=udp, turn::3478, turn:@x, turns:/x) passed and would reach every client; entries now follow the TURN URI grammar strictly. server.rtc.livekit.service_url is refused when it has a query or a path ending in /, since clients append /sfu/get or /get_token to it. ADR-0821, ADR-0822, the manual and the example configs state both rules and that server.turn.uris uses the ordinary comma-separated list syntax. Co-Authored-By: Claude Sonnet 5.5 --- config/merovingian.conf.bsd.example | 6 +- config/merovingian.conf.example | 6 +- ...beside-server-turn-server-which-is-kept.md | 16 ++- ...s-type-livekit-with-livekit-service-url.md | 5 +- docs/user-manual.md | 4 +- include/merovingian/config/RtcConfig.hpp | 6 +- src/config/Config.cpp | 99 +++++++++++++++-- tests/unit/test_config_rtc.cpp | 25 +++++ tests/unit/test_config_turn_uris.cpp | 103 ++++++++++++++++++ 9 files changed, 249 insertions(+), 21 deletions(-) diff --git a/config/merovingian.conf.bsd.example b/config/merovingian.conf.bsd.example index 105c5c086..c526135db 100644 --- a/config/merovingian.conf.bsd.example +++ b/config/merovingian.conf.bsd.example @@ -138,7 +138,9 @@ server.http.ipv6_client_prefix_length=64 # (use-auth-secret): each user then gets time-limited credentials of their own. # Set one or the other; both is refused. # server.turn.server=turn:turn.example.org:3478?transport=udp -# Or several URIs, in the order clients should try them (instead of server.turn.server; both is refused): +# Or several URIs, comma-separated, in the order clients should try them (instead of server.turn.server; both is +# refused). Each must be turn: or turns:, a host, an optional :port and an optional ?transport=udp or ?transport=tcp +# (RFC 7065), for example turn:[2001:db8::1]:3478. # server.turn.uris=turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp # server.turn.username=merovingian # server.turn.password=set-in-owner-only-secret-file @@ -149,6 +151,8 @@ server.http.ipv6_client_prefix_length=64 # The HTTPS URL of an lk-jwt-service (Element's LiveKit authorisation service) that you run beside a LiveKit SFU. # This server never contacts it: it only tells authenticated clients, and the /.well-known/matrix/client # document, where it is. Empty means no group-call transport is advertised. Reloadable. +# Clients append /sfu/get or /get_token to it, so write it with no trailing slash and no query: a URL ending in / +# or holding ? is refused. https://host, https://host:8443 and https://host/jwt are fine. # server.rtc.livekit.service_url=https://livekit-jwt.example.org/lk-jwt-service # Listener exposure diff --git a/config/merovingian.conf.example b/config/merovingian.conf.example index 082e1e84d..596edd463 100644 --- a/config/merovingian.conf.example +++ b/config/merovingian.conf.example @@ -111,7 +111,9 @@ server.http.ipv6_client_prefix_length=64 # (use-auth-secret): each user then gets time-limited credentials of their own. # Set one or the other; both is refused. # server.turn.server=turn:turn.example.org:3478?transport=udp -# Or several URIs, in the order clients should try them (instead of server.turn.server; both is refused): +# Or several URIs, comma-separated, in the order clients should try them (instead of server.turn.server; both is +# refused). Each must be turn: or turns:, a host, an optional :port and an optional ?transport=udp or ?transport=tcp +# (RFC 7065), for example turn:[2001:db8::1]:3478. # server.turn.uris=turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp # server.turn.username=merovingian # server.turn.password=set-in-owner-only-secret-file @@ -122,6 +124,8 @@ server.http.ipv6_client_prefix_length=64 # The HTTPS URL of an lk-jwt-service (Element's LiveKit authorisation service) that you run beside a LiveKit SFU. # This server never contacts it: it only tells authenticated clients, and the /.well-known/matrix/client # document, where it is. Empty means no group-call transport is advertised. Reloadable. +# Clients append /sfu/get or /get_token to it, so write it with no trailing slash and no query: a URL ending in / +# or holding ? is refused. https://host, https://host:8443 and https://host/jwt are fine. # server.rtc.livekit.service_url=https://livekit-jwt.example.org/lk-jwt-service # Listener exposure diff --git a/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md b/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md index db0a4a6cc..691af561b 100644 --- a/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md +++ b/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md @@ -14,13 +14,19 @@ that blocks UDP needs the second one. How does the configuration name several UR Chosen option: "a new list key beside the old one", because a 1.0.x configuration must still parse in 1.1 and the old value keeps its meaning. -* `server.turn.uris` is a comma-separated list (as `server.identity_server.trusted_servers` is). The response's `uris` - array carries the entries in the configured order. `server.turn.server` stays, and still gives a one-element array. +* `server.turn.uris` uses the configuration's ordinary comma-separated list syntax (as + `server.identity_server.trusted_servers` does): whitespace around an entry and empty entries are ignored, by the shared + `parse_string_list`, which this change does not alter. The response's `uris` array carries the entries in the + configured order. `server.turn.server` stays, and still gives a one-element array. * TURN is enabled when either key is set. Setting both is refused at parse time, like `shared_secret_file` with `username` and `password`, so it is never unclear which one clients get. -* Each `server.turn.uris` entry is refused, never repaired, unless it starts with `turn:` or `turns:` followed by - something, is at most 512 bytes of printable ASCII (no whitespace or control character), is not a repeat of an - earlier entry, and the list holds at most 16 entries. `server.turn.server` has never been validated, and is not +* "Refused, never repaired" applies to the content of each entry. An entry is refused unless it follows the TURN URI + grammar of RFC 7065, strictly: `turn:` or `turns:` in lower case; a host, being a name or IPv4 address of ASCII + letters, digits, `.` and `-` that does not start or end with `.` or `-`, or an IPv6 literal in square brackets of hex + digits, `:` and `.`; an optional `:port`, decimal, 1 to 65535, no sign, not empty; an optional `?transport=udp` or + `?transport=tcp`, exactly. No userinfo, path, fragment or other query. An entry with no host (`turn:?transport=udp`, + `turn::3478`) would be sent to every client, so it is refused. An entry is also at most 512 bytes of printable ASCII, + is not a repeat of an earlier entry, and the list holds at most 16 entries. `server.turn.server` has never been validated, and is not tightened here: a value an existing configuration holds must not start to be refused by an upgrade. * The key is reloadable and diffed by the reload plan like the other `server.turn.*` keys. diff --git a/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md b/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md index d6aaaa28a..dd7a1bfb2 100644 --- a/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md +++ b/docs/adr/0822-the-rtc-transport-is-advertised-as-type-livekit-with-livekit-service-url.md @@ -17,7 +17,10 @@ Chosen option: "advertise the operator's service as type `livekit`", because rel Element Call understand only that type. * `server.rtc.livekit.service_url` (an HTTPS URL, refused when not one; never fetched by this server) is advertised as - `{"type":"livekit","livekit_service_url":""}`. The entry has no `url` member. + `{"type":"livekit","livekit_service_url":""}`. The entry has no `url` member. Clients build + `/sfu/get` and `/get_token` by appending to it, so a URL with a query, or whose path ends + in `/`, would give a broken request and is refused; a URL with no path (`https://host`, `https://host:8443`) is + accepted, as are an IPv6 literal host and a port. * Both routes need an access token and are rate limited like any client route. The stable path answers `{"transports":[...]}` and, with nothing configured, `{"transports":[]}`. The unstable path answers `{"rtc_transports":[...]}` and, with nothing configured, `{}`. Both mirror the reference server. The transport diff --git a/docs/user-manual.md b/docs/user-manual.md index 5239d1151..38487fbfd 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -807,7 +807,7 @@ support. | Key | Default | When to change | |---|---|---| | `server.turn.server` | (empty) | TURN server URI advertised to clients, e.g. `turn:turn.example.org:3478?transport=udp`. Leave empty to keep the endpoint returning `{}`. | -| `server.turn.uris` | (empty) | Several TURN URIs, comma-separated, advertised in this order, e.g. `turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp`. An alternative to `server.turn.server`, which stays for existing files; setting both is refused. Each entry must start with `turn:` or `turns:`, be at most 512 bytes of printable ASCII with no whitespace, and appear once; at most 16 entries; a bad entry refuses the configuration. TURN is on when either key is set (ADR-0821). | +| `server.turn.uris` | (empty) | Several TURN URIs advertised in the order listed, e.g. `turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp`. An alternative to `server.turn.server`, which stays for existing files; setting both is refused. The list uses the configuration's ordinary comma-separated syntax: whitespace around an entry and empty entries are ignored. Each entry is then checked, and refused (never repaired) unless it follows RFC 7065: `turn:` or `turns:` in lower case, a host (a name or IPv4 address of ASCII letters, digits, `.` and `-` that does not start or end with `.` or `-`, or an IPv6 literal in square brackets such as `[2001:db8::1]`), an optional `:port` from 1 to 65535, and an optional `?transport=udp` or `?transport=tcp`. No user name, path, fragment or other query. An entry is at most 512 bytes of printable ASCII, may appear once, and the list holds at most 16 entries. A bad entry refuses the configuration. `server.turn.server` is not checked this way, so an existing value keeps working. TURN is on when either key is set (ADR-0821). | | `server.turn.username` | (empty) | Static username issued to authenticated clients. Required when `server.turn.server` or `server.turn.uris` is set (unless `shared_secret_file` is used). | | `server.turn.password` | (empty) | Static password issued alongside the username. Required when `server.turn.server` or `server.turn.uris` is set (unless `shared_secret_file` is used). | | `server.turn.shared_secret_file` | (empty) | Path to an owner-read-only file (mode `0400`, as every secret file) holding the TURN server's `static-auth-secret`, for coturn run with `use-auth-secret`. Each client then gets its own credentials: username `:`, password the base64 HMAC-SHA1 of that username under the secret (the coturn REST API), valid for `ttl_seconds`. Cannot be combined with `username` and `password`: the configuration is refused if both are set. | @@ -835,7 +835,7 @@ Merovingian does not carry call media. A call needs a service you run beside it: | Key | Default | When to change | |---|---|---| -| `server.rtc.livekit.service_url` | (empty) | The HTTPS URL of your `lk-jwt-service`, for example `https://livekit-jwt.example.org/lk-jwt-service`. Empty means no group-call transport is advertised. Must be an absolute `https` URL with a host, no user name or password, no `#fragment`, no whitespace and at most 2048 bytes; anything else refuses the configuration. The server never contacts this URL. | +| `server.rtc.livekit.service_url` | (empty) | The HTTPS URL of your `lk-jwt-service`, for example `https://livekit-jwt.example.org/lk-jwt-service`. Empty means no group-call transport is advertised. Clients build `/sfu/get` and `/get_token` by appending to it, so give the address without a trailing `/` and without a query: `https://host`, `https://host:8443` and `https://host/jwt` are accepted, `https://host/` and `https://host/jwt?x=1` are refused. It must be an absolute `https` URL with a host, no user name or password, no `#fragment`, no whitespace and at most 2048 bytes; anything else refuses the configuration. The server never contacts this URL. | With the key set, an authenticated client reading `GET /_matrix/client/v1/rtc/transports` (or the older `GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports`) learns a transport of type `livekit` with that diff --git a/include/merovingian/config/RtcConfig.hpp b/include/merovingian/config/RtcConfig.hpp index 803c26c3c..20acf4567 100644 --- a/include/merovingian/config/RtcConfig.hpp +++ b/include/merovingian/config/RtcConfig.hpp @@ -13,8 +13,10 @@ struct RtcConfig final { // The HTTPS URL of an external MatrixRTC authorisation service (Element's `lk-jwt-service`) that the operator // runs beside a LiveKit SFU. Advertised as a `livekit` transport. Empty means no transport is configured. - // validate() refuses a value that is not an absolute `https` URL with a host, no userinfo, no fragment, no - // whitespace or control character, and at most 2048 bytes. + // Clients build `/sfu/get` and `/get_token` by appending to it. validate() refuses a + // value that is not an absolute `https` URL with a host, no userinfo, no query, no fragment, no whitespace or + // control character and at most 2048 bytes, or whose path ends in `/`. A URL with no path (`https://host`, + // `https://host:8443`) is accepted. std::string livekit_service_url{}; }; diff --git a/src/config/Config.cpp b/src/config/Config.cpp index c986866ca..56be99b00 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -348,6 +349,85 @@ namespace constexpr auto max_turn_uri_bytes = std::size_t{512U}; constexpr auto max_turn_uris = std::size_t{16U}; + // Whether `uri` is a TURN URI of RFC 7065, strictly: `turn:` or `turns:`, a host (a name or IPv4 address of ASCII + // letters, digits, `.` and `-`, not starting or ending with `.` or `-`, or a bracketed IPv6 literal of hex digits, + // `:` and `.`), an optional `:port` of 1 to 65535, and an optional `?transport=udp` or `?transport=tcp`. No + // userinfo, path, fragment or other query. + auto is_turn_uri(std::string_view uri) -> bool + { + auto const rest = starts_with(uri, "turn:") ? uri.substr(5U) + : starts_with(uri, "turns:") ? uri.substr(6U) + : std::string_view{}; + auto const question = rest.find('?'); + if (question != std::string_view::npos) + { + auto const query = rest.substr(question + 1U); + if (query != "transport=udp" && query != "transport=tcp") + { + return false; + } + } + auto const authority = rest.substr(0U, question); + auto host = std::string_view{}; + auto port = std::optional{}; + if (!authority.empty() && authority.front() == '[') + { + auto const close = authority.find(']'); + if (close == std::string_view::npos) + { + return false; + } + host = authority.substr(1U, close - 1U); + auto const after = authority.substr(close + 1U); + if (!after.empty()) + { + if (after.front() != ':') + { + return false; + } + port = after.substr(1U); + } + if (host.empty() || host.find(':') == std::string_view::npos || !std::ranges::all_of(host, [](char c) { + return std::isxdigit(static_cast(c)) != 0 || c == ':' || c == '.'; + })) + { + return false; + } + } + else + { + auto const colon = authority.find(':'); + host = authority.substr(0U, colon); + if (colon != std::string_view::npos) + { + port = authority.substr(colon + 1U); + } + auto const is_host_byte = [](char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || + c == '-'; + }; + if (host.empty() || !std::ranges::all_of(host, is_host_byte) || host.front() == '.' || + host.front() == '-' || host.back() == '.' || host.back() == '-') + { + return false; + } + } + if (port.has_value()) + { + constexpr auto max_port_digits = std::size_t{5U}; + if (port->empty() || port->size() > max_port_digits || !std::ranges::all_of(*port, is_ascii_digit)) + { + return false; + } + auto const value = std::stoul(std::string{*port}); + if (value == 0U || value > 65535U) + { + return false; + } + } + return true; + } + // Why `uris` is not an acceptable server.turn.uris list, or nullopt. Fixed text, never the input. auto turn_uris_error(std::vector const& uris) -> std::optional { @@ -358,12 +438,10 @@ namespace for (auto index = std::size_t{0U}; index < uris.size(); ++index) { auto const& uri = uris[index]; - auto const rest = starts_with(uri, "turn:") ? std::string_view{uri}.substr(5U) - : starts_with(uri, "turns:") ? std::string_view{uri}.substr(6U) - : std::string_view{}; - if (rest.empty()) + if (!is_turn_uri(uri)) { - return "each TURN URI must start with turn: or turns: and name a host"; + return "each TURN URI must be turn: or turns: and a host, an optional :port and an optional " + "?transport=udp or ?transport=tcp"; } if (uri.size() > max_turn_uri_bytes) { @@ -1240,11 +1318,14 @@ auto validate(Config const& config) -> std::vector { auto const uri = oauth::ParsedUri::read(service_url); auto const* parsed = std::get_if(&uri); - if (parsed == nullptr || parsed->scheme() != "https" || !parsed->has_authority() || parsed->host().empty()) + // Clients append `/sfu/get` or `/get_token` to it, so a query or a trailing slash would build a broken URL. + if (parsed == nullptr || parsed->scheme() != "https" || !parsed->has_authority() || parsed->host().empty() || + parsed->has_query() || parsed->path().ends_with('/')) { - findings.push_back({"server.rtc.livekit.service_url", - "must be an https URL with a host, no userinfo, no fragment, no whitespace and at most " - "2048 bytes"}); + findings.push_back( + {"server.rtc.livekit.service_url", + "must be an https URL with a host, no userinfo, query or fragment, a path that does not " + "end in /, no whitespace and at most 2048 bytes"}); } } diff --git a/tests/unit/test_config_rtc.cpp b/tests/unit/test_config_rtc.cpp index d128ca9a1..0329a8cbf 100644 --- a/tests/unit/test_config_rtc.cpp +++ b/tests/unit/test_config_rtc.cpp @@ -67,6 +67,25 @@ SCENARIO("server.rtc.livekit.service_url accepts an https URL and defaults to no } } +SCENARIO("server.rtc.livekit.service_url accepts a URL a client can append to", "[config][rtc]") +{ + GIVEN("URLs that /sfu/get or /get_token can be appended to") + { + THEN("each is accepted: a bare host, a port, an IPv6 literal, a path with no trailing slash") + { + for (auto const* good : + {"https://livekit-jwt.example.org", "https://livekit-jwt.example.org:8443", "https://[2001:db8::1]", + "https://[2001:db8::1]:8443/jwt", "https://example.org/a/b"}) + { + INFO(good); + auto const parsed = parse_url(good); + REQUIRE_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + REQUIRE(parsed.config.server().rtc.livekit_service_url == good); + } + } + } +} + SCENARIO("server.rtc.livekit.service_url refuses anything but a plain https URL", "[config][rtc]") { GIVEN("values that are not acceptable") @@ -86,6 +105,12 @@ SCENARIO("server.rtc.livekit.service_url refuses anything but a plain https URL" "https://livekit.example.org/a b", "https://livekit.example.org/\x01", "https://livekit.example.org/caf\xC3\xA9", + "https://livekit.example.org/x?a=1", + "https://livekit.example.org?a=1", + "https://livekit.example.org/x?", + "https://livekit.example.org/jwt/", + "https://livekit.example.org/", + "https://livekit.example.org:8443/", long_url, }; diff --git a/tests/unit/test_config_turn_uris.cpp b/tests/unit/test_config_turn_uris.cpp index 972d3b0dd..34058c753 100644 --- a/tests/unit/test_config_turn_uris.cpp +++ b/tests/unit/test_config_turn_uris.cpp @@ -97,6 +97,109 @@ SCENARIO("server.turn.uris lists several TURN URIs in order", "[config][turn][tu } } +SCENARIO("server.turn.uris accepts the RFC 7065 forms and refuses everything else", "[config][turn][turn-uris]") +{ + GIVEN("entries that follow the TURN URI grammar") + { + THEN("each is accepted") + { + for (auto const* good : + {"turn:turn.example.org", "turn:turn.example.org:3478", "turn:turn.example.org:3478?transport=udp", + "turns:turn.example.org:5349?transport=tcp", "turn:192.0.2.1:3478", "turn:[2001:db8::1]:3478", + "turn:[2001:db8::1]", "turn:a-b.example.org:65535", "turn:turn.example.org:1?transport=tcp"}) + { + INFO(good); + auto const parsed = parse_uris(good); + REQUIRE(merovingian::config::is_valid(parsed.config)); + REQUIRE_FALSE(has_finding(parsed.findings, "server.turn.uris")); + } + } + } + + GIVEN("entries with no host, or that leave the grammar") + { + THEN("each is refused") + { + for (auto const* bad : {"turn:?transport=udp", + "turn::3478", + "turn:@x", + "turns:/x", + "turn:turn.example.org:", + "turn:turn.example.org:0", + "turn:turn.example.org:65536", + "turn:turn.example.org:+3478", + "turn:turn.example.org:34a8", + "turn:turn.example.org:3478?transport=sctp", + "turn:turn.example.org:3478?transport=udp&x=1", + "turn:turn.example.org:3478?x=1", + "turn:turn.example.org:3478?transport=", + "turn:turn.example.org:3478?transport=udp?x", + "turn:turn.example.org.", + "turn:.example.org", + "turn:-example.org", + "turn:example.org-", + "turn:u@turn.example.org", + "turn:turn.example.org/path", + "turn:turn.example.org#frag", + "turn:turn.example.org:3478#frag", + "turn:turn_example.org", + "turn:[2001:db8::1", + "turn:2001:db8::1]:3478", + "turn:[]:3478", + "turn:[2001:db8::g]:3478", + "turn:[2001:db8::1]x", + "turn:[2001:db8::1]:", + "turn:[example.org]"}) + { + INFO(bad); + auto const parsed = parse_uris(std::string{"turn:ok.example.org,"} + bad); + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } + } + } +} + +SCENARIO("server.turn.uris uses the config's ordinary comma-separated list syntax", "[config][turn][turn-uris]") +{ + GIVEN("a list with an empty entry between two URIs") + { + auto const parsed = parse_uris("turn:a.example,,turn:b.example"); + + THEN("the empty entry is ignored") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + REQUIRE(parsed.config.server().turn.uris.size() == 2U); + CHECK(parsed.config.server().turn.uris[0U] == "turn:a.example"); + CHECK(parsed.config.server().turn.uris[1U] == "turn:b.example"); + } + } + + GIVEN("a list with a trailing comma and whitespace around entries") + { + auto const parsed = parse_uris(" turn:a.example ,\tturn:b.example , "); + + THEN("whitespace around an entry and the empty last entry are ignored") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + REQUIRE(parsed.config.server().turn.uris.size() == 2U); + CHECK(parsed.config.server().turn.uris[0U] == "turn:a.example"); + CHECK(parsed.config.server().turn.uris[1U] == "turn:b.example"); + } + } + + GIVEN("a value of only a comma") + { + auto const parsed = merovingian::config::parse_key_value_config("server.turn.uris=,\n"); + + THEN("the list is empty and TURN is not configured") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + CHECK(parsed.config.server().turn.uris.empty()); + } + } +} + SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][turn][turn-uris]") { GIVEN("both server.turn.server and server.turn.uris") From 038bb2122a6a6e0f2576ba7b08eebff9ec235a5e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 21:25:35 +0000 Subject: [PATCH 21/34] Test the MatrixRTC routes against voip/turnServer for methods and moderated accounts Pins that POST and PUT answer what voip/turnServer answers, that a locked or suspended account is treated alike, and the whole 1.0.x turnServer body for a single server.turn.server with static credentials. Co-Authored-By: Claude Sonnet 5.5 --- .../integration/test_voip_discovery_flow.cpp | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/integration/test_voip_discovery_flow.cpp b/tests/integration/test_voip_discovery_flow.cpp index f73310245..9f91f749d 100644 --- a/tests/integration/test_voip_discovery_flow.cpp +++ b/tests/integration/test_voip_discovery_flow.cpp @@ -16,6 +16,7 @@ #include "merovingian/config/LogModulesConfig.hpp" #include "merovingian/config/SecurityConfig.hpp" #include "merovingian/config/ServerConfig.hpp" +#include "merovingian/homeserver/AuthService.hpp" #include "merovingian/homeserver/ClientServerRuntime.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" #include "merovingian/homeserver/LocalHttpResponse.hpp" @@ -384,3 +385,140 @@ SCENARIO("/.well-known/matrix/client carries rtc_foci only when a service is con } } } + +namespace +{ + +[[nodiscard]] auto send(merovingian::homeserver::ClientServerRuntime& runtime, std::string const& method, + std::string const& target, std::string const& token = {}) + -> merovingian::homeserver::LocalHttpResponse +{ + return merovingian::homeserver::handle_client_server_request(runtime, {method, target, token, {}}).response; +} + +[[nodiscard]] auto header_of(merovingian::homeserver::LocalHttpResponse const& response, std::string_view name) + -> std::string +{ + for (auto const& [header_name, value] : response.headers) + { + if (header_name == name) + { + return value; + } + } + return {}; +} + +[[nodiscard]] auto errcode_of(merovingian::homeserver::LocalHttpResponse const& response) -> std::string +{ + auto const body = parse_object(response.body); + auto const* errcode = string_member(body, "errcode"); + return errcode == nullptr ? std::string{} : *errcode; +} + +constexpr auto turn_path = "/_matrix/client/v3/voip/turnServer"; +constexpr auto rtc_stable_path = "/_matrix/client/v1/rtc/transports"; +constexpr auto rtc_unstable_path = "/_matrix/client/unstable/org.matrix.msc4143/rtc/transports"; + +} // namespace + +SCENARIO("The MatrixRTC routes refuse other methods as voip/turnServer does", "[homeserver][rtc][client-server]") +{ + GIVEN("a server with a LiveKit service URL and a logged-in user") + { + auto started = merovingian::homeserver::start_client_server(voip_config(rtc_server(livekit_url))); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("POST and PUT are sent to each route and to voip/turnServer") + { + THEN("each route answers the same status, errcode and Allow header as voip/turnServer") + { + for (auto const* method : {"POST", "PUT"}) + { + auto const reference = send(started.runtime, method, turn_path, token); + REQUIRE(reference.status == 405U); + for (auto const* path : {rtc_stable_path, rtc_unstable_path}) + { + INFO(method << ' ' << path); + auto const response = send(started.runtime, method, path, token); + CHECK(response.status == reference.status); + CHECK(errcode_of(response) == errcode_of(reference)); + CHECK(header_of(response, "Allow") == header_of(reference, "Allow")); + CHECK(header_of(response, "Allow") == "GET"); + } + } + } + } + } +} + +SCENARIO("The MatrixRTC routes treat a locked or suspended account as voip/turnServer does", + "[homeserver][rtc][client-server]") +{ + GIVEN("a logged-in user whose account is then locked or suspended") + { + auto server = rtc_server(livekit_url); + server.turn.server = "turn:turn.example.org:3478"; + server.turn.username = "turnuser"; + server.turn.password = "turnpass"; + auto started = merovingian::homeserver::start_client_server(voip_config(std::move(server))); + REQUIRE(started.started); + auto const token = login(started.runtime); + auto const suspended = GENERATE(false, true); + REQUIRE( + merovingian::homeserver::bootstrap_admin_user(started.runtime.homeserver, "admin", "CorrectHorse7!").ok); + REQUIRE(merovingian::homeserver::set_local_user_moderation(started.runtime.homeserver, "@admin:example.org", + "@alice:example.org", suspended, !suspended, + "account.locked", "locked") + .ok); + + WHEN("the user asks for TURN credentials and for each transports route") + { + auto const reference = send(started.runtime, "GET", turn_path, token); + + THEN("each route answers the same status and errcode, and a refusal names no service URL") + { + for (auto const* path : {rtc_stable_path, rtc_unstable_path}) + { + INFO((suspended ? "suspended " : "locked ") << path); + auto const response = send(started.runtime, "GET", path, token); + CHECK(response.status == reference.status); + CHECK(errcode_of(response) == errcode_of(reference)); + if (reference.status != 200U) + { + CHECK(response.body.find("livekit") == std::string::npos); + } + } + } + } + } +} + +SCENARIO("With only server.turn.server and static credentials voip/turnServer answers as in 1.0.x", + "[homeserver][turn][client-server]") +{ + GIVEN("a server configured the 1.0.x way") + { + auto server = merovingian::config::ServerConfig{}; + server.turn.server = "turn:turn.example.org:3478?transport=udp"; + server.turn.username = "turnuser"; + server.turn.password = "turnpass"; + auto started = merovingian::homeserver::start_client_server(voip_config(std::move(server))); + REQUIRE(started.started); + auto const token = login(started.runtime); + + WHEN("a client asks for TURN credentials") + { + auto const response = get(started.runtime, turn_path, token); + + THEN("the whole body is the one the 1.0 series sent") + { + REQUIRE(response.status == 200U); + CHECK( + response.body == + R"({"password":"turnpass","ttl":86400,"uris":["turn:turn.example.org:3478?transport=udp"],"username":"turnuser"})"); + } + } + } +} From 02ecb194b5b283a0888329ba0fdb0df5ef6b4478 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 21:25:36 +0000 Subject: [PATCH 22/34] Record MatrixRTC transport discovery in the threat model Co-Authored-By: Claude Sonnet 5.5 --- docs/threat-model.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/threat-model.md b/docs/threat-model.md index 960c8e506..1d8e45e31 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -3324,3 +3324,16 @@ directly, so these are unverified: the browser's handling of the `__Host-` cooki Chromium's `form-action` lets the 303 after a form post through to the client, IPv6 loopback redirects, and a client's deep links to the account pages. Milestone A in [the OAuth 2.0 plan](todos/oauth-2-0-api.md) (Element X on iOS and Android against a server built from this branch) closes this item. + +### MatrixRTC transport discovery (1.1.1, ADR-0822) + +New surface: `server.rtc.livekit.service_url` is advertised in an unauthenticated document, the `org.matrix.msc4143.rtc_foci` +member of `/.well-known/matrix/client`, and by two routes, `GET /_matrix/client/v1/rtc/transports` and its MSC4143 +unstable path, which need an access token and are rate limited like any client route. Disclosed: an address the +operator chose for their MatrixRTC authorisation service, public by design because clients must be told where it is. +Threats: an attacker steering clients to a service of their choosing, or injecting into the document. Mitigations: the +value comes only from the operator's configuration file, never from a request; it is refused at parse time unless it +is an absolute `https` URL with a host, no userinfo, query or fragment, no whitespace and a path that does not end in `/`; +every body is built with the JSON serializer, never by concatenating the URL into a string; and the server never +fetches the URL, so it adds no outbound request and no SSRF path. Residual risk: an operator who points it at a service +they do not control hands that service the OpenID tokens clients send it. From d4763fa20ca8dd0af29b43a493f07bac9d343ef1 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 21:34:02 +0000 Subject: [PATCH 23/34] Include the Catch2 generators header the moderated-account scenario needs Co-Authored-By: Claude Sonnet 5.5 --- tests/integration/test_voip_discovery_flow.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_voip_discovery_flow.cpp b/tests/integration/test_voip_discovery_flow.cpp index 9f91f749d..131395ce8 100644 --- a/tests/integration/test_voip_discovery_flow.cpp +++ b/tests/integration/test_voip_discovery_flow.cpp @@ -22,6 +22,7 @@ #include "merovingian/homeserver/LocalHttpResponse.hpp" #include +#include #include #include From f4cd7748231966b65d2c5961866d223f82de55f8 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 16 Nov 2026 21:53:19 +0000 Subject: [PATCH 24/34] Require a bracketed TURN host to be a valid IPv6 address The character-set check let turn:[:], turn:[::::] and turn:[12345::1] through; inet_pton now judges the text between the brackets, after the character check that still refuses a zone identifier. Co-Authored-By: Claude Sonnet 5.5 --- ...-beside-server-turn-server-which-is-kept.md | 4 ++-- docs/user-manual.md | 2 +- src/config/Config.cpp | 18 ++++++++++++++---- tests/unit/test_config_turn_uris.cpp | 11 +++++++++-- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md b/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md index 691af561b..e310d5d11 100644 --- a/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md +++ b/docs/adr/0821-server-turn-uris-is-added-beside-server-turn-server-which-is-kept.md @@ -22,8 +22,8 @@ value keeps its meaning. `username` and `password`, so it is never unclear which one clients get. * "Refused, never repaired" applies to the content of each entry. An entry is refused unless it follows the TURN URI grammar of RFC 7065, strictly: `turn:` or `turns:` in lower case; a host, being a name or IPv4 address of ASCII - letters, digits, `.` and `-` that does not start or end with `.` or `-`, or an IPv6 literal in square brackets of hex - digits, `:` and `.`; an optional `:port`, decimal, 1 to 65535, no sign, not empty; an optional `?transport=udp` or + letters, digits, `.` and `-` that does not start or end with `.` or `-`, or a valid IPv6 address in square brackets (hex + digits, `:` and `.` only, so no zone identifier, and `inet_pton` must accept it); an optional `:port`, decimal, 1 to 65535, no sign, not empty; an optional `?transport=udp` or `?transport=tcp`, exactly. No userinfo, path, fragment or other query. An entry with no host (`turn:?transport=udp`, `turn::3478`) would be sent to every client, so it is refused. An entry is also at most 512 bytes of printable ASCII, is not a repeat of an earlier entry, and the list holds at most 16 entries. `server.turn.server` has never been validated, and is not diff --git a/docs/user-manual.md b/docs/user-manual.md index 38487fbfd..05fca380c 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -807,7 +807,7 @@ support. | Key | Default | When to change | |---|---|---| | `server.turn.server` | (empty) | TURN server URI advertised to clients, e.g. `turn:turn.example.org:3478?transport=udp`. Leave empty to keep the endpoint returning `{}`. | -| `server.turn.uris` | (empty) | Several TURN URIs advertised in the order listed, e.g. `turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp`. An alternative to `server.turn.server`, which stays for existing files; setting both is refused. The list uses the configuration's ordinary comma-separated syntax: whitespace around an entry and empty entries are ignored. Each entry is then checked, and refused (never repaired) unless it follows RFC 7065: `turn:` or `turns:` in lower case, a host (a name or IPv4 address of ASCII letters, digits, `.` and `-` that does not start or end with `.` or `-`, or an IPv6 literal in square brackets such as `[2001:db8::1]`), an optional `:port` from 1 to 65535, and an optional `?transport=udp` or `?transport=tcp`. No user name, path, fragment or other query. An entry is at most 512 bytes of printable ASCII, may appear once, and the list holds at most 16 entries. A bad entry refuses the configuration. `server.turn.server` is not checked this way, so an existing value keeps working. TURN is on when either key is set (ADR-0821). | +| `server.turn.uris` | (empty) | Several TURN URIs advertised in the order listed, e.g. `turn:turn.example.org:3478?transport=udp,turns:turn.example.org:5349?transport=tcp`. An alternative to `server.turn.server`, which stays for existing files; setting both is refused. The list uses the configuration's ordinary comma-separated syntax: whitespace around an entry and empty entries are ignored. Each entry is then checked, and refused (never repaired) unless it follows RFC 7065: `turn:` or `turns:` in lower case, a host (a name or IPv4 address of ASCII letters, digits, `.` and `-` that does not start or end with `.` or `-`, or a valid IPv6 address in square brackets such as `[2001:db8::1]`, with no zone identifier), an optional `:port` from 1 to 65535, and an optional `?transport=udp` or `?transport=tcp`. No user name, path, fragment or other query. An entry is at most 512 bytes of printable ASCII, may appear once, and the list holds at most 16 entries. A bad entry refuses the configuration. `server.turn.server` is not checked this way, so an existing value keeps working. TURN is on when either key is set (ADR-0821). | | `server.turn.username` | (empty) | Static username issued to authenticated clients. Required when `server.turn.server` or `server.turn.uris` is set (unless `shared_secret_file` is used). | | `server.turn.password` | (empty) | Static password issued alongside the username. Required when `server.turn.server` or `server.turn.uris` is set (unless `shared_secret_file` is used). | | `server.turn.shared_secret_file` | (empty) | Path to an owner-read-only file (mode `0400`, as every secret file) holding the TURN server's `static-auth-secret`, for coturn run with `use-auth-secret`. Each client then gets its own credentials: username `:`, password the base64 HMAC-SHA1 of that username under the secret (the coturn REST API), valid for `ttl_seconds`. Cannot be combined with `username` and `password`: the configuration is refused if both are set. | diff --git a/src/config/Config.cpp b/src/config/Config.cpp index 56be99b00..c7c8b1dc0 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -350,9 +350,9 @@ namespace constexpr auto max_turn_uris = std::size_t{16U}; // Whether `uri` is a TURN URI of RFC 7065, strictly: `turn:` or `turns:`, a host (a name or IPv4 address of ASCII - // letters, digits, `.` and `-`, not starting or ending with `.` or `-`, or a bracketed IPv6 literal of hex digits, - // `:` and `.`), an optional `:port` of 1 to 65535, and an optional `?transport=udp` or `?transport=tcp`. No - // userinfo, path, fragment or other query. + // letters, digits, `.` and `-`, not starting or ending with `.` or `-`, or a valid IPv6 address (checked with + // inet_pton) in square brackets, of hex digits, `:` and `.`, so no zone identifier), an optional `:port` of 1 to + // 65535, and an optional `?transport=udp` or `?transport=tcp`. No userinfo, path, fragment or other query. auto is_turn_uri(std::string_view uri) -> bool { auto const rest = starts_with(uri, "turn:") ? uri.substr(5U) @@ -387,12 +387,22 @@ namespace } port = after.substr(1U); } - if (host.empty() || host.find(':') == std::string_view::npos || !std::ranges::all_of(host, [](char c) { + if (host.empty() || host.size() >= INET6_ADDRSTRLEN || host.find(':') == std::string_view::npos || + !std::ranges::all_of(host, [](char c) { return std::isxdigit(static_cast(c)) != 0 || c == ':' || c == '.'; })) { return false; } + // Only hex digits, `:` and `.` get here (so no zone identifier); the system parser then says whether they + // spell an IPv6 address. The copy is NUL-terminated because the view is not. + auto literal = std::array{}; + std::ranges::copy(host, literal.begin()); + auto v6 = in6_addr{}; + if (::inet_pton(AF_INET6, literal.data(), &v6) != 1) + { + return false; + } } else { diff --git a/tests/unit/test_config_turn_uris.cpp b/tests/unit/test_config_turn_uris.cpp index 34058c753..444a74646 100644 --- a/tests/unit/test_config_turn_uris.cpp +++ b/tests/unit/test_config_turn_uris.cpp @@ -106,7 +106,8 @@ SCENARIO("server.turn.uris accepts the RFC 7065 forms and refuses everything els for (auto const* good : {"turn:turn.example.org", "turn:turn.example.org:3478", "turn:turn.example.org:3478?transport=udp", "turns:turn.example.org:5349?transport=tcp", "turn:192.0.2.1:3478", "turn:[2001:db8::1]:3478", - "turn:[2001:db8::1]", "turn:a-b.example.org:65535", "turn:turn.example.org:1?transport=tcp"}) + "turn:[::1]", "turn:[::ffff:192.0.2.1]:3478?transport=tcp", "turn:[2001:db8::1]", + "turn:a-b.example.org:65535", "turn:turn.example.org:1?transport=tcp"}) { INFO(good); auto const parsed = parse_uris(good); @@ -149,7 +150,13 @@ SCENARIO("server.turn.uris accepts the RFC 7065 forms and refuses everything els "turn:[2001:db8::g]:3478", "turn:[2001:db8::1]x", "turn:[2001:db8::1]:", - "turn:[example.org]"}) + "turn:[example.org]", + "turn:[:]", + "turn:[::::]", + "turn:[12345::1]", + "turn:[]", + "turn:[::1", + "turn:[fe80::1%eth0]"}) { INFO(bad); auto const parsed = parse_uris(std::string{"turn:ok.example.org,"} + bad); From cd0cd410dd3f305642b2a31da31f254c0f631166 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:20:40 +0000 Subject: [PATCH 25/34] State the use_state_after fork limit in the 1.1.1 release notes ADR-0819 accepts that a key flipped back to the value recorded at since after a state fork is not sent in state_after; operators and clients should read it where release notes are read. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796b51fbd..d71a659cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) 1.1.1 adds `use_state_after` to `/sync`, `server.turn.uris` and MatrixRTC transport discovery. The new keys (`server.turn.uris`, `server.rtc.livekit.service_url`) are optional; no configuration change is needed. Group calls -still need your own LiveKit SFU and `lk-jwt-service`, and delayed events (MSC4140) are not implemented. +still need your own LiveKit SFU and `lk-jwt-service`, and delayed events (MSC4140) are not implemented. Known limit +of `use_state_after`: after a state fork at the client's `since`, a key that state resolution later flips back to the +value recorded at `since` is not sent in `state_after`; a client repairs itself with `full_state=true` (ADR-0819). # Version 1.1.0 From 8bd9914ea53260eaf95941649cea99f56d4fbf24 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:21:48 +0000 Subject: [PATCH 26/34] Name the upgrade fixture's scratch files uniquely and check their writes The fixture scenarios' key file, URI file and SQLite copy took a steady-clock count as their only uniqueness. They now use unique_temporary_path(), and the key and URI writers require the stream opened, wrote and closed. Co-Authored-By: Claude Sonnet 5.5 --- tests/integration/test_database_upgrade_flow.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_database_upgrade_flow.cpp b/tests/integration/test_database_upgrade_flow.cpp index 07b221afd..1c32fbaa6 100644 --- a/tests/integration/test_database_upgrade_flow.cpp +++ b/tests/integration/test_database_upgrade_flow.cpp @@ -473,13 +473,14 @@ struct FixtureExpectations final auto length = std::size_t{0U}; REQUIRE(sodium_hex2bin(key.data(), key.size(), hex.c_str(), hex.size(), nullptr, &length, nullptr) == 0); REQUIRE(length == key.size()); - auto const path = merovingian::tests::temporary_directory() / - ("merovingian-upgrade-fixture-master-" + - std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".key"); + auto const path = merovingian::tests::unique_temporary_path("merovingian-upgrade-fixture-master", ".key"); { auto output = std::ofstream{path, std::ios::binary}; + REQUIRE(output.is_open()); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) output.write(reinterpret_cast(key.data()), static_cast(key.size())); + output.close(); + REQUIRE_FALSE(output.fail()); } std::filesystem::permissions(path, std::filesystem::perms::owner_read); return path; @@ -529,12 +530,13 @@ struct FixtureExpectations final // The runtime reads its database URI only from an owner-read-only file (ISO-7). [[nodiscard]] auto write_uri_file(std::string const& conninfo) -> std::filesystem::path { - auto const path = - merovingian::tests::temporary_directory() / - ("merovingian-upgrade-uri-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + auto const path = merovingian::tests::unique_temporary_path("merovingian-upgrade-uri", ""); { auto output = std::ofstream{path}; + REQUIRE(output.is_open()); output << conninfo << '\n'; + output.close(); + REQUIRE_FALSE(output.fail()); } std::filesystem::permissions(path, std::filesystem::perms::owner_read); return path; @@ -1319,7 +1321,7 @@ SCENARIO("The 1.0.3 release's own SQLite database is migrated and served by this GIVEN("a copy of the SQLite database the last 1.0.x release wrote, and its master key") { auto const expected = load_fixture_expectations("sqlite"); - auto const path = unique_sqlite_path("fixture"); + auto const path = merovingian::tests::unique_temporary_path("merovingian-upgrade-fixture", ".sqlite3"); std::filesystem::copy_file(fixture_directory() / "merovingian.sqlite3", path, std::filesystem::copy_options::overwrite_existing); std::filesystem::permissions(path, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); From ebe8bd3953d3030398af60f4e5ae78f1d0c37aae Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:22:27 +0000 Subject: [PATCH 27/34] Require JSON members and variant pointers to exist before the TURN and RTC flow tests dereference them A response missing a field now fails one scenario instead of crashing the binary. Co-Authored-By: Claude Sonnet 5.5 --- .../test_turn_shared_secret_flow.cpp | 9 +++-- .../integration/test_voip_discovery_flow.cpp | 37 ++++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_turn_shared_secret_flow.cpp b/tests/integration/test_turn_shared_secret_flow.cpp index 21296dd28..ffdc24c2b 100644 --- a/tests/integration/test_turn_shared_secret_flow.cpp +++ b/tests/integration/test_turn_shared_secret_flow.cpp @@ -225,9 +225,12 @@ SCENARIO("GET /voip/turnServer lists every server.turn.uris entry in order with auto const* uris = object_member_as_array(body, "uris"); REQUIRE(uris != nullptr); REQUIRE(uris->size() == 2U); - REQUIRE(*std::get_if(&(*uris)[0U].storage()) == - "turn:turn.example.org:3478?transport=udp"); - REQUIRE(*std::get_if(&(*uris)[1U].storage()) == "turns:turn.example.org:5349"); + auto const* first = std::get_if(&(*uris)[0U].storage()); + auto const* second = std::get_if(&(*uris)[1U].storage()); + REQUIRE(first != nullptr); + REQUIRE(second != nullptr); + REQUIRE(*first == "turn:turn.example.org:3478?transport=udp"); + REQUIRE(*second == "turns:turn.example.org:5349"); } } } diff --git a/tests/integration/test_voip_discovery_flow.cpp b/tests/integration/test_voip_discovery_flow.cpp index 131395ce8..75d6454f5 100644 --- a/tests/integration/test_voip_discovery_flow.cpp +++ b/tests/integration/test_voip_discovery_flow.cpp @@ -109,10 +109,15 @@ SCENARIO("GET /voip/turnServer returns every server.turn.uris entry in order wit { REQUIRE(response.status == 200U); auto const body = parse_object(response.body); - REQUIRE(string_member(body, "username") != nullptr); - CHECK(*string_member(body, "username") == "turnuser"); - CHECK(*string_member(body, "password") == "turnpass"); - CHECK(*int_member(body, "ttl") == 600); + auto const* username = string_member(body, "username"); + REQUIRE(username != nullptr); + CHECK(*username == "turnuser"); + auto const* password = string_member(body, "password"); + REQUIRE(password != nullptr); + CHECK(*password == "turnpass"); + auto const* ttl = int_member(body, "ttl"); + REQUIRE(ttl != nullptr); + CHECK(*ttl == 600); auto const* uris = object_member_as_array(body, "uris"); REQUIRE(uris != nullptr); REQUIRE(uris->size() == 3U); @@ -207,8 +212,12 @@ SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", auto const* entry = std::get_if(&(*transports)[0U].storage()); REQUIRE(entry != nullptr); CHECK(entry->size() == 2U); - CHECK(*string_member(*entry, "type") == "livekit"); - CHECK(*string_member(*entry, "livekit_service_url") == livekit_url); + auto const* type = string_member(*entry, "type"); + REQUIRE(type != nullptr); + CHECK(*type == "livekit"); + auto const* service_url = string_member(*entry, "livekit_service_url"); + REQUIRE(service_url != nullptr); + CHECK(*service_url == livekit_url); } } @@ -230,8 +239,12 @@ SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", auto const* entry = std::get_if(&(*transports)[0U].storage()); REQUIRE(entry != nullptr); CHECK(entry->size() == 2U); - CHECK(*string_member(*entry, "type") == "livekit"); - CHECK(*string_member(*entry, "livekit_service_url") == livekit_url); + auto const* type = string_member(*entry, "type"); + REQUIRE(type != nullptr); + CHECK(*type == "livekit"); + auto const* service_url = string_member(*entry, "livekit_service_url"); + REQUIRE(service_url != nullptr); + CHECK(*service_url == livekit_url); } } @@ -361,8 +374,12 @@ SCENARIO("/.well-known/matrix/client carries rtc_foci only when a service is con auto const* entry = std::get_if(&(*foci)[0U].storage()); REQUIRE(entry != nullptr); CHECK(entry->size() == 2U); - CHECK(*string_member(*entry, "type") == "livekit"); - CHECK(*string_member(*entry, "livekit_service_url") == livekit_url); + auto const* type = string_member(*entry, "type"); + REQUIRE(type != nullptr); + CHECK(*type == "livekit"); + auto const* service_url = string_member(*entry, "livekit_service_url"); + REQUIRE(service_url != nullptr); + CHECK(*service_url == livekit_url); } } } From d7730e85d5d601c1d3099dea00b3cd1453d302cc Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:22:56 +0000 Subject: [PATCH 28/34] Report a room's state or state_after as present whatever type its value has has_part asked for an object, so a null or array member read as absent and require_state_after_only passed a response that still carried it. Co-Authored-By: Claude Sonnet 5.5 --- tests/support/SyncStateAfterSupport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/SyncStateAfterSupport.hpp b/tests/support/SyncStateAfterSupport.hpp index 518e045a2..0cb562f4c 100644 --- a/tests/support/SyncStateAfterSupport.hpp +++ b/tests/support/SyncStateAfterSupport.hpp @@ -173,7 +173,7 @@ struct Synced final // Whether `room` has the member `part`, whatever it holds. [[nodiscard]] inline auto has_part(Object const& room, std::string_view part) -> bool { - return object_member_as_object(room, part) != nullptr; + return object_member(room, part) != nullptr; } // The events of `room.`, which must be present. From a110e5fc59ff8e5f17d20dd6bb223797d2005265 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:27:42 +0000 Subject: [PATCH 29/34] Put the config parse in the WHEN of the RTC and TURN URI unit scenarios The scenarios parsed in GIVEN or ran the action inside THEN, so the action was not a When phase as docs/testing-standards.md asks. The assertions are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- tests/unit/test_config_rtc.cpp | 75 +++++-- tests/unit/test_config_turn_uris.cpp | 319 ++++++++++++++++----------- 2 files changed, 246 insertions(+), 148 deletions(-) diff --git a/tests/unit/test_config_rtc.cpp b/tests/unit/test_config_rtc.cpp index 0329a8cbf..dccb8374f 100644 --- a/tests/unit/test_config_rtc.cpp +++ b/tests/unit/test_config_rtc.cpp @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -45,24 +46,30 @@ SCENARIO("server.rtc.livekit.service_url accepts an https URL and defaults to no { GIVEN("a configuration that does not mention the key") { - auto const parsed = merovingian::config::parse_key_value_config(""); - - THEN("no transport is configured and nothing is reported") + WHEN("it is parsed") { - CHECK(parsed.config.server().rtc.livekit_service_url.empty()); - CHECK_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + auto const parsed = merovingian::config::parse_key_value_config(""); + + THEN("no transport is configured and nothing is reported") + { + CHECK(parsed.config.server().rtc.livekit_service_url.empty()); + CHECK_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + } } } GIVEN("a valid https URL, with a port and a path") { - auto const parsed = parse_url("https://livekit-jwt.example.org:8443/lk-jwt-service"); - - THEN("it is kept exactly as written") + WHEN("it is parsed") { - CHECK_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); - CHECK(parsed.config.server().rtc.livekit_service_url == - "https://livekit-jwt.example.org:8443/lk-jwt-service"); + auto const parsed = parse_url("https://livekit-jwt.example.org:8443/lk-jwt-service"); + + THEN("it is kept exactly as written") + { + CHECK_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); + CHECK(parsed.config.server().rtc.livekit_service_url == + "https://livekit-jwt.example.org:8443/lk-jwt-service"); + } } } } @@ -71,16 +78,27 @@ SCENARIO("server.rtc.livekit.service_url accepts a URL a client can append to", { GIVEN("URLs that /sfu/get or /get_token can be appended to") { - THEN("each is accepted: a bare host, a port, an IPv6 literal, a path with no trailing slash") + auto const good_urls = std::vector{"https://livekit-jwt.example.org", + "https://livekit-jwt.example.org:8443", "https://[2001:db8::1]", + "https://[2001:db8::1]:8443/jwt", "https://example.org/a/b"}; + + WHEN("each is parsed") { - for (auto const* good : - {"https://livekit-jwt.example.org", "https://livekit-jwt.example.org:8443", "https://[2001:db8::1]", - "https://[2001:db8::1]:8443/jwt", "https://example.org/a/b"}) + auto parsed = std::vector{}; + for (auto const& good : good_urls) { - INFO(good); - auto const parsed = parse_url(good); - REQUIRE_FALSE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); - REQUIRE(parsed.config.server().rtc.livekit_service_url == good); + parsed.push_back(parse_url(good)); + } + + THEN("each is accepted: a bare host, a port, an IPv6 literal, a path with no trailing slash") + { + REQUIRE(parsed.size() == good_urls.size()); + for (auto index = std::size_t{0U}; index < good_urls.size(); ++index) + { + INFO(good_urls[index]); + REQUIRE_FALSE(has_finding(parsed[index].findings, "server.rtc.livekit.service_url")); + REQUIRE(parsed[index].config.server().rtc.livekit_service_url == good_urls[index]); + } } } } @@ -114,14 +132,23 @@ SCENARIO("server.rtc.livekit.service_url refuses anything but a plain https URL" long_url, }; - THEN("each is refused with a finding on the key, and is never repaired") + WHEN("each is parsed") { + auto parsed = std::vector{}; for (auto const& url : bad) { - INFO(url.substr(0U, 60U)); - auto const parsed = parse_url(url); - REQUIRE(has_finding(parsed.findings, "server.rtc.livekit.service_url")); - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + parsed.push_back(parse_url(url)); + } + + THEN("each is refused with a finding on the key, and is never repaired") + { + REQUIRE(parsed.size() == bad.size()); + for (auto index = std::size_t{0U}; index < bad.size(); ++index) + { + INFO(bad[index].substr(0U, 60U)); + REQUIRE(has_finding(parsed[index].findings, "server.rtc.livekit.service_url")); + REQUIRE_FALSE(merovingian::config::is_valid(parsed[index].config)); + } } } } diff --git a/tests/unit/test_config_turn_uris.cpp b/tests/unit/test_config_turn_uris.cpp index 444a74646..8c5b965bf 100644 --- a/tests/unit/test_config_turn_uris.cpp +++ b/tests/unit/test_config_turn_uris.cpp @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -40,6 +41,30 @@ namespace "\nserver.turn.username=u\nserver.turn.password=p\n"); } +// Each entry parsed as the whole list, after `prefix`, in order. +[[nodiscard]] auto parse_each(std::vector const& entries, std::string const& prefix = {}) + -> std::vector +{ + auto results = std::vector{}; + for (auto const& entry : entries) + { + results.push_back(parse_uris(prefix + entry)); + } + return results; +} + +// `count` distinct turn: URIs, comma-separated. +[[nodiscard]] auto numbered_uris(int count) -> std::string +{ + auto list = std::string{}; + for (int i = 0; i < count; ++i) + { + list += (i == 0 ? "" : ","); + list += "turn:t" + std::to_string(i) + ".example.org"; + } + return list; +} + } // namespace SCENARIO("server.turn.uris lists several TURN URIs in order", "[config][turn][turn-uris]") @@ -101,67 +126,87 @@ SCENARIO("server.turn.uris accepts the RFC 7065 forms and refuses everything els { GIVEN("entries that follow the TURN URI grammar") { - THEN("each is accepted") + auto const good_entries = std::vector{"turn:turn.example.org", + "turn:turn.example.org:3478", + "turn:turn.example.org:3478?transport=udp", + "turns:turn.example.org:5349?transport=tcp", + "turn:192.0.2.1:3478", + "turn:[2001:db8::1]:3478", + "turn:[::1]", + "turn:[::ffff:192.0.2.1]:3478?transport=tcp", + "turn:[2001:db8::1]", + "turn:a-b.example.org:65535", + "turn:turn.example.org:1?transport=tcp"}; + + WHEN("each is parsed as the whole list") { - for (auto const* good : - {"turn:turn.example.org", "turn:turn.example.org:3478", "turn:turn.example.org:3478?transport=udp", - "turns:turn.example.org:5349?transport=tcp", "turn:192.0.2.1:3478", "turn:[2001:db8::1]:3478", - "turn:[::1]", "turn:[::ffff:192.0.2.1]:3478?transport=tcp", "turn:[2001:db8::1]", - "turn:a-b.example.org:65535", "turn:turn.example.org:1?transport=tcp"}) + auto const parsed = parse_each(good_entries); + + THEN("each is accepted") { - INFO(good); - auto const parsed = parse_uris(good); - REQUIRE(merovingian::config::is_valid(parsed.config)); - REQUIRE_FALSE(has_finding(parsed.findings, "server.turn.uris")); + REQUIRE(parsed.size() == good_entries.size()); + for (auto index = std::size_t{0U}; index < good_entries.size(); ++index) + { + INFO(good_entries[index]); + REQUIRE(merovingian::config::is_valid(parsed[index].config)); + REQUIRE_FALSE(has_finding(parsed[index].findings, "server.turn.uris")); + } } } } GIVEN("entries with no host, or that leave the grammar") { - THEN("each is refused") + auto const bad_entries = std::vector{"turn:?transport=udp", + "turn::3478", + "turn:@x", + "turns:/x", + "turn:turn.example.org:", + "turn:turn.example.org:0", + "turn:turn.example.org:65536", + "turn:turn.example.org:+3478", + "turn:turn.example.org:34a8", + "turn:turn.example.org:3478?transport=sctp", + "turn:turn.example.org:3478?transport=udp&x=1", + "turn:turn.example.org:3478?x=1", + "turn:turn.example.org:3478?transport=", + "turn:turn.example.org:3478?transport=udp?x", + "turn:turn.example.org.", + "turn:.example.org", + "turn:-example.org", + "turn:example.org-", + "turn:u@turn.example.org", + "turn:turn.example.org/path", + "turn:turn.example.org#frag", + "turn:turn.example.org:3478#frag", + "turn:turn_example.org", + "turn:[2001:db8::1", + "turn:2001:db8::1]:3478", + "turn:[]:3478", + "turn:[2001:db8::g]:3478", + "turn:[2001:db8::1]x", + "turn:[2001:db8::1]:", + "turn:[example.org]", + "turn:[:]", + "turn:[::::]", + "turn:[12345::1]", + "turn:[]", + "turn:[::1", + "turn:[fe80::1%eth0]"}; + + WHEN("each is parsed after a valid entry") { - for (auto const* bad : {"turn:?transport=udp", - "turn::3478", - "turn:@x", - "turns:/x", - "turn:turn.example.org:", - "turn:turn.example.org:0", - "turn:turn.example.org:65536", - "turn:turn.example.org:+3478", - "turn:turn.example.org:34a8", - "turn:turn.example.org:3478?transport=sctp", - "turn:turn.example.org:3478?transport=udp&x=1", - "turn:turn.example.org:3478?x=1", - "turn:turn.example.org:3478?transport=", - "turn:turn.example.org:3478?transport=udp?x", - "turn:turn.example.org.", - "turn:.example.org", - "turn:-example.org", - "turn:example.org-", - "turn:u@turn.example.org", - "turn:turn.example.org/path", - "turn:turn.example.org#frag", - "turn:turn.example.org:3478#frag", - "turn:turn_example.org", - "turn:[2001:db8::1", - "turn:2001:db8::1]:3478", - "turn:[]:3478", - "turn:[2001:db8::g]:3478", - "turn:[2001:db8::1]x", - "turn:[2001:db8::1]:", - "turn:[example.org]", - "turn:[:]", - "turn:[::::]", - "turn:[12345::1]", - "turn:[]", - "turn:[::1", - "turn:[fe80::1%eth0]"}) + auto const parsed = parse_each(bad_entries, "turn:ok.example.org,"); + + THEN("each is refused") { - INFO(bad); - auto const parsed = parse_uris(std::string{"turn:ok.example.org,"} + bad); - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); - REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + REQUIRE(parsed.size() == bad_entries.size()); + for (auto index = std::size_t{0U}; index < bad_entries.size(); ++index) + { + INFO(bad_entries[index]); + REQUIRE_FALSE(merovingian::config::is_valid(parsed[index].config)); + REQUIRE(has_finding(parsed[index].findings, "server.turn.uris")); + } } } } @@ -171,38 +216,47 @@ SCENARIO("server.turn.uris uses the config's ordinary comma-separated list synta { GIVEN("a list with an empty entry between two URIs") { - auto const parsed = parse_uris("turn:a.example,,turn:b.example"); - - THEN("the empty entry is ignored") + WHEN("it is parsed") { - REQUIRE(merovingian::config::is_valid(parsed.config)); - REQUIRE(parsed.config.server().turn.uris.size() == 2U); - CHECK(parsed.config.server().turn.uris[0U] == "turn:a.example"); - CHECK(parsed.config.server().turn.uris[1U] == "turn:b.example"); + auto const parsed = parse_uris("turn:a.example,,turn:b.example"); + + THEN("the empty entry is ignored") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + REQUIRE(parsed.config.server().turn.uris.size() == 2U); + CHECK(parsed.config.server().turn.uris[0U] == "turn:a.example"); + CHECK(parsed.config.server().turn.uris[1U] == "turn:b.example"); + } } } GIVEN("a list with a trailing comma and whitespace around entries") { - auto const parsed = parse_uris(" turn:a.example ,\tturn:b.example , "); - - THEN("whitespace around an entry and the empty last entry are ignored") + WHEN("it is parsed") { - REQUIRE(merovingian::config::is_valid(parsed.config)); - REQUIRE(parsed.config.server().turn.uris.size() == 2U); - CHECK(parsed.config.server().turn.uris[0U] == "turn:a.example"); - CHECK(parsed.config.server().turn.uris[1U] == "turn:b.example"); + auto const parsed = parse_uris(" turn:a.example ,\tturn:b.example , "); + + THEN("whitespace around an entry and the empty last entry are ignored") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + REQUIRE(parsed.config.server().turn.uris.size() == 2U); + CHECK(parsed.config.server().turn.uris[0U] == "turn:a.example"); + CHECK(parsed.config.server().turn.uris[1U] == "turn:b.example"); + } } } GIVEN("a value of only a comma") { - auto const parsed = merovingian::config::parse_key_value_config("server.turn.uris=,\n"); - - THEN("the list is empty and TURN is not configured") + WHEN("it is parsed") { - REQUIRE(merovingian::config::is_valid(parsed.config)); - CHECK(parsed.config.server().turn.uris.empty()); + auto const parsed = merovingian::config::parse_key_value_config("server.turn.uris=,\n"); + + THEN("the list is empty and TURN is not configured") + { + REQUIRE(merovingian::config::is_valid(parsed.config)); + CHECK(parsed.config.server().turn.uris.empty()); + } } } } @@ -211,45 +265,58 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t { GIVEN("both server.turn.server and server.turn.uris") { - auto const parsed = merovingian::config::parse_key_value_config( - "server.turn.server=turn:a.example.org\nserver.turn.uris=turn:b.example.org\n" - "server.turn.username=u\nserver.turn.password=p\n"); - - THEN("the configuration is refused naming server.turn.uris") + WHEN("the configuration is parsed") { - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); - REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + auto const parsed = merovingian::config::parse_key_value_config( + "server.turn.server=turn:a.example.org\nserver.turn.uris=turn:b.example.org\n" + "server.turn.username=u\nserver.turn.password=p\n"); + + THEN("the configuration is refused naming server.turn.uris") + { + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } } } GIVEN("an entry with a scheme other than turn: or turns:") { - THEN("each is refused") + auto const bad_entries = std::vector{"https://t.example.org", "stun:t.example.org", + "TURN:t.example.org", "turn.example.org:3478", "turn:"}; + + WHEN("each is parsed after a valid entry") { - for (auto const* bad : {"https://t.example.org", "stun:t.example.org", "TURN:t.example.org", - "turn.example.org:3478", "turn:"}) + auto const parsed = parse_each(bad_entries, "turn:ok.example.org,"); + + THEN("each is refused") { - INFO(bad); - auto const parsed = parse_uris(std::string{"turn:ok.example.org,"} + bad); - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); - REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + REQUIRE(parsed.size() == bad_entries.size()); + for (auto index = std::size_t{0U}; index < bad_entries.size(); ++index) + { + INFO(bad_entries[index]); + REQUIRE_FALSE(merovingian::config::is_valid(parsed[index].config)); + REQUIRE(has_finding(parsed[index].findings, "server.turn.uris")); + } } } } GIVEN("an entry holding whitespace or a control character") { - auto const inner_space = parse_uris("turn:a b.example.org"); - auto const tab = parse_uris("turn:a\tb.example.org"); - auto const bell = parse_uris("turn:a\x07.example.org"); - auto const high = parse_uris("turn:caf\xC3\xA9.example.org"); - - THEN("each is refused") + WHEN("each is parsed") { - for (auto const* parsed : {&inner_space, &tab, &bell, &high}) + auto const inner_space = parse_uris("turn:a b.example.org"); + auto const tab = parse_uris("turn:a\tb.example.org"); + auto const bell = parse_uris("turn:a\x07.example.org"); + auto const high = parse_uris("turn:caf\xC3\xA9.example.org"); + + THEN("each is refused") { - REQUIRE_FALSE(merovingian::config::is_valid(parsed->config)); - REQUIRE(has_finding(parsed->findings, "server.turn.uris")); + for (auto const* parsed : {&inner_space, &tab, &bell, &high}) + { + REQUIRE_FALSE(merovingian::config::is_valid(parsed->config)); + REQUIRE(has_finding(parsed->findings, "server.turn.uris")); + } } } } @@ -259,57 +326,61 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t auto const longest = "turn:" + std::string(507U, 'a'); auto const too_long = "turn:" + std::string(508U, 'a'); - THEN("512 bytes is accepted and 513 is refused") + WHEN("an entry of 512 bytes and one of 513 bytes are parsed") { - REQUIRE(merovingian::config::is_valid(parse_uris(longest).config)); - auto const parsed = parse_uris(too_long); - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); - REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + auto const accepted = parse_uris(longest); + auto const refused = parse_uris(too_long); + + THEN("512 bytes is accepted and 513 is refused") + { + REQUIRE(merovingian::config::is_valid(accepted.config)); + REQUIRE_FALSE(merovingian::config::is_valid(refused.config)); + REQUIRE(has_finding(refused.findings, "server.turn.uris")); + } } } GIVEN("more than 16 entries") { - auto list = std::string{}; - auto const build = [&list](int count) { - list.clear(); - for (int i = 0; i < count; ++i) + WHEN("a list of 16 and a list of 17 are parsed") + { + auto const accepted = parse_uris(numbered_uris(16)); + auto const refused = parse_uris(numbered_uris(17)); + + THEN("16 are accepted and 17 are refused") { - list += (i == 0 ? "" : ","); - list += "turn:t" + std::to_string(i) + ".example.org"; + REQUIRE(merovingian::config::is_valid(accepted.config)); + REQUIRE_FALSE(merovingian::config::is_valid(refused.config)); + REQUIRE(has_finding(refused.findings, "server.turn.uris")); } - }; - - THEN("16 are accepted and 17 are refused") - { - build(16); - REQUIRE(merovingian::config::is_valid(parse_uris(list).config)); - build(17); - auto const parsed = parse_uris(list); - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); - REQUIRE(has_finding(parsed.findings, "server.turn.uris")); } } GIVEN("the same entry twice") { - auto const parsed = parse_uris("turn:t.example.org,turns:t.example.org,turn:t.example.org"); - - THEN("the configuration is refused") + WHEN("the list is parsed") { - REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); - REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + auto const parsed = parse_uris("turn:t.example.org,turns:t.example.org,turn:t.example.org"); + + THEN("the configuration is refused") + { + REQUIRE_FALSE(merovingian::config::is_valid(parsed.config)); + REQUIRE(has_finding(parsed.findings, "server.turn.uris")); + } } } GIVEN("server.turn.uris set but credentials missing") { - auto const parsed = merovingian::config::parse_key_value_config("server.turn.uris=turn:t.example.org\n"); - - THEN("the missing username and password are reported, as for server.turn.server") + WHEN("the configuration is parsed") { - REQUIRE(has_finding(parsed.findings, "server.turn.username")); - REQUIRE(has_finding(parsed.findings, "server.turn.password")); + auto const parsed = merovingian::config::parse_key_value_config("server.turn.uris=turn:t.example.org\n"); + + THEN("the missing username and password are reported, as for server.turn.server") + { + REQUIRE(has_finding(parsed.findings, "server.turn.username")); + REQUIRE(has_finding(parsed.findings, "server.turn.password")); + } } } } From 2c399036936487af4b0086c30c602d46224e6bdc Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:27:43 +0000 Subject: [PATCH 30/34] Annotate the state_after conformance file as the other conformance files are Add the notice that a failure means the implementation is fixed before an assertion is weakened, and a Spec MUST line on the presence and omission checks. Co-Authored-By: Claude Sonnet 5.5 --- .../test_sync_state_after_conformance.cpp | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/conformance/test_sync_state_after_conformance.cpp b/tests/conformance/test_sync_state_after_conformance.cpp index 7b6f1538b..aa535882c 100644 --- a/tests/conformance/test_sync_state_after_conformance.cpp +++ b/tests/conformance/test_sync_state_after_conformance.cpp @@ -1,6 +1,15 @@ // SPDX-FileCopyrightText: 2026 James Chapman // SPDX-License-Identifier: GPL-3.0-or-later // +// +-------------------------------------------------------------------------+ +// | !! IMPORTANT - FOR HUMANS AND LLMs ALIKE !! | +// | | +// | Every REQUIRE here encodes a MUST or SHOULD of the Matrix v1.19 spec | +// | (the scenario header quotes it). If one fails, fix the IMPLEMENTATION | +// | first. Do NOT weaken, comment out or remove an assertion, or change an | +// | expected value, without citing the spec section that changed. | +// +-------------------------------------------------------------------------+ +// // GET /_matrix/client/v3/sync `use_state_after` and the `state_after` section of joined and left rooms // (client-server-api.md, GET /sync, "Joined Room", "Left Room"; ADR-0818 to ADR-0820). #include "../support/JsonTestSupport.hpp" @@ -59,6 +68,7 @@ SCENARIO("An initial sync answers state_after or state as use_state_after asks", THEN("state_after is the whole state at the end of the timeline, and state is omitted") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); auto const after = events_of(joined, "state_after"); REQUIRE(topics_in(after) == std::vector{"second"}); @@ -73,6 +83,7 @@ SCENARIO("An initial sync answers state_after or state as use_state_after asks", THEN("state is the state at the start of the timeline, and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(joined); REQUIRE(topics_in(events_of(joined, "state")) == std::vector{"first"}); } @@ -84,6 +95,7 @@ SCENARIO("An initial sync answers state_after or state as use_state_after asks", THEN("it is as if it were false") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(joined); REQUIRE(topics_in(events_of(joined, "state")) == std::vector{"first"}); } @@ -120,6 +132,7 @@ SCENARIO("An incremental sync's state_after holds what changed up to the end of THEN("state_after is present and empty, and state is omitted") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); REQUIRE(events_of(joined, "state_after").empty()); } @@ -132,7 +145,9 @@ SCENARIO("An incremental sync's state_after holds what changed up to the end of THEN("state is present and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(required_room(off, "join", room)); + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(required_room(absent, "join", room)); } } @@ -146,6 +161,7 @@ SCENARIO("An incremental sync's state_after holds what changed up to the end of THEN("state_after repeats the change, as the client takes no state from the timeline, and nothing else") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); REQUIRE(topics_in(events_of(joined, "timeline")) == std::vector{"second"}); auto const after = events_of(joined, "state_after"); @@ -178,6 +194,7 @@ SCENARIO("An incremental sync's state_after holds what changed up to the end of auto const* timeline = object_member_as_object(joined, "timeline"); REQUIRE(timeline != nullptr); REQUIRE(events_of(joined, "timeline").size() == 1U); + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); auto const after = events_of(joined, "state_after"); REQUIRE(event_types(after) == Keys{"m.room.topic"}); @@ -209,6 +226,7 @@ SCENARIO("An incremental sync's state_after holds what changed up to the end of THEN("state_after is present and does not hold it") { auto const& joined = required_room(synced, "join", room); + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); REQUIRE(events_of(joined, "state_after").empty()); } @@ -290,6 +308,7 @@ SCENARIO("full_state=true returns the whole state in the section use_state_after THEN("state_after holds all state, the timeline's change included, and state is omitted") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); auto const after = events_of(joined, "state_after"); REQUIRE(event_types(after).contains("m.room.create")); @@ -304,6 +323,7 @@ SCENARIO("full_state=true returns the whole state in the section use_state_after THEN("state holds all state up to the start of the timeline, and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(joined); auto const state = events_of(joined, "state"); REQUIRE(event_types(state).contains("m.room.create")); @@ -319,6 +339,7 @@ SCENARIO("full_state=true returns the whole state in the section use_state_after THEN("the room is still reported, with all state") { auto const& joined = required_room(synced, "join", room); + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); REQUIRE(event_types(events_of(joined, "state_after")).contains("m.room.create")); } @@ -361,6 +382,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state_after is the state up to his leave, the leave included, and state is omitted") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); auto const after = events_of(left, "state_after"); REQUIRE(topics_in(after) == std::vector{"changed before the leave"}); @@ -376,7 +398,9 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state is present and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(required_room(absent, "leave", room)); + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(required_room(off, "leave", room)); } } @@ -387,6 +411,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state_after holds what changed up to his leave and the leave, though the timeline carries it") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); REQUIRE(content_of(events_of(left, "timeline"), "m.room.member", user_id("dave"), "membership") == "leave"); @@ -402,6 +427,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state is present and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(required_room(synced, "leave", room)); } } @@ -413,6 +439,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state_after is present and state is omitted") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); } } @@ -423,6 +450,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state is present and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(required_room(synced, "leave", room)); } } @@ -448,6 +476,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state_after holds his ban and nothing of what happened after he left") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); auto const after = events_of(left, "state_after"); REQUIRE(topics_in(after).empty()); @@ -487,6 +516,7 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", THEN("state_after holds only her own leave, as she never could read the room") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); auto const after = events_of(left, "state_after"); REQUIRE(event_types(after) == Keys{"m.room.member"}); @@ -529,6 +559,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien THEN("state_after keeps other state and only the sender's and her own membership") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); auto const after = events_of(joined, "state_after"); REQUIRE(topics_in(after) == std::vector{"first"}); @@ -542,6 +573,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien THEN("state holds those members, and state_after is omitted") { + // Spec MUST: with use_state_after false or absent the room carries state and omits state_after. require_state_only(joined); REQUIRE(member_keys(events_of(joined, "state")) == Keys{user_id("alice"), user_id("bob")}); } @@ -559,6 +591,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(member_keys(events_of(required_room(second, "join", room), "state_after")) == Keys{user_id("carol")}); auto const after = events_of(required_room(third, "join", room), "state_after"); + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(required_room(third, "join", room)); REQUIRE(member_keys(after).empty()); } @@ -614,6 +647,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien THEN("state_after holds her own membership and all other state") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(joined); auto const after = events_of(joined, "state_after"); REQUIRE(event_types(after).contains("m.room.create")); @@ -644,6 +678,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien THEN("state_after holds his leave and the sender's membership") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); auto const after = events_of(left, "state_after"); REQUIRE(member_keys(after) == Keys{user_id("bob"), user_id("dave")}); @@ -659,6 +694,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien THEN("state_after keeps other state and holds only his own membership among members") { + // Spec MUST: with use_state_after=true the room carries state_after and omits state. require_state_after_only(left); auto const after = events_of(left, "state_after"); REQUIRE(event_types(after).contains("m.room.create")); From 503984549e12dda768ed9ef1c6c40f554db5a153 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:59:32 +0000 Subject: [PATCH 31/34] Prove the 1.0.3 sync token stays incremental and its stream token epoch survives migration The fixture scenarios asserted only a 200 for the old next_batch, but a token from another epoch is served as an initial sync with a 200 too. They now require the answer to omit the manifest's already-delivered events, which an unfiltered initial sync does contain, and the returned next_batch to carry the old token's epoch, decoded with decode_stream_token. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 3 +++ .../test_database_upgrade_flow.cpp | 23 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d71a659cb..62912fd9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0820). * The sync row of `docs/todos/capability-gaps.md` no longer says a limited sync gives no gap state, and tracks the `state_after` limit after a state fork (ADR-0819). +* Tests: the upgrade fixture scenarios now prove the 1.0.3 sync token stays incremental and its stream token epoch + survives migration; scratch files are uniquely named and their writes checked; the RTC and TURN URI tests fail one + scenario, not the binary, on a missing field. ## Release notes diff --git a/tests/integration/test_database_upgrade_flow.cpp b/tests/integration/test_database_upgrade_flow.cpp index 1c32fbaa6..7379a1e7e 100644 --- a/tests/integration/test_database_upgrade_flow.cpp +++ b/tests/integration/test_database_upgrade_flow.cpp @@ -38,6 +38,7 @@ #include "merovingian/homeserver/ClientServerRuntime.hpp" #include "merovingian/homeserver/ClientServerStartResult.hpp" #include "merovingian/homeserver/StoredRoomVersion.hpp" +#include "merovingian/sync/StreamToken.hpp" #include #include @@ -643,6 +644,7 @@ struct FixtureView final Response media{}; Response device_keys{}; Response sync_since_old_token{}; + Response initial_sync{}; Response replay_old_transaction{}; Response send_into_old_room{}; Response read_new_event{}; @@ -675,6 +677,8 @@ struct FixtureView final R"({"device_keys":{")" + expected.alice.user_id + R"(":[]}})"); view.sync_since_old_token = request(runtime, "GET", "/_matrix/client/v3/sync?timeout=0&since=" + expected.next_batch, alice); + // The control: a sync with no token, which must replay the room's timeline. + view.initial_sync = request(runtime, "GET", "/_matrix/client/v3/sync?timeout=0", alice); // The old release sent the first fixture message as `upgrade-txn-1` from this // device, so repeating it must replay that event, not create one. view.replay_old_transaction = request(runtime, "PUT", private_room + "/send/m.room.message/upgrade-txn-1", alice, @@ -727,8 +731,25 @@ auto require_fixture_served(FixtureView const& view, FixtureExpectations const& REQUIRE(sha256_hex(view.media.body) == expected.media_sha256); REQUIRE(view.device_keys.status == 200U); REQUIRE(view.device_keys.body.find(expected.device_ed25519_key) != std::string::npos); - // A client that synced against the old release carries on from its token. + // A client that synced against the old release carries on from its token. A token whose epoch is not the + // store's is served as an initial sync, also with a 200 (src/sync/AGENTS.md), so the status alone would pass a + // migration that replaced the stream token epoch. Prove the answer is incremental and the epoch survived. REQUIRE(view.sync_since_old_token.status == 200U); + REQUIRE(view.initial_sync.status == 200U); + for (auto const& message : expected.messages) + { + INFO("message " << message.event_id); + // The control replays what the old release had already delivered; the answer to the old token must not. + REQUIRE(view.initial_sync.body.find(message.event_id) != std::string::npos); + REQUIRE(view.sync_since_old_token.body.find(message.event_id) == std::string::npos); + } + auto const old_token = merovingian::sync::decode_stream_token(expected.next_batch); + REQUIRE(old_token.has_value()); + REQUIRE(old_token->epoch != 0U); + auto const new_token = + merovingian::sync::decode_stream_token(json_string_field(view.sync_since_old_token.body, "next_batch")); + REQUIRE(new_token.has_value()); + REQUIRE(new_token->epoch == old_token->epoch); // The old release's transaction IDs still deduplicate: a retried send replays its event. REQUIRE(view.replay_old_transaction.status == 200U); REQUIRE(json_string_field(view.replay_old_transaction.body, "event_id") == expected.messages.front().event_id); From 1fde7d587e8b02ab99d66058b49fd57985896399 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 01:59:43 +0000 Subject: [PATCH 32/34] Fit the 1.1.1 test bug-fix bullet to the changelog format's two lines Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62912fd9e..1c5925ae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,9 +30,8 @@ Changes before 1.0.0 are in [CHANGELOG-pre-release.md](CHANGELOG-pre-release.md) `state_after`, not a delta that hid the create event, power levels and join rules (ADR-0820). * The sync row of `docs/todos/capability-gaps.md` no longer says a limited sync gives no gap state, and tracks the `state_after` limit after a state fork (ADR-0819). -* Tests: the upgrade fixture scenarios now prove the 1.0.3 sync token stays incremental and its stream token epoch - survives migration; scratch files are uniquely named and their writes checked; the RTC and TURN URI tests fail one - scenario, not the binary, on a missing field. +* Tests: the upgrade fixture scenarios prove the 1.0.3 sync token stays incremental and its epoch survives + migration; scratch files are uniquely named; a missing JSON field fails one scenario, not the binary. ## Release notes From cec01d83d6efc274f1fc9d29a1f826f8770beadd Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 04:36:39 +0000 Subject: [PATCH 33/34] Explain what the long state_after, TURN URI, RTC and anchor scenarios prove and why their steps exist Co-Authored-By: Claude Sonnet 5.5 --- .../test_sync_state_after_conformance.cpp | 23 +++++++++++++++++++ .../test_sync_state_after_flow.cpp | 19 +++++++++++++++ .../integration/test_voip_discovery_flow.cpp | 10 ++++++++ tests/unit/test_config_turn_uris.cpp | 17 ++++++++++++++ tests/unit/test_sync_state_anchor.cpp | 10 ++++++++ 5 files changed, 79 insertions(+) diff --git a/tests/conformance/test_sync_state_after_conformance.cpp b/tests/conformance/test_sync_state_after_conformance.cpp index aa535882c..41e5d9af9 100644 --- a/tests/conformance/test_sync_state_after_conformance.cpp +++ b/tests/conformance/test_sync_state_after_conformance.cpp @@ -535,8 +535,14 @@ SCENARIO("A left room answers state_after or state as use_state_after asks", // response." Lazy-loading room members: "servers MUST include the syncing user's own membership event ... when the full // state of rooms is requested". Redundancy follows ADR-0450; a timeline event is not state here, so ADR-0819 forces // every change. +// +// The scenario has four GIVENs: a joined room seen through lazy-loading (initial, acknowledged and lost-response +// incrementals, a display-name change, a kick, a limited gap, `full_state`), a departed member's own view, a user who +// joins, speaks and leaves inside one window, and a user banned after a later joiner arrived. SCENARIO("Lazy-loading selects the members of state_after", "[conformance][client-server][sync][state-after]") { + // Bob and carol have both spoken, but a one-event timeline holds only bob's message, so a lazy-loaded answer must + // name bob and leave carol out until she is in a timeline. GIVEN("a room of alice, bob and carol, in which bob and carol have spoken") { auto started = merovingian::homeserver::start_client_server(state_after_config()); @@ -578,6 +584,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(member_keys(events_of(joined, "state")) == Keys{user_id("alice"), user_id("bob")}); } } + // The tracker (ADR-0450) records that the client was sent carol's membership once the next request proves it + // arrived, so the later sync repeats nothing it was told. WHEN("carol speaks in two incremental syncs, each acknowledging the last") { auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); @@ -596,6 +604,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(member_keys(after).empty()); } } + // Two events about bob in one window must not produce two member events: state_after is the state at the end + // of the timeline, so only the later one is sent. WHEN("bob speaks and then changes his display name in the same window") { auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); @@ -610,6 +620,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(content_of(after, "m.room.member", user_id("bob"), "displayname") == "Bobby"); } } + // A timeline event is not state to a `use_state_after` client, so lazy-loading cannot drop a change merely + // because its subject is not a sender (ADR-0819 forces every change). WHEN("the timeline carries a membership change about someone who did not speak") { auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); @@ -622,6 +634,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(content_of(after, "m.room.member", user_id("carol"), "membership") == "leave"); } } + // With a one-event timeline limit the leave falls before the timeline's start; the client cannot read it from + // the timeline, so state_after must carry it, while unchanged state (the topic) stays out. WHEN("a membership change falls in the gap of a limited sync") { auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); @@ -637,6 +651,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(topics_in(after).empty()); } } + // The spec requires the syncing user's own membership "when the full state of rooms is requested"; carol, who + // is not in the timeline, is not sent. WHEN("alice asks for the full state with lazy-loading and use_state_after=true") { auto const first = sync(rt, alice, k_lazy_one_event, {}, k_state_after); @@ -656,6 +672,7 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien } } } + // A left room takes the same lazy-loading rule: the sender (bob) and the user's own leave are sent, nobody else. GIVEN("a room of alice and dave that dave leaves after bob spoke") { auto started = merovingian::homeserver::start_client_server(state_after_config()); @@ -685,6 +702,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); } } + // An initial sync with include_leave reports the left room as new to the client, so the other state is sent but + // members are still limited to his own. WHEN("dave makes an initial sync with include_leave, lazy-loading and use_state_after=true") { constexpr auto lazy = std::string_view{ @@ -702,6 +721,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien } } } + // Dave is a sender in his own timeline, so the sender rule and the own-membership rule both name him: his member + // key must still appear once, as the leave, not as the earlier join the timeline also carries. GIVEN("a room dave joins, speaks in and leaves, all after his last sync") { auto started = merovingian::homeserver::start_client_server(state_after_config()); @@ -727,6 +748,8 @@ SCENARIO("Lazy-loading selects the members of state_after", "[conformance][clien } } } + // What a departed user may learn is capped at his leave (ADR-0820): the ban is his own membership and may be told, + // but erin's join after he left may not. GIVEN("a room dave left and synced, which erin then joined, and from which dave was then banned") { auto started = merovingian::homeserver::start_client_server(state_after_config()); diff --git a/tests/integration/test_sync_state_after_flow.cpp b/tests/integration/test_sync_state_after_flow.cpp index 8748cb20d..b229d100c 100644 --- a/tests/integration/test_sync_state_after_flow.cpp +++ b/tests/integration/test_sync_state_after_flow.cpp @@ -246,6 +246,10 @@ SCENARIO("full_state=true leaves a left room's state_after a delta", "[integrati // ADR-0820: a client that was not joined at `since` held nothing of the room, so a left room's `state_after` is the // whole state as of the leave, not a delta that would hide the create event, power levels and join rules. +// +// Each GIVEN is one way the join that the leave ended can fall relative to `since`: after it (a join and leave inside +// one window, the first GIVEN; a rejoin after an earlier departure, the second), before it (the third, which must stay +// a delta so the rule is not widened to every left room), or never at all (the fourth, an invite that was declined). SCENARIO("A room joined and left inside one sync window gives state_after its whole readable state", "[integration][sync][state-after][flow]") { @@ -257,11 +261,14 @@ SCENARIO("A room joined and left inside one sync window gives state_after its wh auto const alice = register_user(rt, "alice"); auto const dave = register_user(rt, "dave"); auto const room = create_room(rt, alice, "public_chat"); + // The sync that fixes `since` is taken before dave is in the room, so he holds none of its state. auto const before = sync(rt, dave, k_no_filter, {}, k_state_after); + // A topic set before his join is state he is entitled to read, and the one a delta would wrongly leave out. set_topic(rt, alice, room, "before dave"); room_post(rt, dave, room, "join"); say(rt, dave, room, "hello and goodbye"); room_post(rt, dave, room, "leave"); + // A change after the leave is beyond what he may read (ADR-0357), so it must not appear in his state_after. set_topic(rt, alice, room, "after dave left"); WHEN("dave syncs with use_state_after=true") @@ -274,14 +281,20 @@ SCENARIO("A room joined and left inside one sync window gives state_after its wh require_state_after_only(left); auto const after = events_of(left, "state_after"); auto const types = event_types(after); + // The create event, power levels and join rules never changed in the window, so only a whole-state + // answer, not a delta, sends them. REQUIRE(types.contains("m.room.create")); REQUIRE(types.contains("m.room.power_levels")); REQUIRE(types.contains("m.room.join_rules")); + // The topic is the one in force at his leave: "before dave", not the later "after dave left". REQUIRE(topics_in(after) == std::vector{"before dave"}); + // His own key is the current member event, the leave, and it appears once (ADR-0820). REQUIRE(content_of(after, "m.room.member", user_id("dave"), "membership") == "leave"); REQUIRE(member_keys(after).count(user_id("dave")) == 1U); } } + // ADR-0820: a whole-state answer treats the room as new to the client, so lazy-loading does not consult the + // tracker and does not send his membership a second time. WHEN("dave syncs with lazy-loading and use_state_after=true") { auto const synced = sync(rt, dave, k_lazy, before.next_batch, k_state_after); @@ -295,6 +308,8 @@ SCENARIO("A room joined and left inside one sync window gives state_after its wh } } } + // The join this leave ended began after `since` although dave had an earlier membership of the room: the rule + // reads the join's ordering, not whether the room was ever known to him. GIVEN("a room dave left and synced, then rejoined and left again before his next sync") { auto started = merovingian::homeserver::start_client_server(state_after_config()); @@ -322,6 +337,8 @@ SCENARIO("A room joined and left inside one sync window gives state_after its wh } } } + // The control: he was joined at `since`, so he holds the room's state and a delta is correct. Without this case a + // rule that made every left room whole would pass the scenarios above. GIVEN("a room dave was joined in at his last sync and left after its topic changed") { auto started = merovingian::homeserver::start_client_server(state_after_config()); @@ -347,6 +364,8 @@ SCENARIO("A room joined and left inside one sync window gives state_after its wh } } } + // An invitee was never joined, so there is no join to cap the readable state at: the answer is only her own + // membership (ADR-0820, a user who never joined), not the private room's state. GIVEN("a private room erin was invited to and declined inside one window") { auto started = merovingian::homeserver::start_client_server(state_after_config()); diff --git a/tests/integration/test_voip_discovery_flow.cpp b/tests/integration/test_voip_discovery_flow.cpp index 75d6454f5..5055442e2 100644 --- a/tests/integration/test_voip_discovery_flow.cpp +++ b/tests/integration/test_voip_discovery_flow.cpp @@ -187,6 +187,9 @@ constexpr auto livekit_url = "https://livekit-jwt.example.org/lk-jwt-service"; } // namespace +// The same configured LiveKit service URL is served on two routes that differ in path and in the name of the array: +// the stable route as `transports`, the MSC4143 unstable route as `rtc_transports`. A client written against either +// must find the same single entry. With no URL configured neither route fails: the two answer their own empty shape. SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", "[homeserver][rtc][client-server]") { GIVEN("a server with a LiveKit service URL") @@ -206,11 +209,13 @@ SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", auto const* transports = object_member_as_array(body, "transports"); REQUIRE(transports != nullptr); REQUIRE(transports->size() == 1U); + // The exact body pins the serialised form (keys in canonical order, nothing extra) ... CHECK( response.body == R"({"transports":[{"livekit_service_url":"https://livekit-jwt.example.org/lk-jwt-service","type":"livekit"}]})"); auto const* entry = std::get_if(&(*transports)[0U].storage()); REQUIRE(entry != nullptr); + // ... and the parsed checks name the two members a client reads, so a failure points at the member. CHECK(entry->size() == 2U); auto const* type = string_member(*entry, "type"); REQUIRE(type != nullptr); @@ -221,6 +226,8 @@ SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", } } + // The query string is ignored by the route, as on the other client-server routes, but must not stop the route + // from matching. WHEN("a client reads the unstable transports route, with a query string") { auto const response = @@ -248,6 +255,7 @@ SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", } } + // The stable route's own match with a query string; the unstable one is covered by the previous WHEN. WHEN("a client reads the stable route with a query string") { auto const response = get(started.runtime, "/_matrix/client/v1/rtc/transports?unused=1", token); @@ -259,6 +267,8 @@ SCENARIO("MatrixRTC transports are advertised on the stable and unstable paths", } } + // Without a URL the feature is off, but a client that asks is not told an error: it gets an empty answer in the + // shape of each route, so it can tell "none offered" from "route unknown". GIVEN("a server with no LiveKit service URL") { auto started = merovingian::homeserver::start_client_server(voip_config({})); diff --git a/tests/unit/test_config_turn_uris.cpp b/tests/unit/test_config_turn_uris.cpp index 8c5b965bf..0c8547e0f 100644 --- a/tests/unit/test_config_turn_uris.cpp +++ b/tests/unit/test_config_turn_uris.cpp @@ -194,6 +194,7 @@ SCENARIO("server.turn.uris accepts the RFC 7065 forms and refuses everything els "turn:[::1", "turn:[fe80::1%eth0]"}; + // The valid first entry shows that one good entry does not carry a bad one with it. WHEN("each is parsed after a valid entry") { auto const parsed = parse_each(bad_entries, "turn:ok.example.org,"); @@ -261,8 +262,13 @@ SCENARIO("server.turn.uris uses the config's ordinary comma-separated list synta } } +// ADR-0821: every entry of server.turn.uris is sent to every client, so a bad one is refused at parse time and never +// repaired (trimmed, lower-cased, dropped). Each GIVEN is one way a list can be unsafe or ambiguous: which key sets it, +// the entry's scheme, its characters, its length, the list's length, a repeat, and missing credentials. The valid +// shapes, and the host, port and transport grammar, are in the scenarios above. SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][turn][turn-uris]") { + // Both keys would leave it unclear which URI clients get, so the configuration is refused rather than one chosen. GIVEN("both server.turn.server and server.turn.uris") { WHEN("the configuration is parsed") @@ -281,6 +287,8 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t GIVEN("an entry with a scheme other than turn: or turns:") { + // Each entry fails the scheme rule alone: a foreign scheme, a STUN URI, an upper-case scheme (the grammar is + // case-sensitive, so it is not folded), a bare host:port, and a scheme with nothing after it. auto const bad_entries = std::vector{"https://t.example.org", "stun:t.example.org", "TURN:t.example.org", "turn.example.org:3478", "turn:"}; @@ -305,6 +313,9 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t { WHEN("each is parsed") { + // Whitespace inside an entry is not the separator the list trims around, so it is not repaired by splitting + // or trimming. The last entry is non-ASCII (UTF-8 for an e with an acute accent); only printable ASCII + // is accepted. auto const inner_space = parse_uris("turn:a b.example.org"); auto const tab = parse_uris("turn:a\tb.example.org"); auto const bell = parse_uris("turn:a\x07.example.org"); @@ -323,6 +334,7 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t GIVEN("an entry longer than 512 bytes") { + // "turn:" is five bytes, so 507 further bytes is exactly the 512-byte limit and 508 is the first byte over it. auto const longest = "turn:" + std::string(507U, 'a'); auto const too_long = "turn:" + std::string(508U, 'a'); @@ -342,6 +354,7 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t GIVEN("more than 16 entries") { + // The boundary is tested from both sides, so the limit is exactly 16 and not an off-by-one either way. WHEN("a list of 16 and a list of 17 are parsed") { auto const accepted = parse_uris(numbered_uris(16)); @@ -360,6 +373,8 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t { WHEN("the list is parsed") { + // The repeat is the third entry, not adjacent to the first, and the different scheme between them must not + // hide it from the check. auto const parsed = parse_uris("turn:t.example.org,turns:t.example.org,turn:t.example.org"); THEN("the configuration is refused") @@ -370,6 +385,8 @@ SCENARIO("server.turn.uris refuses a bad list and never repairs it", "[config][t } } + // The list turns TURN on by itself, so it must be held to the same credential rule as server.turn.server; it is + // not a way to publish a relay with no username or password. GIVEN("server.turn.uris set but credentials missing") { WHEN("the configuration is parsed") diff --git a/tests/unit/test_sync_state_anchor.cpp b/tests/unit/test_sync_state_anchor.cpp index 507baf690..d92b9759a 100644 --- a/tests/unit/test_sync_state_anchor.cpp +++ b/tests/unit/test_sync_state_anchor.cpp @@ -46,8 +46,13 @@ constexpr auto room = "!room:example.org"; } // namespace +// `state_anchor_event` finds the newest accepted event at or below a stream position, the event whose state a +// state_after delta is read from. It must be a bounded lookup, not a scan of every newer event, and it must never +// anchor on an event that was rejected or soft-failed. `state_anchor_rows_examined` counts the rows it looks at, so a +// scan shows up as a number, not as a slow test. SCENARIO("The state anchor of a room is found without visiting the rows above the position", "[sync][state-anchor]") { + // 3000 events lie above the position and only one below it, so a scan from the newest end would examine them all. GIVEN("a room with an old event and thousands of newer ones") { auto store = PersistentStore{}; @@ -71,6 +76,7 @@ SCENARIO("The state anchor of a room is found without visiting the rows above th } } + // The control: with no position the newest event is the anchor, so the bound above is the position's doing. WHEN("the anchor is read with no position") { auto const* const anchor = @@ -84,6 +90,8 @@ SCENARIO("The state anchor of a room is found without visiting the rows above th } } + // Two events can share one ordering; the choice among them must be stable and must not pick the rejected row, + // which was added last. GIVEN("rows sharing the newest ordering at the position, the newest added row being withheld") { auto store = PersistentStore{}; @@ -104,6 +112,7 @@ SCENARIO("The state anchor of a room is found without visiting the rows above th } } + // A soft-failed event is not part of the room's state, so the walk steps over it to the accepted row below. GIVEN("a withheld event at the position and an accepted one below it") { auto store = PersistentStore{}; @@ -122,6 +131,7 @@ SCENARIO("The state anchor of a room is found without visiting the rows above th } } + // A position before the room's first event anchors nothing, rather than the first event above the position. GIVEN("a room whose events all lie above the position") { auto store = PersistentStore{}; From a04ef0002309b08aa0fc669b6cd4935503db73d3 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 17 Nov 2026 04:48:21 +0000 Subject: [PATCH 34/34] Require the required state section to be an object holding an events array Presence alone passed a room whose state_after was null. The forbidden section is still judged by whether the key exists at all. Co-Authored-By: Claude Sonnet 5.5 --- tests/support/SyncStateAfterSupport.hpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/support/SyncStateAfterSupport.hpp b/tests/support/SyncStateAfterSupport.hpp index 0cb562f4c..d4f75b66b 100644 --- a/tests/support/SyncStateAfterSupport.hpp +++ b/tests/support/SyncStateAfterSupport.hpp @@ -249,15 +249,25 @@ using Keys = std::multiset; return topics; } +// The required section must exist, be an object and hold an `events` array, even an empty one (the spec's `State`). +inline auto require_state_section(Object const& room, std::string_view part) -> void +{ + REQUIRE(has_part(room, part)); + auto const* section = object_member_as_object(room, part); + REQUIRE(section != nullptr); + REQUIRE(object_member_as_array(*section, "events") != nullptr); +} + +// The forbidden section is judged by presence: the key must not exist, whatever its value. inline auto require_state_after_only(Object const& room) -> void { - REQUIRE(has_part(room, "state_after")); + require_state_section(room, "state_after"); REQUIRE_FALSE(has_part(room, "state")); } inline auto require_state_only(Object const& room) -> void { - REQUIRE(has_part(room, "state")); + require_state_section(room, "state"); REQUIRE_FALSE(has_part(room, "state_after")); }