From 3f6e54a2e9391b5bf9d2e6927b1053c68d2dadcd Mon Sep 17 00:00:00 2001 From: D3SOX Date: Tue, 10 Nov 2026 16:48:45 +0100 Subject: [PATCH 1/2] fix(sync): stop small appends from holding the database lock for seconds Appending a change updated the revision columns of the row that also holds the collection's payload. SQLite reads, and when the record size changes rewrites, a row's whole overflow chain for any column update, so every append to a large collection cost as much as a full upload while holding the write lock. On slow storage this held the lock for over 30 seconds, and every request waiting for it failed. Appends now only insert into the change log. A collection's revision is its newest change above the row's revision, selected in the same statement as the row. Lock timeouts answer 503 with Retry-After instead of a bare "internal database error", and database errors are logged. --- src/database/encrypted_sync.rs | 259 ++++++++++++++++++++++++----- src/handlers.rs | 73 ++++++++ src/handlers/encrypted_sync.rs | 30 ++-- src/handlers/pairing.rs | 12 +- src/handlers/playlist_bookmarks.rs | 12 +- src/handlers/playlists.rs | 16 +- src/handlers/subscriptions.rs | 28 ++-- src/handlers/user.rs | 14 +- src/handlers/watch_history.rs | 22 +-- 9 files changed, 354 insertions(+), 112 deletions(-) diff --git a/src/database/encrypted_sync.rs b/src/database/encrypted_sync.rs index f4efabc..19346e5 100644 --- a/src/database/encrypted_sync.rs +++ b/src/database/encrypted_sync.rs @@ -35,6 +35,66 @@ fn base_or_revision(base: i64, revision: i64) -> i64 { if base == 0 { revision } else { base } } +/// The revision of an `encrypted_sync` row's collection. Appends leave the +/// row alone, because SQLite reads, and often rewrites, a row's whole payload +/// to update any of its columns, so its own revision is that of its payload. +/// Select it with the row, so PostgreSQL reads both at the same snapshot. +macro_rules! latest_revision { + () => { + "COALESCE((SELECT MAX(change.revision) FROM encrypted_sync_change change \ + WHERE change.account_id = encrypted_sync.account_id AND change.collection = encrypted_sync.collection \ + AND change.revision > encrypted_sync.revision), encrypted_sync.revision)" + }; +} + +#[derive(QueryableByName)] +struct CollectionRevision { + #[diesel(sql_type = diesel::sql_types::Text)] + name: String, + #[diesel(sql_type = diesel::sql_types::BigInt)] + revision: i64, +} + +#[derive(QueryableByName)] +struct CollectionHead { + /// The revision of the stored payload. + #[diesel(sql_type = diesel::sql_types::BigInt)] + stored_revision: i64, + #[diesel(sql_type = diesel::sql_types::BigInt)] + base: i64, + #[diesel(sql_type = diesel::sql_types::BigInt)] + revision: i64, +} + +#[derive(QueryableByName)] +struct FullCollection { + #[diesel(sql_type = diesel::sql_types::BigInt)] + stored_revision: i64, + #[diesel(sql_type = diesel::sql_types::BigInt)] + base: i64, + #[diesel(sql_type = diesel::sql_types::BigInt)] + revision: i64, + #[diesel(sql_type = diesel::sql_types::Text)] + payload: String, +} + +async fn get_head( + conn: &mut DbConnection, + owner_id: &str, + collection_name: &str, +) -> Result, DbError> { + diesel::sql_query(concat!( + "SELECT revision AS stored_revision, base_revision AS base, ", + latest_revision!(), + " AS revision FROM encrypted_sync WHERE account_id = $1 AND collection = $2", + )) + .bind::(owner_id) + .bind::(collection_name) + .get_result(conn) + .await + .optional() +} + /// A collection read for a client that may already hold an older revision. #[derive(Debug, PartialEq, Eq)] pub struct CollectionRead { @@ -78,14 +138,17 @@ async fn get_stored_collection( owner_id: &str, collection_name: &str, ) -> Result, DbError> { - diesel::sql_query( - "SELECT revision, CAST(OCTET_LENGTH(payload) AS BIGINT) AS bytes, \ + diesel::sql_query(concat!( + "SELECT ", + latest_revision!(), + " AS revision, \ + CAST(OCTET_LENGTH(payload) AS BIGINT) AS bytes, \ CAST((SELECT COALESCE(SUM(OCTET_LENGTH(change.payload)), 0) FROM encrypted_sync_change change \ WHERE change.account_id = encrypted_sync.account_id AND change.collection = encrypted_sync.collection) AS BIGINT) AS change_bytes, \ CAST((SELECT COUNT(*) FROM encrypted_sync_change change \ WHERE change.account_id = encrypted_sync.account_id AND change.collection = encrypted_sync.collection) AS BIGINT) AS changes \ FROM encrypted_sync WHERE account_id = ? AND collection = ?", - ) + )) .bind::(owner_id) .bind::(collection_name) .get_result(conn) @@ -99,14 +162,17 @@ async fn get_stored_collection( owner_id: &str, collection_name: &str, ) -> Result, DbError> { - diesel::sql_query( - "SELECT revision, OCTET_LENGTH(payload)::BIGINT AS bytes, \ + diesel::sql_query(concat!( + "SELECT ", + latest_revision!(), + " AS revision, \ + OCTET_LENGTH(payload)::BIGINT AS bytes, \ (SELECT COALESCE(SUM(OCTET_LENGTH(change.payload)), 0) FROM encrypted_sync_change change \ WHERE change.account_id = encrypted_sync.account_id AND change.collection = encrypted_sync.collection)::BIGINT AS change_bytes, \ (SELECT COUNT(*) FROM encrypted_sync_change change \ WHERE change.account_id = encrypted_sync.account_id AND change.collection = encrypted_sync.collection)::BIGINT AS changes \ FROM encrypted_sync WHERE account_id = $1 AND collection = $2", - ) + )) .bind::(owner_id) .bind::(collection_name) .get_result(conn) @@ -196,12 +262,17 @@ pub async fn revisions( conn: &mut DbConnection, owner: &str, ) -> Result, DbError> { - encrypted_sync - .filter(account_id.eq(owner)) - .order(collection.asc()) - .select((collection, encrypted_revision)) - .load(conn) - .await + Ok(diesel::sql_query(concat!( + "SELECT collection AS name, ", + latest_revision!(), + " AS revision FROM encrypted_sync WHERE account_id = $1 ORDER BY collection", + )) + .bind::(owner) + .load::(conn) + .await? + .into_iter() + .map(|row| (row.name, row.revision)) + .collect()) } pub async fn save( @@ -302,16 +373,15 @@ pub async fn read_since( for _ in 0..3 { let read = conn .transaction(async |conn| { - let Some((revision, base)) = encrypted_sync - .filter(account_id.eq(owner_id).and(collection.eq(collection_name))) - .select((encrypted_revision, base_revision)) - .first::<(i64, i64)>(conn) - .await - .optional()? + let Some(CollectionHead { + stored_revision, + base, + revision, + }) = get_head(conn, owner_id, collection_name).await? else { return Ok(Some(None)); }; - let base = base_or_revision(base, revision); + let base = base_or_revision(base, stored_revision); let incremental = since >= base && since <= revision; let payload = if incremental { None @@ -319,7 +389,7 @@ pub async fn read_since( Some( encrypted_sync .filter(account_id.eq(owner_id).and(collection.eq(collection_name))) - .filter(encrypted_revision.eq(revision)) + .filter(encrypted_revision.eq(stored_revision)) .select(encrypted_payload) .first::(conn) .await @@ -362,19 +432,29 @@ pub async fn read_since( } /// The revision, base revision and full payload, read together so a concurrent -/// append cannot pair an older payload with a newer revision. +/// upload cannot pair an older payload with a newer revision. pub async fn get_full( conn: &mut DbConnection, owner_id: &str, collection_name: &str, ) -> Result, DbError> { - Ok(encrypted_sync - .filter(account_id.eq(owner_id).and(collection.eq(collection_name))) - .select((encrypted_revision, base_revision, encrypted_payload)) - .first::<(i64, i64, String)>(conn) - .await - .optional()? - .map(|(revision, base, payload)| (revision, base_or_revision(base, revision), payload))) + Ok(diesel::sql_query(concat!( + "SELECT revision AS stored_revision, base_revision AS base, payload, ", + latest_revision!(), + " AS revision FROM encrypted_sync WHERE account_id = $1 AND collection = $2", + )) + .bind::(owner_id) + .bind::(collection_name) + .get_result::(conn) + .await + .optional()? + .map(|row| { + ( + row.revision, + base_or_revision(row.base, row.stored_revision), + row.payload, + ) + })) } /// Record that a session reads appended changes. Appends stay disabled while @@ -453,12 +533,9 @@ pub async fn append_change( // without the account lock, which takes the SQLite write lock. Stale // revisions still get their conflict below, and the checks repeat under // the lock. - let revision: Option = encrypted_sync - .filter(account_id.eq(owner_id).and(collection.eq(collection_name))) - .select(encrypted_revision) - .first(conn) - .await - .optional()?; + let revision = get_head(conn, owner_id, collection_name) + .await? + .map(|head| head.revision); if revision == Some(expected_revision) && !all_sessions_read_changes(conn, owner_id, now).await? { @@ -503,17 +580,8 @@ pub async fn append_change( )) .execute(conn) .await?; - let filter = encrypted_sync - .filter(account_id.eq(owner_id)) - .filter(collection.eq(collection_name)); - let base: i64 = filter.select(base_revision).first(conn).await?; - diesel::update(filter) - .set(( - encrypted_revision.eq(expected_revision + 1), - base_revision.eq(base_or_revision(base, expected_revision)), - )) - .execute(conn) - .await?; + // The collection row keeps the revision and base of its payload; see + // `latest_revision`. if let Some((payload, now)) = activity { crate::database::sync_event::append(conn, owner_id, "", payload, now).await?; @@ -859,6 +927,107 @@ mod tests { ); } + #[cfg(feature = "sqlite")] + #[actix_rt::test] + async fn appends_do_not_rewrite_the_stored_collection() { + use diesel_async::SimpleAsyncConnection; + + use crate::models::EncryptedSync; + + let (database, mut conn) = crate::database::TestFileDatabase::new().await; + conn.batch_execute( + "INSERT INTO account (id, name_hash, password_hash, oidc_sub) \ + VALUES ('account-a', 'hash-a', NULL, NULL);", + ) + .await + .unwrap(); + let payload = "x".repeat(1024 * 1024); + super::create( + &mut conn, + &EncryptedSync { + account_id: "account-a".to_owned(), + collection: "history".to_owned(), + revision: 1, + payload: payload.clone(), + }, + ) + .await + .unwrap(); + conn.batch_execute("PRAGMA wal_checkpoint(TRUNCATE);") + .await + .unwrap(); + + // SQLite stores revision 1 in zero bytes, so writing revision 2 into + // the collection row would rewrite its whole payload. + let result = super::append_change( + &mut conn, + "account-a", + "history", + 1, + "change", + usize::MAX, + None, + 1, + ) + .await + .unwrap(); + assert_eq!(result, super::SaveResult::Saved); + let written = std::fs::metadata(format!("{}-wal", database.url)) + .unwrap() + .len(); + assert!( + written < 64 * 1024, + "appending a change wrote {written} bytes" + ); + + assert_eq!( + super::revisions(&mut conn, "account-a").await.unwrap(), + [("history".to_owned(), 2)] + ); + let change = vec![(2, "change".to_owned())]; + assert_eq!( + super::read_since(&mut conn, "account-a", "history", 1) + .await + .unwrap(), + Some(super::CollectionRead { + revision: 2, + base_revision: 1, + payload: None, + changes: change.clone(), + }) + ); + assert_eq!( + super::read_since(&mut conn, "account-a", "history", 0) + .await + .unwrap(), + Some(super::CollectionRead { + revision: 2, + base_revision: 1, + payload: Some(payload.clone()), + changes: change, + }) + ); + assert_eq!( + super::get_full(&mut conn, "account-a", "history") + .await + .unwrap(), + Some((2, 1, payload)) + ); + let stale = super::append_change( + &mut conn, + "account-a", + "history", + 1, + "late", + usize::MAX, + None, + 1, + ) + .await + .unwrap(); + assert_eq!(stale, super::SaveResult::Conflict); + } + #[test] fn storage_quota_accounts_for_replaced_payload() { assert!(!exceeds_storage_quota(120, 20, 28, 128)); diff --git a/src/handlers.rs b/src/handlers.rs index 1801b00..925ff1f 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -55,6 +55,8 @@ pub enum HandlerError { NotInWatchHistory, #[error("internal database error")] InternalDatabaseError, + #[error("the database is busy; try again shortly")] + DatabaseBusy, #[error("internal database error: {0}")] InternalDatabaseErrorWithContext(String), #[error("provided metadata seems to be wrong")] @@ -118,6 +120,7 @@ impl ResponseError for HandlerError { Self::InvalidToken => StatusCode::UNAUTHORIZED, Self::NotInWatchHistory => StatusCode::NOT_FOUND, Self::InternalDatabaseError => StatusCode::INTERNAL_SERVER_ERROR, + Self::DatabaseBusy => StatusCode::SERVICE_UNAVAILABLE, Self::InternalDatabaseErrorWithContext(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::ValidationError => StatusCode::BAD_REQUEST, Self::ValidationErrorWithContext(_) => StatusCode::BAD_REQUEST, @@ -140,6 +143,46 @@ impl ResponseError for HandlerError { Self::OidcLoginDisabledForAccount => StatusCode::BAD_REQUEST, } } + + fn error_response(&self) -> actix_web::HttpResponse { + let mut response = actix_web::HttpResponse::build(self.status_code()); + if matches!(self, Self::DatabaseBusy) { + response.insert_header((actix_web::http::header::RETRY_AFTER, "30")); + } + response + .insert_header(actix_web::http::header::ContentType::plaintext()) + .body(self.to_string()) + } +} + +impl HandlerError { + /// Log a database error that clients only see as busy or internal. + pub fn database(error: diesel::result::Error) -> Self { + if is_database_busy(&error) { + log::warn!("database busy: {error}"); + Self::DatabaseBusy + } else { + log::error!("database error: {error}"); + Self::InternalDatabaseError + } + } +} + +/// SQLite reports a write lock it could not get within its busy timeout, and +/// PostgreSQL a transaction that conflicted with a concurrent one. +fn is_database_busy(error: &diesel::result::Error) -> bool { + use diesel::result::{DatabaseErrorKind, Error}; + + match error { + Error::DatabaseError(DatabaseErrorKind::SerializationFailure, _) => true, + Error::DatabaseError(_, info) => { + matches!( + info.message(), + "database is locked" | "database table is locked" + ) + } + _ => false, + } } pub type HandlerResult = Result; @@ -184,6 +227,9 @@ pub fn check_stored_rows(stored_rows: i64) -> HandlerResult<()> { impl From for HandlerError { fn from(error: diesel::result::Error) -> Self { + if is_database_busy(&error) { + return Self::database(error); + } Self::InternalDatabaseErrorWithContext(error.to_string()) } } @@ -240,3 +286,30 @@ macro_rules! get_db_conn { .expect("Couldn't get db connection from the pool") }; } + +#[cfg(all(test, feature = "sqlite"))] +mod tests { + use actix_web::{ResponseError, http::StatusCode}; + use diesel_async::SimpleAsyncConnection; + + use super::HandlerError; + + #[actix_rt::test] + async fn a_held_write_lock_is_reported_as_retryable() { + let (database, mut writer) = crate::database::TestFileDatabase::new().await; + writer.batch_execute("BEGIN IMMEDIATE;").await.unwrap(); + let mut blocked = database.connect_without_waiting().await; + let error = blocked + .batch_execute("DELETE FROM account;") + .await + .unwrap_err(); + + let response = HandlerError::database(error).error_response(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!( + response + .headers() + .contains_key(actix_web::http::header::RETRY_AFTER) + ); + } +} diff --git a/src/handlers/encrypted_sync.rs b/src/handlers/encrypted_sync.rs index c3c8c84..6462767 100644 --- a/src/handlers/encrypted_sync.rs +++ b/src/handlers/encrypted_sync.rs @@ -123,10 +123,10 @@ async fn get_encrypted_sync_manifest( let mut conn = get_db_conn!(pool); let documents = encrypted_sync::revisions(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let legacy_data = encrypted_sync::has_legacy_data(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let has_all_migrated_collections = LEGACY_ENCRYPTED_COLLECTIONS .iter() .all(|collection| documents.iter().any(|(name, _)| name == collection)); @@ -135,7 +135,7 @@ async fn get_encrypted_sync_manifest( let legacy_encrypted_data = !(has_all_migrated_collections && has_migrated_playback_speeds) && encrypted_sync::get_legacy_encrypted(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .is_some(); Ok(web::Json(EncryptedSyncManifest { @@ -160,7 +160,7 @@ async fn get_legacy_encrypted_sync( let mut conn = get_db_conn!(pool); let document = encrypted_sync::get_legacy_encrypted(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(web::Json(match document { Some(document) => EncryptedSyncCollectionResponse { @@ -201,7 +201,7 @@ async fn get_encrypted_sync_collection( // payload would let them revert those changes with their next upload. let document = encrypted_sync::get_full(&mut conn, &account.id, &collection) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; return Ok(web::Json(match document { Some((revision, base_revision, _)) if revision != base_revision => { return Err(HandlerError::EncryptedSyncChangesPending); @@ -222,11 +222,11 @@ async fn get_encrypted_sync_collection( if let Some(session) = session { encrypted_sync::mark_session_reads_changes(&mut conn, &session.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; } let read = encrypted_sync::read_since(&mut conn, &account.id, &collection, since) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(web::Json(match read { Some(read) => EncryptedSyncCollectionResponse { collection, @@ -298,7 +298,7 @@ async fn put_encrypted_sync_collection( form.activity.as_deref().map(|payload| (payload, now)), ) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { encrypted_sync::SaveResult::Saved => {} encrypted_sync::SaveResult::Conflict => { @@ -362,7 +362,7 @@ async fn append_encrypted_sync_change( // current may not read the collection before its first append. encrypted_sync::mark_session_reads_changes(&mut conn, &session.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; match encrypted_sync::append_change( &mut conn, &account.id, @@ -374,7 +374,7 @@ async fn append_encrypted_sync_change( now, ) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { encrypted_sync::SaveResult::Saved => {} encrypted_sync::SaveResult::Conflict => return Err(HandlerError::EncryptedSyncConflict), @@ -406,10 +406,10 @@ async fn change_cursor(pool: &WebData, owner: &str, device: &str) -> HandlerResu let mut conn = get_db_conn!(pool); let revisions = encrypted_sync::revisions(&mut conn, owner) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let events = sync_event::event_ids(&mut conn, owner, device, now_ms()?) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; use base64::Engine; use sha2::{Digest, Sha256}; let bytes = @@ -459,7 +459,7 @@ async fn get_sync_events( let mut conn = get_db_conn!(pool); let events = sync_event::list(&mut conn, &account.id, &session.device_id, now_ms()?) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let events: Vec<_> = events .into_iter() .filter(|event| !event.recipient.is_empty() || event.id > query.since) @@ -529,7 +529,7 @@ async fn send_device_request( diesel::result::DatabaseErrorKind::CheckViolation, _, ) => HandlerError::EncryptedSyncQuotaExceeded, - _ => HandlerError::InternalDatabaseError, + error => HandlerError::database(error), })?; notify(&account.id, Some(&form.recipient)); Ok(HttpResponse::NoContent().finish()) @@ -546,7 +546,7 @@ async fn acknowledge_device_request( let mut conn = get_db_conn!(pool); sync_event::acknowledge(&mut conn, &account.id, &session.device_id, &event) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; notify(&account.id, Some(&session.device_id)); Ok(HttpResponse::NoContent().finish()) } diff --git a/src/handlers/pairing.rs b/src/handlers/pairing.rs index 9d0c689..8e6251b 100644 --- a/src/handlers/pairing.rs +++ b/src/handlers/pairing.rs @@ -239,7 +239,7 @@ async fn create_pairing_session( let mut conn = get_db_conn!(pool); match pairing::create(&mut conn, &session, now) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { pairing::CreateResult::Created => Ok(HttpResponse::Created().json(response(session))), pairing::CreateResult::Duplicate => Err(HandlerError::PairingConflict), @@ -259,7 +259,7 @@ async fn get_pairing_session( let mut conn = get_db_conn!(pool); let session = pairing::get(&mut conn, &id, &token_hash, now_ms()?) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::PairingNotFound)?; Ok(web::Json(response(session))) } @@ -303,7 +303,7 @@ async fn claim_pairing_session( now_ms()?, ) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { pairing::ClaimResult::Claimed { pairing, @@ -341,7 +341,7 @@ async fn approve_pairing_session( now_ms()?, ) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; if !approved { return Err(HandlerError::PairingConflict); } @@ -360,7 +360,7 @@ async fn consume_pairing_session( let mut conn = get_db_conn!(pool); let session = pairing::consume(&mut conn, &id, &token_hash, now_ms()?) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::PairingNotFound)?; Ok(web::Json(PairingPayloadResponse { approving_device_id: session @@ -384,7 +384,7 @@ async fn cancel_pairing_session( let mut conn = get_db_conn!(pool); if !pairing::cancel(&mut conn, &id, &token_hash) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { return Err(HandlerError::PairingNotFound); } diff --git a/src/handlers/playlist_bookmarks.rs b/src/handlers/playlist_bookmarks.rs index 78299a8..ab9ee56 100644 --- a/src/handlers/playlist_bookmarks.rs +++ b/src/handlers/playlist_bookmarks.rs @@ -45,7 +45,7 @@ async fn get_playlist_bookmarks(account: Account, pool: WebData) -> HandlerResul let playlists = get_playlist_bookmarks_by_account_id(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let playlists: Vec<_> = playlists .iter() @@ -67,7 +67,7 @@ async fn get_playlist_bookmark( let Some((playlist, channel)) = get_playlist_bookmark_by_id(&mut conn, &playlist_id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? else { return Err(HandlerError::BookmarkNotExists); }; @@ -90,7 +90,7 @@ async fn create_playlist_bookmark( create_or_update_channel(&mut conn, &playlist.uploader) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; create_or_update_public_playlist( &mut conn, @@ -100,10 +100,10 @@ async fn create_playlist_bookmark( .into_public_playlist(&playlist.uploader.id), ) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; create_playlist_bookmark_by_playlist_id(&mut conn, &playlist.playlist.id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(HttpResponse::Ok().json(playlist)) } @@ -119,7 +119,7 @@ async fn delete_playlist_bookmark( delete_playlist_bookmark_by_playlist_id(&mut conn, &playlist_id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(HttpResponse::Ok()) } diff --git a/src/handlers/playlists.rs b/src/handlers/playlists.rs index 5d63415..1f8a11a 100644 --- a/src/handlers/playlists.rs +++ b/src/handlers/playlists.rs @@ -59,7 +59,7 @@ async fn get_playlist( let Some((playlist, videos)) = get_playlist_by_id_with_videos(&mut conn, &playlist_id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? else { return Err(HandlerError::PlaylistNotExists); }; @@ -85,7 +85,7 @@ async fn get_playlists(account: Account, pool: WebData) -> HandlerResult = vec![]; for playlist in &playlists { @@ -143,7 +143,7 @@ async fn get_owned_playlist_or_error( ) -> HandlerResult { get_playlist_by_id(conn, playlist_id, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::PlaylistNotExists) } @@ -229,7 +229,7 @@ async fn add_to_playlist( check_already_over_quota( count_playlist_videos(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; for (_, videos) in &groups { needs_validation.push(videos_requiring_validation(&mut conn, videos).await); @@ -247,7 +247,7 @@ async fn add_to_playlist( (&groups, &verified, &*playlist_id, &account.id); lock_account_quota(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; // Re-check ownership: the earlier check happened before the RSS // round-trips, which can take seconds, so the playlist may have been // deleted since. Without this the inserts fail as an opaque database @@ -258,12 +258,12 @@ async fn add_to_playlist( // store channel information first before storing video to ensure data integrity create_or_update_channel(conn, channel) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; for (video, verified) in videos.iter().zip(verified) { add_video_to_playlist(conn, playlist_id, account_id, &video.into(), *verified) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; } } @@ -273,7 +273,7 @@ async fn add_to_playlist( check_stored_rows( count_playlist_videos(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; Ok(()) }) diff --git a/src/handlers/subscriptions.rs b/src/handlers/subscriptions.rs index 6c0b483..9abd76b 100644 --- a/src/handlers/subscriptions.rs +++ b/src/handlers/subscriptions.rs @@ -66,7 +66,7 @@ async fn get_subscriptions(account: Account, pool: WebData) -> HandlerResult(async |conn| { lock_account_quota(conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; for channel in &channels { add_subscription_by_account_id(conn, channel, &account.id).await?; } @@ -112,7 +112,7 @@ async fn subscribe_bulk( check_stored_rows( count_subscriptions(conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; Ok(()) }) @@ -157,7 +157,7 @@ async fn subscribe( check_already_over_quota( count_subscriptions(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; !channels_requiring_validation(&mut conn, std::slice::from_ref(&channel)) .await @@ -175,12 +175,12 @@ async fn subscribe( let (channel, account_id) = (&channel, &account.id); lock_account_quota(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; add_subscription_by_account_id(conn, channel, account_id).await?; check_stored_rows( count_subscriptions(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; Ok(()) }) @@ -200,13 +200,13 @@ async fn unsubscribe( remove_subscription_by_account_id(&mut conn, &channel_id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; // now that the user no longer subscribed to the channel, the channel may also no // longer be part of any subscription groups, so we auto-wipe it from all groups remove_channel_from_all_subscription_groups(&mut conn, &channel_id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(HttpResponse::Ok()) } @@ -246,14 +246,14 @@ async fn get_subscription_group( let Some(group) = get_subscription_group_by_id(&mut conn, &subscription_group_id, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? else { return Err(HandlerError::SubscriptionGroupNotFound); }; let channels = get_subscription_group_channels_by_id(&mut conn, &subscription_group_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let extended_subscription_group = ExtendedSubscriptionGroup { group, channels }; Ok(HttpResponse::Ok().json(extended_subscription_group)) @@ -327,7 +327,7 @@ async fn verify_is_subscription_group_owner( ) -> HandlerResult<()> { if get_subscription_group_by_id(conn, subscription_group_id, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .is_none() { return Err(HandlerError::SubscriptionGroupNotFound); @@ -350,7 +350,7 @@ async fn add_to_subscription_group( let subscription = get_subscription_channel_by_account_id(&mut conn, &account.id, &channel_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; if subscription.is_none() { return Err(HandlerError::SubscribeBeforeChannelGroup); } @@ -360,7 +360,7 @@ async fn add_to_subscription_group( add_channel_to_subscription_group(&mut conn, &subscription_group_id, &channel_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(HttpResponse::Ok()) } diff --git a/src/handlers/user.rs b/src/handlers/user.rs index 5b71550..0f35e2a 100644 --- a/src/handlers/user.rs +++ b/src/handlers/user.rs @@ -200,7 +200,7 @@ async fn resolve_account_session( now, ) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::InvalidToken); } @@ -213,7 +213,7 @@ async fn resolve_account_session( .await .map_err(|error| match error { diesel::result::Error::NotFound => HandlerError::InvalidToken, - _ => HandlerError::InternalDatabaseError, + error => HandlerError::database(error), }) } @@ -667,7 +667,7 @@ pub(crate) async fn authenticate_session( let now = now_ms()?; let account = find_account_by_id(&mut conn, &claims.sub) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::AccountNotExists)?; let mut session = resolve_account_session(&mut conn, &account, &jwt, &claims, now).await?; if account_session::touch( @@ -678,7 +678,7 @@ pub(crate) async fn authenticate_session( now, ) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { session.last_active_at = now; } @@ -720,11 +720,11 @@ async fn authorize_request(req: &ServiceRequest) -> Result<(), actix_web::Error> if !exempt { let encrypted_sync_enabled = encrypted_sync::exists(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; if encrypted_sync_enabled { let migration_complete = !encrypted_sync::has_legacy_data(&mut conn, &account.id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; if migration_complete { return Err(HandlerError::EncryptedSyncRequired.into()); } @@ -1544,7 +1544,7 @@ async fn delete_oidc_account_callback( let account = find_account_by_id(&mut conn, &account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::AccountNotExists)?; // the provider must confirm the owner of the account that asked for deletion if account.oidc_sub.as_deref() != Some(oidc_sub.as_str()) { diff --git a/src/handlers/watch_history.rs b/src/handlers/watch_history.rs index edb0e43..cca2104 100644 --- a/src/handlers/watch_history.rs +++ b/src/handlers/watch_history.rs @@ -93,7 +93,7 @@ async fn prepare_watch_history_items( check_already_over_quota( count_watch_history(&mut conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; for videos in &grouped_videos { needs_validation.push(videos_requiring_validation(&mut conn, videos).await); @@ -133,17 +133,17 @@ async fn persist_watch_history_item( ) -> HandlerResult<()> { create_or_update_channel(conn, &item.video.uploader) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; let video = (&item.video).into(); if *verified { create_or_update_video(conn, &video).await } else { create_video_if_missing(conn, &video).await } - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; add_or_update_video_to_watch_history(conn, &item.metadata) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; Ok(()) } @@ -156,7 +156,7 @@ async fn persist_watch_history_items( conn.transaction::<_, HandlerError, _>(async |conn| { lock_account_quota(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; for item in items { persist_watch_history_item(conn, item).await?; } @@ -167,7 +167,7 @@ async fn persist_watch_history_items( check_stored_rows( count_watch_history(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; Ok(()) }) @@ -270,7 +270,7 @@ async fn get_from_watch_history( match get_watch_history_entry(&mut conn, &account.id, &video_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? { Some((metadata, video, channel)) => Ok(HttpResponse::Ok().json(ExtendedWatchHistoryItem { video: CreateVideo::from((&video, &channel)), @@ -295,7 +295,7 @@ async fn add_to_watch_history( let mut conn = get_db_conn!(pool); let (metadata, video, channel) = get_watch_history_entry(&mut conn, &account.id, &video_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)? + .map_err(HandlerError::database)? .ok_or(HandlerError::InternalDatabaseError)?; Ok(HttpResponse::Ok().json(ExtendedWatchHistoryItem { @@ -323,14 +323,14 @@ async fn update_watch_history_video_state( let (item, account_id) = (&watch_history_item, &watch_history_item.account_id); lock_account_quota(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; add_or_update_video_to_watch_history(conn, item) .await - .map_err(|_| HandlerError::InternalDatabaseError)?; + .map_err(HandlerError::database)?; check_stored_rows( count_watch_history(conn, account_id) .await - .map_err(|_| HandlerError::InternalDatabaseError)?, + .map_err(HandlerError::database)?, )?; Ok(()) }) From 85cf2d26614ff52a4bd7e0c0e91a3b77a5ac75d0 Mon Sep 17 00:00:00 2001 From: D3SOX Date: Tue, 10 Nov 2026 16:58:53 +0100 Subject: [PATCH 2/2] fix(sync): report lock timeouts on the remaining write endpoints as busy --- src/handlers/playlists.rs | 16 ++++------------ src/handlers/subscriptions.rs | 24 ++++++------------------ src/handlers/user.rs | 12 ++++-------- src/handlers/watch_history.rs | 12 +++--------- 4 files changed, 17 insertions(+), 47 deletions(-) diff --git a/src/handlers/playlists.rs b/src/handlers/playlists.rs index 1f8a11a..d53da45 100644 --- a/src/handlers/playlists.rs +++ b/src/handlers/playlists.rs @@ -129,9 +129,7 @@ async fn create_playlist( let extended_playlist = ExtendedPlaylist::from_playlist(&playlist, 0); Ok(HttpResponse::Created().json(extended_playlist)) } - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -175,9 +173,7 @@ async fn update_playlist( let extended_playlist = ExtendedPlaylist::from_playlist(&playlist, video_count as u64); Ok(HttpResponse::Ok().json(extended_playlist)) } - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -194,9 +190,7 @@ async fn delete_playlist( match delete_playlist_by_id(&mut conn, &playlist_id, &account.id).await { Ok(()) => Ok(HttpResponse::Ok().json(())), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -297,8 +291,6 @@ async fn remove_from_playlist( match remove_video_from_playlist(&mut conn, &playlist_id, &account.id, &video_id).await { Ok(()) => Ok(HttpResponse::Ok()), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } diff --git a/src/handlers/subscriptions.rs b/src/handlers/subscriptions.rs index 9abd76b..1e2f275 100644 --- a/src/handlers/subscriptions.rs +++ b/src/handlers/subscriptions.rs @@ -135,9 +135,7 @@ async fn get_subscription( Some(channel) => Ok(HttpResponse::Ok().json(channel)), None => Err(HandlerError::NotSubscribed), }, - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -229,9 +227,7 @@ async fn get_subscription_groups(account: Account, pool: WebData) -> HandlerResu Ok(HttpResponse::Ok().json(groups)) } - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -274,9 +270,7 @@ async fn create_subscription_group( match create_new_subscription_group(&mut conn, subscription_group).await { Ok(group) => Ok(HttpResponse::Ok().json(group)), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -297,9 +291,7 @@ async fn update_subscription_group( match update_existing_subscription_group(&mut conn, subscription_group).await { Ok(group) => Ok(HttpResponse::Ok().json(group)), Err(diesel::result::Error::NotFound) => Err(HandlerError::SubscriptionGroupNotFound), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -314,9 +306,7 @@ async fn delete_subscription_group( match delete_subscription_group_by_id(&mut conn, &subscription_group_id, &account.id).await { Ok(_) => Ok(HttpResponse::Ok()), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -381,8 +371,6 @@ async fn remove_from_subscription_group( .await { Ok(_) => Ok(HttpResponse::Ok()), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } diff --git a/src/handlers/user.rs b/src/handlers/user.rs index 0f35e2a..958c20b 100644 --- a/src/handlers/user.rs +++ b/src/handlers/user.rs @@ -349,7 +349,7 @@ async fn register_account( diesel::result::Error::DatabaseError(DatabaseErrorKind::UniqueViolation, _) => { HandlerError::AccountNameTaken } - _ => HandlerError::InternalDatabaseErrorWithContext(err.to_string()), + _ => HandlerError::from(err), })?; let jwt = match issue_account_session(&mut conn, &account, device_id).await { @@ -415,9 +415,7 @@ async fn delete_account( match delete_existing_account(&mut conn, &account.id).await { Ok(_) => Ok(HttpResponse::Ok()), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -1456,7 +1454,7 @@ async fn authenticate_oidc_account_callback( }; insert_new_account(&mut conn, &account) .await - .map_err(|err| HandlerError::InternalDatabaseErrorWithContext(err.to_string()))?; + .map_err(HandlerError::from)?; account }; @@ -1554,8 +1552,6 @@ async fn delete_oidc_account_callback( match delete_existing_account(&mut conn, &account.id).await { Ok(true) => Ok(Redirect::to(redirect_url)), Ok(false) => Err(HandlerError::AccountNotExists), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } diff --git a/src/handlers/watch_history.rs b/src/handlers/watch_history.rs index cca2104..e41794f 100644 --- a/src/handlers/watch_history.rs +++ b/src/handlers/watch_history.rs @@ -253,9 +253,7 @@ async fn get_watch_history( .collect::>(); Ok(HttpResponse::Ok().json(history)) } - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -346,9 +344,7 @@ async fn clear_watch_history(account: Account, pool: WebData) -> HandlerResult Ok(HttpResponse::Ok()), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } } @@ -363,8 +359,6 @@ async fn remove_from_watch_history( match remove_video_from_watch_history(&mut conn, &account.id, &video_id).await { Ok(()) => Ok(HttpResponse::Ok()), - Err(err) => Err(HandlerError::InternalDatabaseErrorWithContext( - err.to_string(), - )), + Err(err) => Err(HandlerError::from(err)), } }