Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP INDEX sync_event_account_expiry;
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- Activity events are filtered by account, recipient and expiry, often only
-- for their IDs. Index those columns so these queries can skip the rows and
-- their large padded payloads.
CREATE INDEX sync_event_account_expiry ON sync_event(account_id, recipient, expires_at, id);
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP INDEX sync_event_account_expiry;
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- Activity payloads are padded to tens of kilobytes. SQLite stores most of
-- such a row in overflow pages, with the columns after `payload` at the end,
-- so filtering events by `expires_at` read every stored payload, inside each
-- write transaction with activity and on every change poll. Index the columns
-- these queries filter and return instead.
CREATE INDEX sync_event_account_expiry ON sync_event(account_id, recipient, expires_at, id);
133 changes: 121 additions & 12 deletions src/database/sync_event.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,13 @@ pub async fn list(
conn: &mut DbConnection,
owner: &str,
target: &str,
since: &str,
now: i64,
) -> Result<Vec<SyncEvent>, DbError> {
// Filter broadcasts here, so already processed payloads are not loaded.
sync_event
.filter(account_id.eq(owner))
.filter(recipient.eq(target).or(recipient.eq("")))
.filter(recipient.eq(target).or(recipient.eq("").and(id.gt(since))))
.filter(expires_at.gt(now))
.order((created_at.desc(), id.desc()))
.limit(200)
Expand Down Expand Up @@ -196,10 +198,18 @@ mod tests {
.unwrap(),
SaveResult::Conflict
);
assert_eq!(list(&mut conn, "a", "phone", now).await.unwrap().len(), 1);
assert!(list(&mut conn, "b", "phone", now).await.unwrap().is_empty());
assert_eq!(
list(&mut conn, "a", "phone", "", now).await.unwrap().len(),
1
);
assert!(
list(&mut conn, "b", "phone", "", now)
.await
.unwrap()
.is_empty()
);
append(&mut conn, "a", "phone", "video", now).await.unwrap();
let events = list(&mut conn, "a", "phone", now).await.unwrap();
let events = list(&mut conn, "a", "phone", "", now).await.unwrap();
assert_eq!(events.len(), 2);
let message = events
.iter()
Expand All @@ -211,10 +221,16 @@ mod tests {
acknowledge(&mut conn, "a", "laptop", &message.id)
.await
.unwrap();
assert_eq!(list(&mut conn, "a", "phone", now).await.unwrap().len(), 2);
assert_eq!(list(&mut conn, "a", "laptop", now).await.unwrap().len(), 1);
assert_eq!(
list(&mut conn, "a", "phone", now + DAY)
list(&mut conn, "a", "phone", "", now).await.unwrap().len(),
2
);
assert_eq!(
list(&mut conn, "a", "laptop", "", now).await.unwrap().len(),
1
);
assert_eq!(
list(&mut conn, "a", "phone", "", now + DAY)
.await
.unwrap()
.len(),
Expand All @@ -223,21 +239,24 @@ mod tests {
acknowledge(&mut conn, "a", "phone", &message.id)
.await
.unwrap();
assert_eq!(list(&mut conn, "a", "phone", now).await.unwrap().len(), 1);
assert_eq!(
list(&mut conn, "a", "phone", "", now).await.unwrap().len(),
1
);
for offset in 1..105 {
append(&mut conn, "a", "", "activity", now + offset)
.await
.unwrap();
}
assert_eq!(
list(&mut conn, "a", "phone", now + 105)
list(&mut conn, "a", "phone", "", now + 105)
.await
.unwrap()
.len(),
100
);
assert!(
list(&mut conn, "a", "phone", now + 31 * DAY)
list(&mut conn, "a", "phone", "", now + 31 * DAY)
.await
.unwrap()
.is_empty()
Expand All @@ -260,7 +279,7 @@ mod tests {
.is_err()
);
assert_eq!(
list(&mut conn, "a", "device-0", now + 106)
list(&mut conn, "a", "device-0", "", now + 106)
.await
.unwrap()
.len(),
Expand All @@ -283,12 +302,102 @@ mod tests {
})
.await;
assert!(result.is_err());
assert!(list(&mut conn, "b", "phone", now).await.unwrap().is_empty());
assert!(
list(&mut conn, "b", "phone", "", now)
.await
.unwrap()
.is_empty()
);
assert!(
crate::database::encrypted_sync::get_full(&mut conn, "b", "settings")
.await
.unwrap()
.is_none()
);
}

/// Activity payloads are padded to tens of kilobytes, so SQLite stores
/// most of each event in a chain of overflow pages, followed by the
/// columns after `payload`. Every write with activity and every change
/// poll must not walk those chains: on a slow disk that held the write
/// lock for seconds per request.
#[actix_rt::test]
async fn writes_and_polls_do_not_read_stored_event_payloads() {
use crate::database::encrypted_sync;
use crate::models::EncryptedSync;
use diesel_async::SimpleAsyncConnection;

let (database, mut conn) = crate::database::TestFileDatabase::new().await;
conn.batch_execute("INSERT INTO account (id, name_hash) VALUES ('a', 'a');")
.await
.unwrap();
encrypted_sync::create(
&mut conn,
&EncryptedSync {
account_id: "a".to_owned(),
collection: "watchStats".to_owned(),
revision: 1,
payload: "stats".to_owned(),
},
)
.await
.unwrap();
let padded = "E".repeat(87_610);
let now = 10 * DAY;
// Below the retention limit, so the next append deletes none of them.
for index in 0..99 {
append(&mut conn, "a", "", &padded, now - 1000 + index)
.await
.unwrap();
}
let newest = list(&mut conn, "a", "phone", "", now).await.unwrap()[0]
.id
.clone();
conn.batch_execute("PRAGMA wal_checkpoint(TRUNCATE);")
.await
.unwrap();
drop(conn);

// Point every full overflow page of the stored payloads at a page
// past the end of the file, so walking a chain fails as corrupt.
let mut file = std::fs::read(&database.url).unwrap();
let mut cut = 0;
for page in file.chunks_mut(4096) {
if page[4..].iter().all(|&byte| byte == b'E') {
page[..4].copy_from_slice(&u32::MAX.to_be_bytes());
cut += 1;
}
}
assert!(cut >= 99 * 20, "only {cut} overflow pages found");
std::fs::write(&database.url, file).unwrap();

let mut conn = database.connect_without_waiting().await;
assert_eq!(
event_ids(&mut conn, "a", "phone", now).await.unwrap().len(),
99
);
assert!(
list(&mut conn, "a", "phone", &newest, now)
.await
.unwrap()
.is_empty()
);
assert_eq!(
encrypted_sync::save(
&mut conn,
"a",
"watchStats",
1,
"new stats",
usize::MAX,
Some(("activity", now)),
)
.await
.unwrap(),
SaveResult::Saved
);
let events = list(&mut conn, "a", "phone", &newest, now).await.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].payload, "activity");
}
}
55 changes: 54 additions & 1 deletion src/handlers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,21 @@ impl HandlerError {
Self::InternalDatabaseError
}
}

/// Every pooled connection may be waiting for the write lock, so a pool
/// timeout is as retryable as the busy error those connections will get.
pub fn pool(error: diesel_async::pooled_connection::bb8::RunError) -> Self {
match error {
diesel_async::pooled_connection::bb8::RunError::TimedOut => {
log::warn!("database busy: no pooled connection became available");
Self::DatabaseBusy
}
diesel_async::pooled_connection::bb8::RunError::User(error) => {
log::error!("database connection error: {error}");
Self::InternalDatabaseError
}
}
}
}

/// SQLite reports a write lock it could not get within its busy timeout, and
Expand Down Expand Up @@ -283,7 +298,7 @@ macro_rules! get_db_conn {
$pool
.get()
.await
.expect("Couldn't get db connection from the pool")
.map_err($crate::handlers::HandlerError::pool)?
};
}

Expand Down Expand Up @@ -312,4 +327,42 @@ mod tests {
.contains_key(actix_web::http::header::RETRY_AFTER)
);
}

/// Requests that cannot get a connection, because every pooled one waits
/// for the write lock, get a retryable response instead of a dropped
/// connection.
#[actix_rt::test]
async fn an_exhausted_pool_is_reported_as_retryable() {
use actix_web::{App, HttpResponse, get, test, web};
use diesel_async::pooled_connection::{AsyncDieselConnectionManager, bb8::Pool};

#[get("/")]
async fn handler(pool: crate::WebData) -> super::HandlerResult<HttpResponse> {
let _conn = crate::get_db_conn!(pool);
Ok(HttpResponse::Ok().finish())
}

let pool = Pool::builder()
.max_size(1)
.connection_timeout(std::time::Duration::from_millis(10))
.build(AsyncDieselConnectionManager::<crate::DbConnection>::new(
":memory:",
))
.await
.unwrap();
let _held = pool.get_owned().await.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(pool.clone()))
.service(handler),
)
.await;
let response = test::call_service(&app, test::TestRequest::get().to_request()).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(
response
.headers()
.contains_key(actix_web::http::header::RETRY_AFTER)
);
}
}
31 changes: 23 additions & 8 deletions src/handlers/encrypted_sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -457,13 +457,15 @@ async fn get_sync_events(
query: web::Query<EventsQuery>,
) -> HandlerResult<impl Responder> {
let mut conn = get_db_conn!(pool);
let events = sync_event::list(&mut conn, &account.id, &session.device_id, now_ms()?)
.await
.map_err(HandlerError::database)?;
let events: Vec<_> = events
.into_iter()
.filter(|event| !event.recipient.is_empty() || event.id > query.since)
.collect();
let events = sync_event::list(
&mut conn,
&account.id,
&session.device_id,
&query.since,
now_ms()?,
)
.await
.map_err(HandlerError::database)?;
Ok(HttpResponse::Ok()
.insert_header(("Cache-Control", "no-store"))
.json(events))
Expand Down Expand Up @@ -698,6 +700,7 @@ mod migration_tests {
&mut conn,
&account.id,
"",
"",
super::now_ms().unwrap(),
)
.await
Expand Down Expand Up @@ -1235,8 +1238,20 @@ mod change_tests {
.await
.unwrap()
.spawn_blocking(|conn| {
use diesel::migration::MigrationSource;

let migrations = MIGRATIONS.migrations().unwrap();
let changes = migrations
.iter()
.find(|migration| {
migration
.name()
.to_string()
.ends_with("_encrypted_sync_changes")
})
.unwrap();
Ok(conn
.revert_last_migration(MIGRATIONS)
.revert_migration(&**changes)
.map_err(|error| error.to_string()))
})
.await
Expand Down
24 changes: 7 additions & 17 deletions src/route_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -856,16 +856,9 @@ fn test_app_parts(
app
}

/// The canonical name of the channel the metadata tests submit.
const FEED_CHANNEL_ID: &str = "UC8-Th83bH_thdKZDJCrn88g";

async fn canonical_feed_channel_name() -> String {
ytrss::RssChannel::fetch_from_channel_id(FEED_CHANNEL_ID)
.await
.unwrap()
.name()
.to_owned()
}
/// The channel the metadata tests submit, and the canonical name its feed reports.
const FEED_CHANNEL_ID: &str = crate::validation::test::FEED_CHANNEL.0;
const CANONICAL_FEED_CHANNEL_NAME: &str = crate::validation::test::FEED_CHANNEL.1;

/// Regression test: PATCH inserted missing entries without charging the quota.
#[actix_web::test]
Expand Down Expand Up @@ -937,7 +930,7 @@ async fn history_keeps_stored_metadata_it_cannot_verify() {
"upload_date": 1,
"thumbnail_url": "https://i.ytimg.com/vi/forged/hqdefault.jpg",
"duration": 1,
"uploader": { "id": channel_id, "name": "Tonight Show", "verified": true }
"uploader": { "id": channel_id, "name": "Feed Channel", "verified": true }
},
"metadata": { "added_date": 1, "watched_state": "completed" }
}))
Expand All @@ -948,7 +941,7 @@ async fn history_keeps_stored_metadata_it_cannot_verify() {
assert_eq!(stored["video"]["duration"], 60);
assert_eq!(
stored["video"]["uploader"]["name"],
canonical_feed_channel_name().await
CANONICAL_FEED_CHANNEL_NAME
);
}

Expand All @@ -960,7 +953,7 @@ async fn bulk_subscriptions_store_canonical_names_for_duplicates() {
let token = seed_account(&pool).await;
let app = test::init_service(test_app_parts(pool)).await;

let channel = json!({ "id": FEED_CHANNEL_ID, "name": "Tonight Show", "verified": true });
let channel = json!({ "id": FEED_CHANNEL_ID, "name": "Feed Channel", "verified": true });
let request = test::TestRequest::put()
.uri("/v1/subscriptions/bulk")
.insert_header(("Authorization", token.clone()))
Expand All @@ -976,8 +969,5 @@ async fn bulk_subscriptions_store_canonical_names_for_duplicates() {
.insert_header(("Authorization", token))
.to_request();
let subscriptions: Value = test::call_and_read_body_json(&app, request).await;
assert_eq!(
subscriptions[0]["name"],
canonical_feed_channel_name().await
);
assert_eq!(subscriptions[0]["name"], CANONICAL_FEED_CHANNEL_NAME);
}
Loading
Loading