diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a81db21a9..e032ba21a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -420,6 +420,14 @@ jobs: if: env.AW_RESEARCH_EDITION == 'true' run: python3 scripts/patch_research_edition_config.py aw-watcher-window/aw_watcher_window/config.py + # Watchers store buckets as `{client}_{hostname}`. That hostname is a + # participant identifier on named machines, and /api/0/export copies it + # into the map key, embedded id, and hostname field. Standard builds + # must not change; this patch is Research-only and fail-closed. + - name: Patch research edition export hostname sanitizer + if: env.AW_RESEARCH_EDITION == 'true' + run: python3 scripts/patch_research_edition_export.py + # The watcher rewrites `app` to a study category before storing, but # aw-webui categorises client-side with its own defaults and never sees # the watcher's map -- so without this the Categories panel reads @@ -698,6 +706,14 @@ jobs: if: env.AW_RESEARCH_EDITION == 'true' run: python3 scripts/patch_research_edition_config.py aw-watcher-window/aw_watcher_window/config.py + # Watchers store buckets as `{client}_{hostname}`. That hostname is a + # participant identifier on named machines, and /api/0/export copies it + # into the map key, embedded id, and hostname field. Standard builds + # must not change; this patch is Research-only and fail-closed. + - name: Patch research edition export hostname sanitizer + if: env.AW_RESEARCH_EDITION == 'true' + run: python3 scripts/patch_research_edition_export.py + # The watcher rewrites `app` to a study category before storing, but # aw-webui categorises client-side with its own defaults and never sees # the watcher's map -- so without this the Categories panel reads diff --git a/scripts/patch_research_edition_export.py b/scripts/patch_research_edition_export.py new file mode 100644 index 000000000..2f5cb35d8 --- /dev/null +++ b/scripts/patch_research_edition_export.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Patch aw-server-rust export endpoints for the Research Edition build. + +Run as part of the CI build for research edition: + + python3 scripts/patch_research_edition_export.py [repo-root] + +Standard builds never run this script, so `/api/0/export` stays byte-for-byte +unchanged outside Research Edition. The patch is fail-closed: missing markers +abort the build rather than shipping an unsanitized artifact. +""" +from __future__ import annotations + +import pathlib +import shutil +import sys + +MARKER = "RESEARCH_EDITION_EXPORT_SANITIZE" + +EXPORT_INSERT_NEEDLE = """ export.buckets.insert(bid, bucket); + } + + Ok(export.into()) +""" + +EXPORT_INSERT_REPLACEMENT = f""" export.buckets.insert(bid, bucket); + }} + + // {MARKER} + let export = match super::export_sanitize::sanitize_buckets_export(export) {{ + Ok(export) => export, + Err(err) => {{ + return Err(HttpErrorJson::new(rocket::http::Status::Conflict, err)) + }} + }}; + Ok(export.into()) +""" + +BUCKET_INSERT_NEEDLE = """ export.buckets.insert(bucket_id.into(), bucket); + + Ok(export.into()) +""" + +BUCKET_INSERT_REPLACEMENT = f""" export.buckets.insert(bucket_id.into(), bucket); + + // {MARKER} + let export = match super::export_sanitize::sanitize_buckets_export(export) {{ + Ok(export) => export, + Err(err) => return Err(HttpErrorJson::new(Status::Conflict, err)), + }}; + Ok(export.into()) +""" + +MOD_NEEDLE = "mod export;\n" +MOD_REPLACEMENT = "mod export;\nmod export_sanitize;\n" + + +def repo_root_from_args(argv: list[str]) -> pathlib.Path: + if len(argv) > 1: + return pathlib.Path(argv[1]).resolve() + return pathlib.Path.cwd().resolve() + + +def _replace_once(path: pathlib.Path, needle: str, replacement: str, already_ok: str) -> None: + text = path.read_text(encoding="utf-8") + if already_ok in text: + return + count = text.count(needle) + if count != 1: + raise ValueError( + f"{path}: expected exactly one export-sanitizer insertion point, found {count}" + ) + path.write_text(text.replace(needle, replacement, 1), encoding="utf-8") + + +def patch_tree(repo_root: pathlib.Path) -> None: + script_dir = pathlib.Path(__file__).resolve().parent + source = script_dir / "research_edition" / "export_sanitize.rs" + if not source.is_file(): + raise FileNotFoundError(f"missing sanitizer module: {source}") + + endpoints = ( + repo_root / "aw-server-rust" / "aw-server" / "src" / "endpoints" + ) + export_rs = endpoints / "export.rs" + bucket_rs = endpoints / "bucket.rs" + mod_rs = endpoints / "mod.rs" + dest = endpoints / "export_sanitize.rs" + + for required in (export_rs, bucket_rs, mod_rs): + if not required.is_file(): + raise FileNotFoundError(f"expected Rust export source at {required}") + + shutil.copyfile(source, dest) + + _replace_once(mod_rs, MOD_NEEDLE, MOD_REPLACEMENT, "mod export_sanitize;") + _replace_once(export_rs, EXPORT_INSERT_NEEDLE, EXPORT_INSERT_REPLACEMENT, MARKER) + _replace_once(bucket_rs, BUCKET_INSERT_NEEDLE, BUCKET_INSERT_REPLACEMENT, MARKER) + + +def main() -> None: + repo_root = repo_root_from_args(sys.argv) + try: + patch_tree(repo_root) + except (OSError, ValueError) as error: + print(f"Error: {error}", file=sys.stderr) + sys.exit(1) + print(f"Patched Research Edition export sanitizer under {repo_root / 'aw-server-rust'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/research_edition/export_sanitize.rs b/scripts/research_edition/export_sanitize.rs new file mode 100644 index 000000000..b16e5f249 --- /dev/null +++ b/scripts/research_edition/export_sanitize.rs @@ -0,0 +1,375 @@ +//! Research Edition `/api/0/export` sanitizer. +//! +//! Copied into `aw-server` only by `scripts/patch_research_edition_export.py` +//! when `AW_RESEARCH_EDITION=true`. Standard builds never see this module. +//! +//! Two jobs: +//! 1. Fail closed if currentwindow events still carry raw titles/URLs/app names +//! (an existing ActivityWatch database is not a study-safe profile). +//! 2. Rewrite each exported bucket's map key, embedded `id`, and `hostname` +//! so the real machine name never leaves the device. Colliding sanitized +//! IDs fail the export rather than silently merging two machines. + +use std::collections::HashMap; + +use aw_models::{Bucket, BucketsExport, TryVec}; +use serde_json::Value; + +pub const SANITIZED_HOSTNAME: &str = "research-participant"; + +/// Study categories plus the two "Excluded" spellings the watcher writes. +/// Keep in sync with `scripts/patch_research_edition_config.py`. +const STUDY_CATEGORIES: &[&str] = &[ + "Sensitive / Excluded", + "Music & Audio", + "Video Streaming", + "Games", + "Travel & Mobility", + "Search & Navigation", + "News & Current Affairs", + "Social Networking", + "Messaging", + "Email", + "AI Chatbots & Assistants", + "Work & Productivity", + "Education & Learning", + "Shopping - Goods", + "Shopping - Groceries & Food", + "Banking & Finance", + "Public Services", + "Excluded", + "excluded", +]; + +/// Browser app names the Research filter leaves in `app` while classifying the +/// title. Copied from `aw-watcher-window/aw_watcher_window/research_filter.py`. +const BROWSER_APPS: &[&str] = &[ + "chrome", + "google chrome", + "google chrome canary", + "google-chrome", + "google-chrome-beta", + "google-chrome-unstable", + "chromium", + "chromium-browser", + "brave browser", + "brave", + "brave-browser", + "firefox", + "firefox developer edition", + "firefox-esr", + "safari", + "edge", + "microsoft edge", + "microsoft-edge", + "microsoft-edge-beta", + "microsoft-edge-dev", + "opera", + "chrome.exe", + "brave.exe", + "firefox.exe", + "msedge.exe", + "opera.exe", +]; + +pub fn sanitize_buckets_export(export: BucketsExport) -> Result { + let mut checked = HashMap::new(); + for (key, mut bucket) in export.buckets { + reject_unfiltered(&mut bucket)?; + checked.insert(key, bucket); + } + rewrite_identities(checked) +} + +fn reject_unfiltered(bucket: &mut Bucket) -> Result<(), String> { + let Some(events) = bucket.events.take() else { + return Ok(()); + }; + let inner = events.take_inner(); + for event in &inner { + if let Some(reason) = unfiltered_reason(&bucket._type, &event.data) { + bucket.events = Some(TryVec::new(inner)); + return Err(format!( + "Research Edition export refused: bucket type '{}' client '{}' contains unfiltered event data ({reason}). \ + Research filtering only applies to newly captured events. Uninstall ActivityWatch, \ + delete the existing database, and install the Research Edition on a clean profile.", + bucket._type, bucket.client + )); + } + } + bucket.events = Some(TryVec::new(inner)); + Ok(()) +} + +fn unfiltered_reason(bucket_type: &str, data: &serde_json::Map) -> Option<&'static str> { + if data.contains_key("url") { + return Some("url field"); + } + if bucket_type != "currentwindow" { + return None; + } + if let Some(title) = data.get("title").and_then(Value::as_str) { + if !is_study_category(title) { + return Some("non-category window title"); + } + } + if let Some(app) = data.get("app").and_then(Value::as_str) { + if !is_allowed_window_app(app) { + return Some("non-category window app"); + } + } + None +} + +fn is_study_category(value: &str) -> bool { + let trimmed = value.trim(); + STUDY_CATEGORIES + .iter() + .any(|category| category.eq_ignore_ascii_case(trimmed)) +} + +fn is_allowed_window_app(app: &str) -> bool { + is_study_category(app) || BROWSER_APPS.iter().any(|name| name.eq_ignore_ascii_case(app.trim())) +} + +fn rewrite_identities(buckets: HashMap) -> Result { + let mut out: HashMap = HashMap::new(); + for (key, mut bucket) in buckets { + let original_id = if bucket.id.is_empty() { + key.clone() + } else { + bucket.id.clone() + }; + let sanitized_id = sanitize_id(&original_id, &bucket.hostname); + let sanitized_key = sanitize_id(&key, &bucket.hostname); + if sanitized_id != sanitized_key { + return Err( + "Research Edition export refused: bucket key and embedded id sanitize to different identities. \ + Start from a clean ActivityWatch profile." + .to_string(), + ); + } + if out.contains_key(&sanitized_key) { + return Err(format!( + "Research Edition export refused: two buckets map to the same sanitized identity '{sanitized_key}'. \ + This usually means data from more than one machine is in the same database. Start from a clean ActivityWatch profile." + )); + } + bucket.id = sanitized_key.clone(); + bucket.hostname = SANITIZED_HOSTNAME.to_string(); + out.insert(sanitized_key, bucket); + } + Ok(BucketsExport { buckets: out }) +} + +pub(crate) fn sanitize_id(id: &str, hostname: &str) -> String { + if hostname.is_empty() { + return id.to_string(); + } + if id == hostname { + return SANITIZED_HOSTNAME.to_string(); + } + let suffix = format!("_{hostname}"); + if let Some(prefix) = id.strip_suffix(suffix.as_str()) { + return format!("{prefix}_{SANITIZED_HOSTNAME}"); + } + if id.contains(hostname) { + return id.replace(hostname, SANITIZED_HOSTNAME); + } + id.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use aw_models::{BucketMetadata, Event, TryVec}; + use chrono::{Duration, TimeZone, Utc}; + use serde_json::{json, Map}; + + fn ts() -> chrono::DateTime { + Utc.with_ymd_and_hms(2026, 8, 24, 12, 0, 0).unwrap() + } + + fn event(data: serde_json::Value) -> Event { + Event { + id: None, + timestamp: ts(), + duration: Duration::seconds(1), + data: data.as_object().cloned().unwrap_or_else(Map::new), + } + } + + fn bucket(id: &str, hostname: &str, bucket_type: &str, client: &str, events: Vec) -> Bucket { + Bucket { + bid: None, + id: id.to_string(), + _type: bucket_type.to_string(), + client: client.to_string(), + hostname: hostname.to_string(), + created: None, + data: Map::new(), + metadata: BucketMetadata::default(), + events: Some(TryVec::new(events)), + last_updated: None, + } + } + + fn export_of(buckets: Vec) -> BucketsExport { + BucketsExport { + buckets: buckets + .into_iter() + .map(|b| (b.id.clone(), b)) + .collect(), + } + } + + #[test] + fn rewrites_key_id_and_hostname_together() { + let host = "Participant-Alice-MacBook"; + let original = export_of(vec![ + bucket( + &format!("aw-watcher-window_{host}"), + host, + "currentwindow", + "aw-watcher-window", + vec![event(json!({"app": "Excluded"}))], + ), + bucket( + &format!("aw-watcher-afk_{host}"), + host, + "afkstatus", + "aw-watcher-afk", + vec![event(json!({"status": "afk"}))], + ), + ]); + + let sanitized = sanitize_buckets_export(original).unwrap(); + let window_id = format!("aw-watcher-window_{SANITIZED_HOSTNAME}"); + let afk_id = format!("aw-watcher-afk_{SANITIZED_HOSTNAME}"); + + assert_eq!(sanitized.buckets.len(), 2); + let window = sanitized.buckets.get(&window_id).unwrap(); + assert_eq!(window.id, window_id); + assert_eq!(window.hostname, SANITIZED_HOSTNAME); + assert_eq!( + window.events.as_ref().unwrap().clone().take_inner()[0].data, + json!({"app": "Excluded"}).as_object().unwrap().clone() + ); + + let afk = sanitized.buckets.get(&afk_id).unwrap(); + assert_eq!(afk.id, afk_id); + assert_eq!(afk.hostname, SANITIZED_HOSTNAME); + assert_eq!( + afk.events.as_ref().unwrap().clone().take_inner()[0].data, + json!({"status": "afk"}).as_object().unwrap().clone() + ); + + let dump = serde_json::to_string(&sanitized).unwrap(); + assert!(!dump.contains(host), "real hostname must not appear in export JSON"); + assert_eq!(dump.matches(SANITIZED_HOSTNAME).count(), 6); // key + id + hostname, twice + } + + #[test] + fn collision_of_two_hostnames_fails_closed() { + let original = export_of(vec![ + bucket( + "aw-watcher-window_host-a", + "host-a", + "currentwindow", + "aw-watcher-window", + vec![event(json!({"app": "Excluded"}))], + ), + bucket( + "aw-watcher-window_host-b", + "host-b", + "currentwindow", + "aw-watcher-window", + vec![event(json!({"app": "Excluded"}))], + ), + ]); + + let err = match sanitize_buckets_export(original) { + Err(err) => err, + Ok(_) => panic!("expected hostname collision to fail closed"), + }; + assert!(err.contains("same sanitized identity")); + assert!(!err.contains("host-a")); + assert!(!err.contains("host-b")); + } + + #[test] + fn unfiltered_title_fails_closed() { + let original = export_of(vec![bucket( + "aw-watcher-window_host", + "host", + "currentwindow", + "aw-watcher-window", + vec![event(json!({"app": "Code", "title": "secret.rs"}))], + )]); + + let err = match sanitize_buckets_export(original) { + Err(err) => err, + Ok(_) => panic!("expected unfiltered title to fail closed"), + }; + assert!(err.contains("unfiltered")); + assert!(err.contains("clean profile")); + assert!(!err.contains("secret.rs")); + assert!(!err.contains("Code")); + } + + #[test] + fn unfiltered_url_fails_closed_even_outside_window_buckets() { + let original = export_of(vec![bucket( + "aw-watcher-web_host", + "host", + "web.tab.current", + "aw-watcher-web", + vec![event(json!({"url": "https://mail.example/inbox", "title": "Inbox"}))], + )]); + + let err = match sanitize_buckets_export(original) { + Err(err) => err, + Ok(_) => panic!("expected url field to fail closed"), + }; + assert!(err.contains("url field")); + assert!(!err.contains("mail.example")); + } + + #[test] + fn browser_event_with_classified_title_is_allowed() { + let original = export_of(vec![bucket( + "aw-watcher-window_host", + "host", + "currentwindow", + "aw-watcher-window", + vec![event(json!({"app": "Firefox", "title": "Work & Productivity"}))], + )]); + + let sanitized = sanitize_buckets_export(original).unwrap(); + let bucket = sanitized + .buckets + .get(&format!("aw-watcher-window_{SANITIZED_HOSTNAME}")) + .unwrap(); + assert_eq!( + bucket.events.as_ref().unwrap().clone().take_inner()[0].data, + json!({"app": "Firefox", "title": "Work & Productivity"}) + .as_object() + .unwrap() + .clone() + ); + } + + #[test] + fn sanitize_id_replaces_suffix_and_embedded_hostname() { + assert_eq!( + sanitize_id("aw-watcher-window_Participant-Alice-MacBook", "Participant-Alice-MacBook"), + format!("aw-watcher-window_{SANITIZED_HOSTNAME}") + ); + assert_eq!( + sanitize_id("Participant-Alice-MacBook", "Participant-Alice-MacBook"), + SANITIZED_HOSTNAME + ); + assert_eq!(sanitize_id("id1", "hostname"), "id1"); + assert_eq!(sanitize_id("keep_me", ""), "keep_me"); + } +} diff --git a/scripts/tests/test_patch_research_edition_export.py b/scripts/tests/test_patch_research_edition_export.py new file mode 100644 index 000000000..00d880c46 --- /dev/null +++ b/scripts/tests/test_patch_research_edition_export.py @@ -0,0 +1,120 @@ +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).parents[1] / "patch_research_edition_export.py" +SPEC = importlib.util.spec_from_file_location("patch_research_edition_export", SCRIPT) +assert SPEC and SPEC.loader +patcher = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(patcher) + +CONFIG_PATCHER_PATH = Path(__file__).parents[1] / "patch_research_edition_config.py" +CONFIG_SPEC = importlib.util.spec_from_file_location( + "patch_research_edition_config", CONFIG_PATCHER_PATH +) +assert CONFIG_SPEC and CONFIG_SPEC.loader +config_patcher = importlib.util.module_from_spec(CONFIG_SPEC) +CONFIG_SPEC.loader.exec_module(config_patcher) + + +def _write_tree(tmp_path: Path, export: str, bucket: str, mod: str) -> Path: + endpoints = tmp_path / "aw-server-rust" / "aw-server" / "src" / "endpoints" + endpoints.mkdir(parents=True) + (endpoints / "export.rs").write_text(export, encoding="utf-8") + (endpoints / "bucket.rs").write_text(bucket, encoding="utf-8") + (endpoints / "mod.rs").write_text(mod, encoding="utf-8") + return tmp_path + + +EXPORT_SRC = """use std::collections::HashMap; + +pub fn buckets_export() { + for (bid, mut bucket) in buckets.drain() { + export.buckets.insert(bid, bucket); + } + + Ok(export.into()) +} +""" + +BUCKET_SRC = """pub fn bucket_export() { + export.buckets.insert(bucket_id.into(), bucket); + + Ok(export.into()) +} +""" + +MOD_SRC = """mod util; +mod export; +mod hostcheck; +""" + + +def test_patch_inserts_module_and_both_call_sites(tmp_path: Path): + root = _write_tree(tmp_path, EXPORT_SRC, BUCKET_SRC, MOD_SRC) + + patcher.patch_tree(root) + + export = (root / "aw-server-rust/aw-server/src/endpoints/export.rs").read_text( + encoding="utf-8" + ) + bucket = (root / "aw-server-rust/aw-server/src/endpoints/bucket.rs").read_text( + encoding="utf-8" + ) + mod = (root / "aw-server-rust/aw-server/src/endpoints/mod.rs").read_text(encoding="utf-8") + copied = root / "aw-server-rust/aw-server/src/endpoints/export_sanitize.rs" + + assert copied.is_file() + assert "mod export_sanitize;" in mod + assert patcher.MARKER in export + assert patcher.MARKER in bucket + assert "sanitize_buckets_export" in export + assert "sanitize_buckets_export" in bucket + assert "Status::Conflict" in export + assert "Status::Conflict" in bucket + + +def test_patch_is_idempotent(tmp_path: Path): + root = _write_tree(tmp_path, EXPORT_SRC, BUCKET_SRC, MOD_SRC) + patcher.patch_tree(root) + first = (root / "aw-server-rust/aw-server/src/endpoints/export.rs").read_text( + encoding="utf-8" + ) + patcher.patch_tree(root) + second = (root / "aw-server-rust/aw-server/src/endpoints/export.rs").read_text( + encoding="utf-8" + ) + assert first == second + mod = (root / "aw-server-rust/aw-server/src/endpoints/mod.rs").read_text(encoding="utf-8") + assert mod.count("mod export_sanitize;") == 1 + + +def test_patch_fails_closed_without_export_marker(tmp_path: Path): + root = _write_tree(tmp_path, "fn buckets_export() {}\n", BUCKET_SRC, MOD_SRC) + with pytest.raises(ValueError, match="insertion point"): + patcher.patch_tree(root) + + +def test_live_tree_is_patchable_or_already_patched(): + root = Path(__file__).resolve().parents[2] + export = root / "aw-server-rust/aw-server/src/endpoints/export.rs" + bucket = root / "aw-server-rust/aw-server/src/endpoints/bucket.rs" + if not export.is_file() or not bucket.is_file(): + pytest.skip("aw-server-rust not checked out") + export_text = export.read_text(encoding="utf-8") + bucket_text = bucket.read_text(encoding="utf-8") + assert patcher.MARKER in export_text or export_text.count(patcher.EXPORT_INSERT_NEEDLE) == 1 + assert patcher.MARKER in bucket_text or bucket_text.count(patcher.BUCKET_INSERT_NEEDLE) == 1 + + +def test_sanitizer_allowlist_covers_config_categories(): + rust = ( + Path(__file__).parents[1] / "research_edition" / "export_sanitize.rs" + ).read_text(encoding="utf-8") + expected = {c for _, c in config_patcher.CATEGORY_MAP} | set( + config_patcher.APP_CATEGORY_MAP.values() + ) + expected.update({"Excluded", "excluded"}) + missing = [category for category in expected if f'"{category}"' not in rust] + assert missing == []