Add opt-out message deduplication; release 0.0.5

PRIVACY.md

@@ -10,7 +10,7 @@ The app does not read existing SMS history, send SMS, or reply to notifications.
 ## On your device
 
 The app stores your webhook URL, editable device code, confirmation preference, enabled sources,
-selected package names, pause state and a random installation ID in private preferences. These
+selected package names, duplicate-filter preference, pause state and a random installation ID in private preferences. These
 preferences do not have additional application-level encryption. A webhook URL may itself contain
 a secret, so treat it as sensitive. Android cloud backup and device transfer are disabled for app data.
 
@@ -21,8 +21,10 @@ Captured event bodies, including message text, notification titles and SMS sende
 a private SQLite outbox encrypted with AES-GCM and a key held in Android Keystore. Each queued
 request includes its original destination, encrypted authentication token and confirmation mode. Delivery metadata (event ID,
 source display name, type, timestamps, state, attempt count and HTTP result) is stored without
-additional application-level encryption. Notification duplicate detection stores hashes of keys
-and contents; these hashes are not a substitute for encryption against guesses of known content.
+additional application-level encryption. Message duplicate detection stores SHA-256 hashes of source package, original timestamp and text,
+including when filtering is disabled. These hashes remain after delivery or journal deletion until
+app data is cleared or the app is uninstalled. They are not a substitute for encryption against
+guesses of known content.
 
 The journal shows delivery metadata and does not display message contents. Confirmed events have
 their encrypted payload removed from the active database record; only recent delivery metadata is

README.md

@@ -51,6 +51,15 @@ forwards only the content the system exposes.
 
 ## Delivery and data
 
+**Skip duplicate messages** in **Sources** is on by default. SMS and notifications are
+compared by source package, exact original timestamp and exact text (including whitespace).
+Titles and SMS senders are not part of this key. SMS uses the system source `android`.
+A different timestamp counts as a new message, even with identical text. Turn the switch off
+to forward repeated captures. Manual connection tests are exempt; network retries keep their event ID.
+Deduplication hashes survive restarts, delivery and journal deletion until app data is cleared.
+They are recorded even while the switch is off so re-enabling it covers those captures too.
+On upgrade, this history starts with newly captured events; existing queued events are preserved.
+
 The n8n confirmation mode requires JSON with `status: "accepted"` and the matching `event_id`.
 This is our fixture contract, not a built-in n8n response. Turn off **n8n confirmation** for a
 custom webhook that acknowledges with HTTP 2xx. Neither response proves delivery to a downstream

app/build.gradle.kts

@@ -31,8 +31,8 @@ android {
         applicationId = "life.andre.message487"
         minSdk = 26
         targetSdk = 36
-        versionCode = 4
-        versionName = "0.0.4"
+        versionCode = 5
+        versionName = "0.0.5"
         buildConfigField("String", "GIT_COMMIT_HASH", "\"$gitCommitHash\"")
     }
     signingConfigs {

app/src/main/java/life/andre/message487/ConnectionViewModel.kt

@@ -106,6 +106,7 @@ class ConnectionViewModel(application: Application) : AndroidViewModel(applicati
 
     fun sendTest() = action(R.string.test_queued) { graph.enqueueTest() }
     fun notifications(enabled: Boolean) = action { graph.settings.update { it.copy(notifications = enabled) } }
+    fun deduplication(enabled: Boolean) = action { graph.settings.update { it.copy(deduplication = enabled) } }
     fun sms(enabled: Boolean) = action { graph.settings.update { it.copy(sms = enabled) } }
     fun selectAllPackages() = action {
         val packages = apps.value.map { it.packageName }.toSet() - getApplication<Application>().packageName

app/src/main/java/life/andre/message487/ForwardingSettings.kt

@@ -17,6 +17,7 @@ data class ForwardingSettings(
     val packages: Set<String> = emptySet(),
     val captureFailed: Boolean = false,
     val authToken: String = "",
+    val deduplication: Boolean = true,
 ) {
     override fun toString(): String = "ForwardingSettings(redacted)"
 
@@ -40,6 +41,7 @@ class SettingsStore(
                 cipher.decrypt(android.util.Base64.decode(it, android.util.Base64.NO_WRAP))
             }.orEmpty()
         }.getOrDefault(""),
+        deduplication = preferences.getBoolean("deduplication", true),
         url = preferences.getString("url", BuildConfig.DEFAULT_WEBHOOK_URL).orEmpty(),
         deviceId = preferences.getString("device_id", "").orEmpty(),
         deviceCode = preferences.getString("device_code", "android-device").orEmpty(),
@@ -66,6 +68,7 @@ class SettingsStore(
                 .putString("url", next.url).putString("device_id", next.deviceId)
                 .putString("device_code", next.deviceCode).putBoolean("require_ack", next.requireAck)
                 .putBoolean("notifications", next.notifications).putBoolean("sms", next.sms)
+                .putBoolean("deduplication", next.deduplication)
                 .putBoolean("paused", next.paused).putStringSet("packages", next.packages)
                 .putBoolean("capture_failed", next.captureFailed).commit()
         ) throw IOException("Could not save settings")

app/src/main/java/life/andre/message487/MessageGraph.kt

@@ -4,7 +4,6 @@ import android.app.Application
 import android.content.Context
 import java.security.MessageDigest
 import java.time.Instant
-import java.util.UUID
 import java.util.concurrent.Executors
 import java.io.File
 import life.andre.message487.diagnostics.CrashHandler
@@ -66,24 +65,22 @@ class MessageGraph internal constructor(private val context: Application) {
             occurredAt = Instant.ofEpochMilli(notification.postedAt).toString(),
             messageType = "notification", text = notification.text, title = notification.title,
         )
-        enqueue(event, config, digest(notification.key), digest(notification.title + "\u0000" + notification.text))
+        enqueue(event, config)
     }
 
     fun captureSms(sender: String, text: String, timestamp: Long) {
         val config = settings.state.value
         if (config.paused || !config.sms || !config.ready()) return
-        val identity = "${config.deviceId}\u0000$sender\u0000$timestamp\u0000$text"
         val event = MessageEvent(
             config.deviceId, config.deviceCode, sources.resolve("android"),
-            eventId = UUID.nameUUIDFromBytes(identity.toByteArray(Charsets.UTF_8)).toString(),
             occurredAt = Instant.ofEpochMilli(timestamp).toString(),
             messageType = "sms", text = text, sender = sender,
         )
         enqueue(event, config)
     }
 
-    private fun enqueue(event: MessageEvent, config: ForwardingSettings, key: String? = null, fingerprint: String? = null) {
-        if (outbox.enqueue(event, config, key, fingerprint)) {
+    private fun enqueue(event: MessageEvent, config: ForwardingSettings) {
+        if (outbox.enqueue(event, config)) {
             diagnostics.record(DiagnosticEvent.EVENT_QUEUED, type = event.messageType)
             scheduler.schedule(event.eventId)
         } else diagnostics.record(DiagnosticEvent.DUPLICATE_SKIPPED, type = event.messageType)

app/src/main/java/life/andre/message487/NotificationCaptureService.kt

@@ -59,10 +59,4 @@ class NotificationCaptureService : NotificationListenerService() {
         } catch (error: Exception) { graph.captureFailed(error) }
     }
 
-    override fun onNotificationRemoved(sbn: StatusBarNotification) {
-        val graph = MessageGraph.get(this)
-        graph.captureExecutor.execute {
-            try { graph.outbox.forgetNotification(digest(sbn.key)) } catch (error: Exception) { graph.captureFailed(error) }
-        }
-    }
 }

app/src/main/java/life/andre/message487/Outbox.kt

@@ -40,7 +40,7 @@ fun deliveryQueueState(result: DeliveryResult): QueueState = when (result.status
     } else QueueState.BLOCKED
 }
 
-class Outbox(context: Context, private val cipher: PayloadCipher) : SQLiteOpenHelper(context, "outbox.db", null, 1) {
+class Outbox(context: Context, private val cipher: PayloadCipher) : SQLiteOpenHelper(context, "outbox.db", null, 2) {
     private val mutableRevision = MutableStateFlow(0L)
     val revision = mutableRevision.asStateFlow()
 
@@ -51,21 +51,34 @@ class Outbox(context: Context, private val cipher: PayloadCipher) : SQLiteOpenHe
             outcome TEXT, http_code INTEGER, payload BLOB, attempt_token TEXT
         )""")
         db.execSQL("CREATE INDEX events_state ON events(state)")
-        db.execSQL("CREATE TABLE notifications (notification_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL)")
+        createDeduplicationTable(db)
     }
 
     override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
-        error("Missing forward migration from $oldVersion to $newVersion")
+        if (oldVersion < 2) {
+            createDeduplicationTable(db)
+            db.execSQL("DROP TABLE IF EXISTS notifications")
+        }
+    }
+
+    private fun createDeduplicationTable(db: SQLiteDatabase) {
+        db.execSQL("CREATE TABLE message_fingerprints (fingerprint TEXT PRIMARY KEY)")
     }
 
     @Synchronized
-    fun enqueue(event: MessageEvent, settings: ForwardingSettings, notificationKey: String? = null, fingerprint: String? = null): Boolean {
+    fun enqueue(event: MessageEvent, settings: ForwardingSettings): Boolean {
         val db = writableDatabase
         db.beginTransaction()
         try {
-            if (notificationKey != null) {
-                db.rawQuery("SELECT fingerprint FROM notifications WHERE notification_key = ?", arrayOf(notificationKey)).use {
-                    if (it.moveToFirst() && it.getString(0) == fingerprint) return false
+            // JSON preserves field boundaries; normalize equivalent timestamp representations.
+            // Test requests are deliberate user actions and are never content-deduplicated.
+            val fingerprint = if (event.messageType in setOf("sms", "notification")) digest(
+                org.json.JSONArray().put(event.source.packageName)
+                    .put(java.time.Instant.parse(event.occurredAt).toString()).put(event.text).toString()
+            ) else null
+            if (settings.deduplication && fingerprint != null) {
+                db.rawQuery("SELECT 1 FROM message_fingerprints WHERE fingerprint = ?", arrayOf(fingerprint)).use {
+                    if (it.moveToFirst()) return false
                 }
             }
             db.rawQuery("SELECT id FROM events WHERE id = ?", arrayOf(event.eventId)).use {
@@ -81,12 +94,8 @@ class Outbox(context: Context, private val cipher: PayloadCipher) : SQLiteOpenHe
                 put("state", QueueState.PENDING.name)
                 put("payload", cipher.encrypt(envelope))
             })
-            if (notificationKey != null) {
-                db.insertWithOnConflict("notifications", null, ContentValues().apply {
-                    put("notification_key", notificationKey)
-                    put("fingerprint", requireNotNull(fingerprint))
-                }, SQLiteDatabase.CONFLICT_REPLACE)
-                db.execSQL("DELETE FROM notifications WHERE rowid NOT IN (SELECT rowid FROM notifications ORDER BY rowid DESC LIMIT 2000)")
+            if (fingerprint != null) {
+                db.execSQL("INSERT OR IGNORE INTO message_fingerprints (fingerprint) VALUES (?)", arrayOf(fingerprint))
             }
             db.setTransactionSuccessful()
         } finally {
@@ -96,11 +105,6 @@ class Outbox(context: Context, private val cipher: PayloadCipher) : SQLiteOpenHe
         return true
     }
 
-    @Synchronized
-    fun forgetNotification(key: String) {
-        writableDatabase.delete("notifications", "notification_key = ?", arrayOf(key))
-    }
-
     @Synchronized
     fun pendingIds(): List<String> = readableDatabase.rawQuery(
         "SELECT id FROM events WHERE state IN ('PENDING', 'SENDING', 'RETRY') ORDER BY created_at", null

app/src/main/java/life/andre/message487/SourcesScreen.kt

@@ -93,6 +93,12 @@ internal fun SourcesScreen(settings: ForwardingSettings, permissions: Permission
                 }
             }
         }
+        item {
+            Panel {
+                SourceSwitch(Icons.Outlined.FilterList, R.string.deduplication_enabled, R.string.deduplication_hint,
+                    settings.deduplication, !busy, model::deduplication)
+            }
+        }
         item {
             SectionTitle(stringResource(R.string.selected_apps, settings.packages.size), stringResource(R.string.add_package)) { addPackage = true }
             SupportingText(stringResource(R.string.app_selection_short))

app/src/main/res/values-ru/strings.xml

@@ -123,4 +123,6 @@
     <string name="app_version">Версия %1$s · %2$s</string>
     <string name="privacy_policy">Политика конфиденциальности</string>
     <string name="no_browser">Не удалось открыть политику конфиденциальности: браузер недоступен.</string>
+    <string name="deduplication_enabled">Не отправлять дубли</string>
+    <string name="deduplication_hint">Пропускать сообщения с одинаковыми приложением, исходным временем и текстом. Отключите, чтобы пересылать и повторные сообщения.</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -123,4 +123,6 @@
     <string name="app_version">Version %1$s · %2$s</string>
     <string name="privacy_policy">Privacy policy</string>
     <string name="no_browser">No browser is available to open the privacy policy.</string>
+    <string name="deduplication_enabled">Skip duplicate messages</string>
+    <string name="deduplication_hint">Skip messages with exactly the same app, original timestamp and text. Turn off to forward repeated messages too.</string>
 </resources>

app/src/test/java/life/andre/message487/OutboxTest.kt

@@ -78,17 +78,59 @@ class OutboxTest {
         }
     }
 
-    @Test fun `notification duplicates survive restart while changed or reposted content is captured`() {
-        assertTrue(outbox.enqueue(event(), settings, "key", "content"))
+    @Test fun `deduplication uses app timestamp and text and survives delivery deletion and restart`() {
+        val original = event().copy(occurredAt = "2026-09-09T10:00:00Z")
+        assertTrue(outbox.enqueue(original, settings))
+        val attempt = outbox.beginAttempt(original.eventId)!!
+        outbox.finish(original.eventId, attempt.token, DeliveryResult(original.eventId, DeliveryStatus.ACCEPTED, 200))
+        outbox.delete(original.eventId)
         outbox.close()
         outbox = Outbox(context, codec)
-        assertFalse(outbox.enqueue(event(), settings, "key", "content"))
-        assertTrue(outbox.enqueue(event(), settings, "key", "changed"))
-        outbox.forgetNotification("key")
-        assertTrue(outbox.enqueue(event(), settings, "key", "changed"))
+        val duplicate = original.copy(eventId = "duplicate", title = "Different title", sender = "Different sender")
+        assertFalse(outbox.enqueue(duplicate, settings))
+        assertFalse(outbox.enqueue(duplicate.copy(occurredAt = "2026-09-09T10:00:00.000Z"), settings))
+        assertTrue(outbox.enqueue(duplicate.copy(eventId = "app", source = AppSource("other.app", "Chat")), settings))
+        assertTrue(outbox.enqueue(duplicate.copy(eventId = "time", occurredAt = "2026-09-09T10:00:00.001Z"), settings))
+        assertTrue(outbox.enqueue(duplicate.copy(eventId = "text", text = original.text + " "), settings))
         assertEquals(3, outbox.pendingCount())
     }
 
+    @Test fun `opt out allows sms and notification duplicates but preserves event id idempotency`() {
+        for (type in listOf("sms", "notification")) {
+            val original = event().copy(messageType = type)
+            assertTrue(outbox.enqueue(original, settings))
+            val duplicate = original.copy(eventId = "duplicate-$type")
+            assertFalse(outbox.enqueue(duplicate, settings))
+            assertTrue(outbox.enqueue(duplicate, settings.copy(deduplication = false)))
+            assertFalse(outbox.enqueue(duplicate, settings.copy(deduplication = false)))
+            assertFalse(outbox.enqueue(original.copy(eventId = "reenabled-$type"), settings))
+        }
+    }
+
+    @Test fun `events captured while opted out are remembered when reenabled and tests are exempt`() {
+        val original = event()
+        assertTrue(outbox.enqueue(original, settings.copy(deduplication = false)))
+        assertFalse(outbox.enqueue(original.copy(eventId = "reenabled"), settings))
+        val test = original.copy(eventId = "test-1", messageType = "test")
+        assertTrue(outbox.enqueue(test, settings))
+        assertTrue(outbox.enqueue(test.copy(eventId = "test-2"), settings))
+    }
+
+    @Test fun `version one migration preserves queued payloads`() {
+        val original = event()
+        outbox.enqueue(original, settings)
+        outbox.writableDatabase.apply {
+            execSQL("DROP TABLE message_fingerprints")
+            execSQL("CREATE TABLE notifications (notification_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL)")
+            version = 1
+        }
+        outbox.close()
+        outbox = Outbox(context, codec)
+        assertEquals(original.text, JSONObject(outbox.beginAttempt(original.eventId)!!.request.json).getString("text"))
+        assertTrue(outbox.enqueue(event(), settings))
+        assertEquals(2, outbox.readableDatabase.version)
+    }
+
     @Test fun `invalid acknowledgement blocks automatic delivery until manual retry`() {
         val event = event()
         outbox.enqueue(event, settings)
@@ -116,15 +158,16 @@ class OutboxTest {
         assertFalse(outbox.enqueue(waiting, settings))
     }
 
-    @Test fun `failed persistence does not advance notification deduplication`() {
+    @Test fun `failed persistence does not advance message deduplication`() {
+        val original = event()
         val broken = Outbox(context, object : PayloadCipher {
             override fun encrypt(value: String): ByteArray = throw java.io.IOException("Storage failure")
             override fun decrypt(value: ByteArray): String = error("Unused")
         })
         broken.use {
-            assertThrows(java.io.IOException::class.java) { it.enqueue(event(), settings, "key", "content") }
+            assertThrows(java.io.IOException::class.java) { it.enqueue(original, settings) }
         }
-        assertTrue(outbox.enqueue(event(), settings, "key", "content"))
+        assertTrue(outbox.enqueue(original, settings))
         assertEquals(1, outbox.pendingCount())
     }
 }

app/src/test/java/life/andre/message487/ScreenInteractionTest.kt

@@ -58,6 +58,17 @@ class ScreenInteractionTest {
         application.diagnostics.close()
     }
 
+    @Test fun `duplicate filter can be disabled and reenabled from sources`() {
+        compose.onNode(hasText(compose.activity.getString(R.string.sources_tab)) and hasClickAction()).performClick()
+        icon(R.string.deduplication_enabled).performScrollTo().assertIsOn().performClick()
+        compose.waitUntil(10_000) { compose.waitForIdle(); !model.state.value.busy }
+        assertFalse(SettingsStore(application).state.value.deduplication)
+        icon(R.string.deduplication_enabled).assertIsOff().performClick()
+        compose.waitUntil(10_000) { compose.waitForIdle(); !model.state.value.busy }
+        assertTrue(SettingsStore(application).state.value.deduplication)
+        icon(R.string.deduplication_enabled).assertIsOn()
+    }
+
     @Test fun `tabs navigate and diagnostics back returns to previous screen`() {
         for (destination in listOf(R.string.sources_tab, R.string.journal, R.string.connection_nav)) {
             compose.onNode(hasText(compose.activity.getString(destination)) and hasClickAction()).performClick()