diff --git a/benchmark-comparison.md b/benchmark-comparison.md new file mode 100644 index 0000000000..f75c338c0e --- /dev/null +++ b/benchmark-comparison.md @@ -0,0 +1,31 @@ +## Backup restore benchmark: v2 vs v3 + +Compared the legacy v2 backup (`1.bench.zip`) with the new v3 backup (`2.bench.zip`). Both runs restored **7,323 novels** with **0 failures**, 9 categories, and 14 plugins. + +### End-to-end result + +| Metric | Legacy v2 | New v3 | Change | +|---|---:|---:|---:| +| Total restore time | 5m 15.6s | 4m 11.0s | **64.7s faster (20.5%)** | + +Total time is measured from `local:start` through `local:finalize:done`. + +### Restore phases + +| Phase | Legacy v2 | New v3 | Change | +|---|---:|---:|---:| +| Copy | 12.8s | 13.2s | 0.5s slower | +| Outer unzip | 46.3s | 30.2s | **16.1s faster (34.7%)** | +| Novel validation | 46.7s | 24.9s | **21.9s faster (46.8%)** | +| Novel restore loop | 206.8s | 179.0s | **27.8s faster (13.4%)** | + +### Novel restore timing breakdown + +| Operation | Legacy v2 | New v3 | Change | +|---|---:|---:|---:| +| Read | 52.5s | 16.6s | **35.9s faster (68.4%)** | +| Parse | 45.5s | 32.7s | **12.8s faster (28.2%)** | +| Database | 135.5s | 136.7s | 1.3s slower (0.9%) | +| Covers | 19.6s | 17.8s | 1.8s faster (9.0%) | + +The v3 run reduced end-to-end restore time by **about one minute**. The largest per-operation improvement was reading novel data, while database time was effectively unchanged. The phase durations and per-operation timings are reported separately from the benchmark logs; the operation timings should not be summed as if they were sequential phases. diff --git a/drizzle/20260922120323_demonic_rattler/migration.sql b/drizzle/20260922120323_demonic_rattler/migration.sql new file mode 100644 index 0000000000..c0b084a3da --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/migration.sql @@ -0,0 +1,58 @@ +CREATE TABLE `RestoreChapterMapping` ( + `restoreRunId` text NOT NULL, + `backupNovelId` integer NOT NULL, + `backupChapterId` integer NOT NULL, + `restoredNovelId` integer NOT NULL, + `restoredChapterId` integer NOT NULL, + CONSTRAINT `fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk` FOREIGN KEY (`restoredNovelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk` FOREIGN KEY (`restoredChapterId`) REFERENCES `Chapter`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_Chapter` ( + `id` integer PRIMARY KEY AUTOINCREMENT, + `novelId` integer NOT NULL, + `path` text NOT NULL, + `name` text NOT NULL, + `releaseTime` text, + `bookmark` integer DEFAULT false, + `unread` integer DEFAULT true, + `readTime` text, + `isDownloaded` integer DEFAULT false, + `updatedTime` text, + `chapterNumber` real, + `page` text DEFAULT '1', + `position` integer DEFAULT 0, + `progress` integer, + `scanlator` text, + `timeSpent` integer DEFAULT 0, + CONSTRAINT `fk_Chapter_novelId_Novel_id_fk` FOREIGN KEY (`novelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +-- Preserve the previous AUTOINCREMENT high-water mark across the rebuild. +CREATE TEMP TABLE `__chapter_sequence` AS +SELECT COALESCE(MAX(`seq`), 0) AS `seq` +FROM `sqlite_sequence` +WHERE `name` = 'Chapter'; +--> statement-breakpoint +-- Chapters without a novel are invalid legacy data and are intentionally discarded. +INSERT INTO `__new_Chapter`(`id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent`) SELECT `id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent` FROM `Chapter` WHERE EXISTS (SELECT 1 FROM `Novel` WHERE `Novel`.`id` = `Chapter`.`novelId`);--> statement-breakpoint +DROP TABLE `Chapter`;--> statement-breakpoint +ALTER TABLE `__new_Chapter` RENAME TO `Chapter`;--> statement-breakpoint +INSERT INTO `sqlite_sequence` (`name`, `seq`) +SELECT 'Chapter', `seq` FROM `__chapter_sequence` +WHERE NOT EXISTS ( + SELECT 1 FROM `sqlite_sequence` WHERE `name` = 'Chapter' +); +--> statement-breakpoint +UPDATE `sqlite_sequence` +SET `seq` = MAX(`seq`, (SELECT `seq` FROM `__chapter_sequence`)) +WHERE `name` = 'Chapter'; +--> statement-breakpoint +DROP TABLE `__chapter_sequence`; +--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE UNIQUE INDEX `chapter_novel_path_unique` ON `Chapter` (`novelId`,`path`);--> statement-breakpoint +CREATE INDEX `chapterNovelIdIndex` ON `Chapter` (`novelId`,`position`,`page`,`id`);--> statement-breakpoint +CREATE UNIQUE INDEX `restore_chapter_mapping_unique` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`,`backupChapterId`);--> statement-breakpoint +CREATE INDEX `restore_chapter_mapping_novel_index` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`); \ No newline at end of file diff --git a/drizzle/20260922120323_demonic_rattler/snapshot.json b/drizzle/20260922120323_demonic_rattler/snapshot.json new file mode 100644 index 0000000000..fc040891a4 --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/snapshot.json @@ -0,0 +1,793 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "bcdf1b04-31ef-45ca-8a9a-8ecd417c3512", + "prevIds": [ + "2fada83f-d142-46d9-ba03-5ae1e232e3ce" + ], + "ddl": [ + { + "name": "Category", + "entityType": "tables" + }, + { + "name": "Chapter", + "entityType": "tables" + }, + { + "name": "NovelCategory", + "entityType": "tables" + }, + { + "name": "Novel", + "entityType": "tables" + }, + { + "name": "Repository", + "entityType": "tables" + }, + { + "name": "RestoreChapterMapping", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Category" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "sort", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "releaseTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "bookmark", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "unread", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "readTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isDownloaded", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "real", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "chapterNumber", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'1'", + "generated": null, + "name": "page", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "position", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "progress", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "scanlator", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "timeSpent", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "categoryId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "pluginId", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cover", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "author", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "artist", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'Unknown'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "genres", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "inLibrary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isLocal", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalPages", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersDownloaded", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersUnread", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalChapters", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastReadAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastUpdatedAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoreRunId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "novelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_Chapter_novelId_Novel_id_fk", + "entityType": "fks", + "table": "Chapter" + }, + { + "columns": [ + "restoredNovelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "restoredChapterId" + ], + "tableTo": "Chapter", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Category_pk", + "table": "Category", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Chapter_pk", + "table": "Chapter", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "NovelCategory_pk", + "table": "NovelCategory", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Novel_pk", + "table": "Novel", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Repository_pk", + "table": "Repository", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "name", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "category_name_unique", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "sort", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "category_sort_idx", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "chapter_novel_path_unique", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "position", + "isExpression": false + }, + { + "value": "page", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "chapterNovelIdIndex", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "categoryId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_category_unique", + "entityType": "indexes", + "table": "NovelCategory" + }, + { + "columns": [ + { + "value": "path", + "isExpression": false + }, + { + "value": "pluginId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_path_plugin_unique", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "pluginId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + }, + { + "value": "inLibrary", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "NovelIndex", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "url", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "repository_url_unique", + "entityType": "indexes", + "table": "Repository" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + }, + { + "value": "backupChapterId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_unique", + "entityType": "indexes", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_novel_index", + "entityType": "indexes", + "table": "RestoreChapterMapping" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/drizzle/migrations.js b/drizzle/migrations.js index beee4c381c..74735a0528 100644 --- a/drizzle/migrations.js +++ b/drizzle/migrations.js @@ -5,6 +5,7 @@ import m0001 from './20260612232322_normal_saracen/migration.sql'; import m0002 from './20260719143427_long_moondragon/migration.sql'; import m0003 from './20260727081855_calm_chimera/migration.sql'; import m0004 from './20260811071655_parched_human_torch/migration.sql'; +import m0005 from './20260922120323_demonic_rattler/migration.sql'; export default { migrations: { @@ -13,5 +14,6 @@ export default { '20260719143427_long_moondragon': m0002, '20260727081855_calm_chimera': m0003, '20260811071655_parched_human_torch': m0004, + '20260922120323_demonic_rattler': m0005, }, }; diff --git a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskDao.kt b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskDao.kt index a9bd9593bd..e922e1c701 100644 --- a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskDao.kt +++ b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskDao.kt @@ -5,13 +5,41 @@ import androidx.room.Insert import androidx.room.OnConflictStrategy import androidx.room.Query +data class BackgroundTaskSummary( + val id: String, + val type: String, + val title: String, + val description: String, + val state: String, + val progress: Double?, + val progressText: String?, + val attempt: Int, + val createdAt: Long, + val updatedAt: Long, +) + @Dao interface BackgroundTaskDao { @Insert(onConflict = OnConflictStrategy.ABORT) suspend fun insert(task: BackgroundTaskEntity) - @Query("SELECT * FROM background_tasks ORDER BY createdAt ASC") - suspend fun getAll(): List + @Query( + "SELECT id, type, title, description, state, progress, progressText, " + + "attempt, createdAt, updatedAt " + + "FROM background_tasks " + + "WHERE state IN ('queued', 'running', 'paused') ORDER BY createdAt ASC", + ) + suspend fun getActive(): List + + @Query( + "DELETE FROM background_tasks " + + "WHERE state IN ('succeeded', 'failed', 'cancelled') " + + "AND updatedAt < :cutoff", + ) + suspend fun deleteOldTerminal(cutoff: Long) + + @Query("DELETE FROM background_tasks WHERE id = :id AND state = :state") + suspend fun deleteIfState(id: String, state: String): Int @Query("SELECT * FROM background_tasks WHERE id = :id LIMIT 1") suspend fun get(id: String): BackgroundTaskEntity? @@ -22,11 +50,39 @@ interface BackgroundTaskDao { @Query("SELECT * FROM background_tasks WHERE queueName = :queueName AND state = 'queued' ORDER BY createdAt ASC") suspend fun getQueuedByQueueName(queueName: String): List - @Query("UPDATE background_tasks SET state = :state, updatedAt = :updatedAt WHERE id = :id") - suspend fun updateState(id: String, state: String, updatedAt: Long) + @Query( + "UPDATE background_tasks SET state = 'paused', updatedAt = :updatedAt " + + "WHERE id = :id AND state IN ('queued', 'running')", + ) + suspend fun markPaused(id: String, updatedAt: Long): Int + + @Query( + "UPDATE background_tasks SET state = 'queued', updatedAt = :updatedAt " + + "WHERE id = :id AND state = 'paused'", + ) + suspend fun markQueued(id: String, updatedAt: Long): Int + + @Query( + "UPDATE background_tasks SET state = 'cancelled', updatedAt = :updatedAt " + + "WHERE id = :id AND state IN ('queued', 'running', 'paused')", + ) + suspend fun markCancelled(id: String, updatedAt: Long): Int @Query("UPDATE background_tasks SET state = :state, updatedAt = :updatedAt WHERE id = :id AND state = 'running'") - suspend fun finishRunning(id: String, state: String, updatedAt: Long) + suspend fun finishRunning(id: String, state: String, updatedAt: Long): Int + + @Query( + "UPDATE background_tasks " + + "SET state = :state, progress = :progress, progressText = :progressText, updatedAt = :updatedAt " + + "WHERE id = :id AND state = 'running'", + ) + suspend fun finishRunningWithProgress( + id: String, + state: String, + progress: Double?, + progressText: String?, + updatedAt: Long, + ): Int @Query( """ @@ -51,11 +107,11 @@ interface BackgroundTaskDao { maxConcurrentDownloads: Int, ): Int - @Query("UPDATE background_tasks SET progress = :progress, progressText = :progressText, updatedAt = :updatedAt WHERE id = :id") - suspend fun updateProgress(id: String, progress: Double?, progressText: String?, updatedAt: Long) + @Query("UPDATE background_tasks SET progress = :progress, progressText = :progressText, updatedAt = :updatedAt WHERE id = :id AND state = 'running'") + suspend fun updateProgress(id: String, progress: Double?, progressText: String?, updatedAt: Long): Int - @Query("UPDATE background_tasks SET checkpoint = :checkpoint, updatedAt = :updatedAt WHERE id = :id") - suspend fun updateCheckpoint(id: String, checkpoint: String?, updatedAt: Long) + @Query("UPDATE background_tasks SET checkpoint = :checkpoint, updatedAt = :updatedAt WHERE id = :id AND state = 'running'") + suspend fun updateCheckpoint(id: String, checkpoint: String?, updatedAt: Long): Int @Query("UPDATE background_tasks SET workId = :workId, updatedAt = :updatedAt WHERE id = :id") suspend fun assignWork(id: String, workId: String, updatedAt: Long) diff --git a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskScheduler.kt b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskScheduler.kt index 7aa6a54fd2..303172d1da 100644 --- a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskScheduler.kt +++ b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/BackgroundTaskScheduler.kt @@ -42,9 +42,11 @@ object BackgroundTaskScheduler { // every task chained after it. if (!isRunning) return - WorkManager.getInstance(context).cancelAllWorkByTag(taskId).await() val dao = BackgroundTaskDatabase.get(context).tasks() + // The running worker may delete the row as it consumes cancellation. + // Capture the queue before cancelling that worker. val queueName = dao.get(taskId)?.queueName ?: return + WorkManager.getInstance(context).cancelAllWorkByTag(taskId).await() // Cancelling running work also cancels its dependents, so rebuild the // remaining queue after the worker has stopped. dao.getQueuedByQueueName(queueName).forEach { enqueue(context, it.id) } diff --git a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/LNReaderTaskWorker.kt b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/LNReaderTaskWorker.kt index dcafd600d1..9db2bad264 100644 --- a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/LNReaderTaskWorker.kt +++ b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/LNReaderTaskWorker.kt @@ -8,6 +8,8 @@ import androidx.work.CoroutineWorker import androidx.work.ForegroundInfo import androidx.work.WorkerParameters import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.withContext class LNReaderTaskWorker( appContext: Context, @@ -16,8 +18,12 @@ class LNReaderTaskWorker( override suspend fun doWork(): Result { val taskId = inputData.getString(BackgroundTaskScheduler.TASK_ID) ?: return Result.failure() val dao = BackgroundTaskDatabase.get(applicationContext).tasks() - val task = dao.get(taskId) ?: return Result.failure() - if (task.state == BackgroundTaskState.CANCELLED || task.state == BackgroundTaskState.PAUSED) { + val task = dao.get(taskId) ?: return Result.success() + if (task.state == BackgroundTaskState.CANCELLED) { + consumeCancellation(dao, taskId) + return Result.success() + } + if (task.state == BackgroundTaskState.PAUSED) { return Result.success() } @@ -28,13 +34,23 @@ class LNReaderTaskWorker( MAX_CONCURRENT_DOWNLOADS, ) if (claimed == 0) { - return if (dao.get(taskId)?.state == BackgroundTaskState.QUEUED) { - Result.retry() - } else { - Result.success() + return when (dao.get(taskId)?.state) { + BackgroundTaskState.QUEUED -> Result.retry() + BackgroundTaskState.CANCELLED -> { + consumeCancellation(dao, taskId) + Result.success() + } + else -> Result.success() + } + } + val runningTask = dao.get(taskId) ?: return Result.success() + if (runningTask.state != BackgroundTaskState.RUNNING) { + if (runningTask.state == BackgroundTaskState.CANCELLED) { + consumeCancellation(dao, taskId) } + return Result.success() } - val runningTask = dao.get(taskId) ?: return Result.failure() + setForeground(createForegroundInfo(runningTask)) val execution = TaskExecutionRegistry.register(taskId) @@ -47,61 +63,106 @@ class LNReaderTaskWorker( when (val executionResult = execution.await()) { TaskExecutionResult.Success -> { - val currentState = dao.get(taskId)?.state - if (currentState == BackgroundTaskState.CANCELLED) { - TaskNotificationFactory.dismiss(applicationContext, taskId) - return Result.success() - } - if (currentState == BackgroundTaskState.PAUSED) { - dao.get(taskId)?.let { TaskNotificationFactory.update(applicationContext, it) } - return Result.success() - } - dao.finishRunning(taskId, BackgroundTaskState.SUCCEEDED, System.currentTimeMillis()) - dao.get(taskId)?.let { - TaskNotificationFactory.postTerminal(applicationContext, it) + if (dao.finishRunning(taskId, BackgroundTaskState.SUCCEEDED, System.currentTimeMillis()) > 0) { + postOwnedTerminal(taskId, dao, BackgroundTaskState.SUCCEEDED) + } else { + resultAfterLostOwnership(taskId, dao) } Result.success() } is TaskExecutionResult.Failure -> { - val currentState = dao.get(taskId)?.state - if (currentState == BackgroundTaskState.PAUSED || currentState == BackgroundTaskState.CANCELLED) { - return Result.success() - } if (executionResult.shouldRetry) { - dao.updateState(taskId, BackgroundTaskState.QUEUED, System.currentTimeMillis()) - Result.retry() - } else { - dao.updateState(taskId, BackgroundTaskState.FAILED, System.currentTimeMillis()) - dao.get(taskId)?.let { - TaskNotificationFactory.postTerminal(applicationContext, it) + if (dao.finishRunning(taskId, BackgroundTaskState.QUEUED, System.currentTimeMillis()) > 0) { + Result.retry() + } else { + resultAfterLostOwnership(taskId, dao) } + } else if ( + dao.finishRunning(taskId, BackgroundTaskState.FAILED, System.currentTimeMillis()) > 0 + ) { + postOwnedTerminal(taskId, dao, BackgroundTaskState.FAILED) Result.success() + } else { + resultAfterLostOwnership(taskId, dao) } } } } catch (error: CancellationException) { - val latestState = dao.get(taskId)?.state - if (latestState !in listOf(BackgroundTaskState.PAUSED, BackgroundTaskState.CANCELLED)) { - dao.updateState(taskId, BackgroundTaskState.QUEUED, System.currentTimeMillis()) + withContext(NonCancellable) { + when (dao.get(taskId)?.state) { + BackgroundTaskState.CANCELLED -> consumeCancellation(dao, taskId) + BackgroundTaskState.RUNNING -> dao.finishRunning( + taskId, + BackgroundTaskState.QUEUED, + System.currentTimeMillis(), + ) + else -> Unit + } } throw error } catch (error: Exception) { - val latestState = dao.get(taskId)?.state - if (latestState !in listOf(BackgroundTaskState.PAUSED, BackgroundTaskState.CANCELLED)) { - val now = System.currentTimeMillis() - val message = error.message ?: error.javaClass.simpleName - dao.updateProgress(taskId, null, message, now) - dao.updateState(taskId, BackgroundTaskState.FAILED, now) - dao.get(taskId)?.let { - TaskNotificationFactory.postTerminal(applicationContext, it) + val now = System.currentTimeMillis() + val message = error.message ?: error.javaClass.simpleName + if ( + dao.finishRunningWithProgress( + taskId, + BackgroundTaskState.FAILED, + null, + message, + now, + ) > 0 + ) { + postOwnedTerminal(taskId, dao, BackgroundTaskState.FAILED) + Result.failure() + } else { + when (dao.get(taskId)?.state) { + BackgroundTaskState.CANCELLED -> { + consumeCancellation(dao, taskId) + Result.success() + } + BackgroundTaskState.PAUSED -> Result.success() + else -> Result.failure() } } - Result.failure() } finally { TaskExecutionRegistry.cancel(taskId) } } + private suspend fun resultAfterLostOwnership( + taskId: String, + dao: BackgroundTaskDao, + ): Result { + return when (dao.get(taskId)?.state) { + BackgroundTaskState.CANCELLED -> { + consumeCancellation(dao, taskId) + Result.success() + } + BackgroundTaskState.PAUSED -> { + dao.get(taskId)?.let { TaskNotificationFactory.update(applicationContext, it) } + Result.success() + } + else -> Result.success() + } + } + + private suspend fun postOwnedTerminal( + taskId: String, + dao: BackgroundTaskDao, + state: String, + ) { + val terminalTask = dao.get(taskId) ?: return + if (terminalTask.state != state) return + TaskNotificationFactory.postTerminal(applicationContext, terminalTask) + dao.deleteIfState(taskId, state) + } + + private suspend fun consumeCancellation(dao: BackgroundTaskDao, taskId: String) { + if (dao.deleteIfState(taskId, BackgroundTaskState.CANCELLED) > 0) { + TaskNotificationFactory.dismiss(applicationContext, taskId) + } + } + private fun createForegroundInfo(task: BackgroundTaskEntity): ForegroundInfo { val id = TaskNotificationFactory.notificationId(task.id) val notification = TaskNotificationFactory.build(applicationContext, task) diff --git a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/NativeBackgroundTasksModule.kt b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/NativeBackgroundTasksModule.kt index db231b4098..9c1f86ef86 100644 --- a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/NativeBackgroundTasksModule.kt +++ b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/NativeBackgroundTasksModule.kt @@ -57,7 +57,10 @@ class NativeBackgroundTasksModule : Module() { AsyncFunction("getTasks") { runBlocking(Dispatchers.IO) { - dao.getAll().map(::toRecord) + dao.deleteOldTerminal( + System.currentTimeMillis() - TERMINAL_TASK_RETENTION_MS, + ) + dao.getActive().map(::toSummary) } } @@ -70,11 +73,12 @@ class NativeBackgroundTasksModule : Module() { AsyncFunction("pause") { taskId: String -> runBlocking(Dispatchers.IO) { requireTask(taskId) - dao.updateState(taskId, BackgroundTaskState.PAUSED, System.currentTimeMillis()) - if (TaskExecutionRegistry.isActive(taskId)) { - emitInterruption(taskId, "pause") + if (dao.markPaused(taskId, System.currentTimeMillis()) > 0) { + if (TaskExecutionRegistry.isActive(taskId)) { + emitInterruption(taskId, "pause") + } + dao.get(taskId)?.let { TaskNotificationFactory.update(appContext.reactContext!!, it) } } - dao.get(taskId)?.let { TaskNotificationFactory.update(appContext.reactContext!!, it) } } } @@ -84,8 +88,9 @@ class NativeBackgroundTasksModule : Module() { if (TaskExecutionRegistry.isActive(taskId)) { throw IllegalStateException("Task is still pausing; try resuming again shortly") } - dao.updateState(taskId, BackgroundTaskState.QUEUED, System.currentTimeMillis()) - BackgroundTaskScheduler.enqueue(appContext.reactContext!!, taskId) + if (dao.markQueued(taskId, System.currentTimeMillis()) > 0) { + BackgroundTaskScheduler.enqueue(appContext.reactContext!!, taskId) + } } } @@ -94,13 +99,18 @@ class NativeBackgroundTasksModule : Module() { val task = requireTask(taskId) val isRunning = task.state == BackgroundTaskState.RUNNING || TaskExecutionRegistry.isActive(taskId) - dao.updateState(taskId, BackgroundTaskState.CANCELLED, System.currentTimeMillis()) + if (dao.markCancelled(taskId, System.currentTimeMillis()) == 0) { + TaskNotificationFactory.dismiss(appContext.reactContext!!, taskId) + return@runBlocking + } if (isRunning) { emitInterruption(taskId, "cancel") } BackgroundTaskScheduler.cancel(appContext.reactContext!!, taskId, isRunning) - dao.updateCheckpoint(taskId, null, System.currentTimeMillis()) TaskNotificationFactory.dismiss(appContext.reactContext!!, taskId) + if (!isRunning && task.state != BackgroundTaskState.QUEUED) { + dao.deleteIfState(taskId, BackgroundTaskState.CANCELLED) + } } } @@ -128,22 +138,13 @@ class NativeBackgroundTasksModule : Module() { val now = System.currentTimeMillis() dao.updateCheckpoint(taskId, null, now) dao.updateProgress(taskId, null, completionText, now) - dao.finishRunning(taskId, BackgroundTaskState.SUCCEEDED, now) TaskExecutionRegistry.complete(taskId, TaskExecutionResult.Success) } } AsyncFunction("fail") { taskId: String, error: String, shouldRetry: Boolean -> runBlocking(Dispatchers.IO) { - val currentState = dao.get(taskId)?.state - if (currentState !in listOf(BackgroundTaskState.PAUSED, BackgroundTaskState.CANCELLED)) { - dao.updateProgress(taskId, null, error, System.currentTimeMillis()) - dao.finishRunning( - taskId, - if (shouldRetry) BackgroundTaskState.QUEUED else BackgroundTaskState.FAILED, - System.currentTimeMillis(), - ) - } + dao.updateProgress(taskId, null, error, System.currentTimeMillis()) TaskExecutionRegistry.complete(taskId, TaskExecutionResult.Failure(error, shouldRetry)) } } @@ -179,22 +180,36 @@ class NativeBackgroundTasksModule : Module() { private suspend fun requireTask(taskId: String): BackgroundTaskEntity = dao.get(taskId) ?: throw IllegalArgumentException("Unknown background task: $taskId") + private fun toSummary(task: BackgroundTaskSummary): Map = mapOf( + "id" to task.id, + "type" to task.type, + "title" to task.title, + "description" to task.description, + "state" to task.state, + "progress" to task.progress, + "progressText" to task.progressText, + "attempt" to task.attempt, + "createdAt" to task.createdAt.toDouble(), + "updatedAt" to task.updatedAt.toDouble(), + ) + private fun toRecord(task: BackgroundTaskEntity): Map = mapOf( "id" to task.id, "type" to task.type, - "payload" to task.payload, "title" to task.title, "description" to task.description, "state" to task.state, "progress" to task.progress, "progressText" to task.progressText, - "checkpoint" to task.checkpoint, "attempt" to task.attempt, "createdAt" to task.createdAt.toDouble(), "updatedAt" to task.updatedAt.toDouble(), + "payload" to task.payload, + "checkpoint" to task.checkpoint, ) companion object { + private const val TERMINAL_TASK_RETENTION_MS = 24 * 60 * 60 * 1000L @Volatile private var reactContextRef: WeakReference? = null diff --git a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/TaskActionReceiver.kt b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/TaskActionReceiver.kt index 12adac9aec..9d4d82ac57 100644 --- a/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/TaskActionReceiver.kt +++ b/modules/native-background-tasks/android/src/main/java/expo/modules/nativebackgroundtasks/TaskActionReceiver.kt @@ -17,26 +17,34 @@ class TaskActionReceiver : BroadcastReceiver() { val task = dao.get(taskId) ?: return@launch when (intent.action) { TaskNotificationFactory.ACTION_PAUSE -> { - dao.updateState(taskId, BackgroundTaskState.PAUSED, System.currentTimeMillis()) - if (TaskExecutionRegistry.isActive(taskId)) { - NativeBackgroundTasksModule.emitInterruption(taskId, "pause") + if (dao.markPaused(taskId, System.currentTimeMillis()) > 0) { + if (TaskExecutionRegistry.isActive(taskId)) { + NativeBackgroundTasksModule.emitInterruption(taskId, "pause") + } + dao.get(taskId)?.let { TaskNotificationFactory.update(context, it) } } - dao.get(taskId)?.let { TaskNotificationFactory.update(context, it) } } TaskNotificationFactory.ACTION_RESUME -> { if (TaskExecutionRegistry.isActive(taskId)) return@launch - dao.updateState(taskId, BackgroundTaskState.QUEUED, System.currentTimeMillis()) - BackgroundTaskScheduler.enqueue(context, taskId) + if (dao.markQueued(taskId, System.currentTimeMillis()) > 0) { + BackgroundTaskScheduler.enqueue(context, taskId) + } } TaskNotificationFactory.ACTION_CANCEL -> { val isRunning = task.state == BackgroundTaskState.RUNNING || TaskExecutionRegistry.isActive(taskId) - dao.updateState(taskId, BackgroundTaskState.CANCELLED, System.currentTimeMillis()) + if (dao.markCancelled(taskId, System.currentTimeMillis()) == 0) { + TaskNotificationFactory.dismiss(context, taskId) + return@launch + } if (isRunning) { NativeBackgroundTasksModule.emitInterruption(taskId, "cancel") } BackgroundTaskScheduler.cancel(context, taskId, isRunning) TaskNotificationFactory.dismiss(context, taskId) + if (!isRunning && task.state != BackgroundTaskState.QUEUED) { + dao.deleteIfState(taskId, BackgroundTaskState.CANCELLED) + } } } } finally { diff --git a/modules/native-background-tasks/index.ts b/modules/native-background-tasks/index.ts index bff18a689c..ad615345a3 100644 --- a/modules/native-background-tasks/index.ts +++ b/modules/native-background-tasks/index.ts @@ -1,5 +1,6 @@ import NativeBackgroundTasks, { NativeBackgroundTaskRecord, + NativeBackgroundTaskSummary, } from './src/NativeBackgroundTasksModule'; export default NativeBackgroundTasks; -export { NativeBackgroundTaskRecord }; +export { NativeBackgroundTaskRecord, NativeBackgroundTaskSummary }; diff --git a/modules/native-background-tasks/src/NativeBackgroundTasksModule.ts b/modules/native-background-tasks/src/NativeBackgroundTasksModule.ts index 503a461797..8dacf87c6d 100644 --- a/modules/native-background-tasks/src/NativeBackgroundTasksModule.ts +++ b/modules/native-background-tasks/src/NativeBackgroundTasksModule.ts @@ -1,21 +1,27 @@ import { requireNativeModule } from 'expo-modules-core'; -export type NativeBackgroundTaskRecord = { +export type NativeBackgroundTaskSummary = { id: string; type: string; - payload: string; title: string; description?: string; state: string; progress?: number; progressText?: string; - checkpoint?: string; attempt: number; createdAt: number; updatedAt: number; }; +export type NativeBackgroundTaskRecord = NativeBackgroundTaskSummary & { + payload: string; + checkpoint?: string; +}; + type NativeBackgroundTasksModule = { + getTasks(): Promise; + getTask(taskId: string): Promise; + enqueue( type: string, payload: string, @@ -24,8 +30,6 @@ type NativeBackgroundTasksModule = { allowsDuplicates: boolean, queueName: string, ): Promise; - getTasks(): Promise; - getTask(taskId: string): Promise; pause(taskId: string): Promise; resume(taskId: string): Promise; cancel(taskId: string): Promise; diff --git a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt index 3a957a4d08..83db1e6d94 100644 --- a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt +++ b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt @@ -28,19 +28,22 @@ import okhttp3.JavaNetCookieJar import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response +import java.io.BufferedInputStream +import java.io.BufferedOutputStream import java.io.File +import java.io.FileInputStream import java.io.FileOutputStream import java.io.FileWriter import java.io.IOException import java.io.InputStream import java.io.OutputStream -import java.util.UUID import java.io.PushbackInputStream +import java.util.UUID import java.util.zip.GZIPInputStream import kotlin.coroutines.coroutineContext class NativeFileModule : Module() { - private val BUFFER_SIZE = 4096 + private val BUFFER_SIZE = 64 * 1024 private val okHttpClient = OkHttpClientProvider.createClient() private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private var pendingDocumentPromise: Promise? = null @@ -108,60 +111,67 @@ class NativeFileModule : Module() { ?: throw Exception("ENOENT: could not open an input stream for '$filepath'") } - private val writeAccessByAPILevel: String - get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private fun getOutputStream(filepath: String): OutputStream { val uri = getFileUri(filepath) return reactContext?.contentResolver?.openOutputStream(uri, writeAccessByAPILevel) ?: throw Exception("ENOENT: could not open an output stream for '$filepath'") } + private val writeAccessByAPILevel: String + get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private suspend fun copyFileContent( - filepath: String, - destPath: String, - onDone: (() -> Unit)? = null, - ): Long { - try { - val inputStream = getInputStream(filepath) - var copiedBytes = 0L - try { - val outputStream = getOutputStream(destPath) - try { - val buffer = ByteArray(BUFFER_SIZE) - var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { - coroutineContext.ensureActive() - outputStream.write(buffer, 0, length) - copiedBytes += length - } - outputStream.flush() - } finally { - outputStream.close() - } - } finally { - inputStream.close() - } - if (onDone != null) { - onDone() - } - return copiedBytes - } catch (e: IOException) { - throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + + private fun localFileOrNull(filepath: String): File? { + val uri = Uri.parse(filepath) + return when { + uri.scheme == null -> File(filepath) + uri.scheme.equals(ContentResolver.SCHEME_FILE, ignoreCase = true) -> + uri.path?.let(::File) + else -> null } } - private fun contentResolver(): ContentResolver = reactContext?.contentResolver ?: throw IOException("React context is unavailable") - private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long { + + private suspend fun copyLocalFile(sourceFile: File, destinationFile: File): Long { + if (sourceFile.isDirectory) { + throw IOException("Invalid file, folder found!") + } + + val sourceSize = sourceFile.length() var copiedBytes = 0L - getInputStream(sourcePath).use { inputStream -> - outputStream.use { output -> + FileInputStream(sourceFile).channel.use { sourceChannel -> + FileOutputStream(destinationFile).channel.use { destinationChannel -> + while (copiedBytes < sourceSize) { + coroutineContext.ensureActive() + val bytesToTransfer = minOf(BUFFER_SIZE.toLong(), sourceSize - copiedBytes) + val transferredBytes = + sourceChannel.transferTo(copiedBytes, bytesToTransfer, destinationChannel) + if (transferredBytes <= 0) { + throw IOException("Could not copy the complete local file") + } + copiedBytes += transferredBytes + } + } + } + + if (destinationFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + return copiedBytes + } + + private suspend fun copyStreams( + inputStream: InputStream, + outputStream: OutputStream, + ): Long { + var copiedBytes = 0L + BufferedInputStream(inputStream, BUFFER_SIZE).use { input -> + BufferedOutputStream(outputStream, BUFFER_SIZE).use { output -> val buffer = ByteArray(BUFFER_SIZE) var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { + while (input.read(buffer).also { length = it } > 0) { coroutineContext.ensureActive() output.write(buffer, 0, length) copiedBytes += length @@ -172,6 +182,44 @@ class NativeFileModule : Module() { return copiedBytes } + private suspend fun copyFileContent( + filepath: String, + destPath: String, + onDone: (() -> Unit)? = null, + ): Long { + try { + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val copiedBytes = if (sourceFile != null && destinationFile != null) { + try { + copyLocalFile(sourceFile, destinationFile) + } catch (_: IOException) { + FileInputStream(sourceFile).use { inputStream -> + FileOutputStream(destinationFile).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + } else { + getInputStream(filepath).use { inputStream -> + getOutputStream(destPath).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + if (sourceFile != null && sourceFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + onDone?.invoke() + return copiedBytes + } catch (e: IOException) { + throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + } + } + + private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long = + copyStreams(getInputStream(sourcePath), outputStream) + private fun resolveDirectoryFile(directoryUri: String): File { val uri = Uri.parse(directoryUri) val directory = if (uri.scheme == null) { @@ -479,7 +527,28 @@ class NativeFileModule : Module() { AsyncFunction("moveFile") { filepath: String, destPath: String, promise: Promise -> coroutineScope.launch { try { - val inFile = File(filepath) + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val renamed = if ( + sourceFile != null && + destinationFile != null && + sourceFile.isFile && + !destinationFile.exists() + ) { + try { + sourceFile.renameTo(destinationFile) + } catch (_: SecurityException) { + false + } + } else { + false + } + if (renamed) { + promise.resolve(null) + return@launch + } + + val inFile = sourceFile ?: File(filepath) copyFileContent(filepath, destPath) { if (!inFile.delete()) { throw IOException("Failed to delete source file '$filepath'") diff --git a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt index 75222b47a1..18ee17ae36 100644 --- a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt +++ b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt @@ -13,11 +13,24 @@ import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream class NativeZipArchiveModule : Module() { - private fun zipProcess(sourceDirPath: String, zos: ZipOutputStream) { + private fun zipProcess( + sourceDirPath: String, + zos: ZipOutputStream, + archivePrefix: String = "", + archiveEntries: MutableSet? = null, + excludedFile: File? = null, + ) { val sourceDir = File(sourceDirPath) - sourceDir.walkBottomUp().filter { it.isFile }.forEach { file -> - val zipFileName = + val excludedCanonicalPath = excludedFile?.canonicalPath + sourceDir.walkBottomUp().filter { file -> + file.isFile && file.canonicalPath != excludedCanonicalPath + }.forEach { file -> + val relativePath = file.absolutePath.removePrefix(sourceDir.absolutePath).removePrefix("/") + val zipFileName = joinArchivePath(archivePrefix, relativePath) + if (archiveEntries != null && !archiveEntries.add(zipFileName)) { + throw IllegalArgumentException("Duplicate ZIP entry: $zipFileName") + } val entry = ZipEntry("$zipFileName${(if (file.isDirectory) "/" else "")}") zos.putNextEntry(entry) file.inputStream().use { fis -> @@ -26,22 +39,95 @@ class NativeZipArchiveModule : Module() { } } + private fun joinArchivePath(prefix: String, relativePath: String): String { + val normalizedRelativePath = relativePath.replace('\\', '/').trimStart('/') + return if (prefix.isEmpty()) { + normalizedRelativePath + } else { + "$prefix/$normalizedRelativePath" + } + } + + private fun normalizeArchivePrefix(prefix: String): String { + require(prefix.isEmpty() || prefix.isNotBlank()) { "ZIP prefix must not be blank" } + val normalized = prefix.replace('\\', '/') + require(!normalized.startsWith("/") && !(normalized.length >= 2 && normalized[1] == ':')) { + "ZIP prefix must be relative" + } + val components = normalized.split('/').filter { it.isNotEmpty() && it != "." } + require(components.none { it == ".." }) { "ZIP prefix must not contain .." } + return components.joinToString("/") + } + + private fun ensureParentDirectory(file: File, createdDirectories: MutableSet) { + val parent = file.parentFile ?: return + val parentPath = parent.absolutePath + if (parentPath !in createdDirectories && (parent.exists() || parent.mkdirs())) { + createdDirectories.add(parentPath) + } + } + private fun resolveZipEntry(destination: File, entryName: String): File { + val canonicalDestination = destination.canonicalFile + val outputFile = File(destination, entryName).canonicalFile + val destinationPath = + canonicalDestination.path.let { if (it.endsWith(File.separator)) it else it + File.separator } + require(outputFile.path.startsWith(destinationPath)) { + "ZIP entry is outside the destination directory: $entryName" + } + return outputFile + } + + override fun definition() = ModuleDefinition { Name("NativeZipArchive") AsyncFunction("unzip") { sourceFilePath: String, distDirPath: String, promise: Promise -> Thread { + val unzipStartedAtNanos = System.nanoTime() try { + val createdDirectories = mutableSetOf() + val stats = mutableMapOf( + "fileCount" to 0L, + "compressedBytes" to 0L, + "uncompressedBytes" to 0L, + "largestUncompressedEntryBytes" to 0L, + ) + val categories = listOf("NovelAndChapters", "Covers", "NovelFiles", "other") + .associateWith { + mutableMapOf( + "fileCount" to 0L, + "compressedBytes" to 0L, + "uncompressedBytes" to 0L, + ) + } ZipFile(sourceFilePath).use { zis -> zis.entries().asSequence().filterNot { it.isDirectory }.forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + val uncompressedBytes = zipEntry.size.coerceAtLeast(0) + val compressedBytes = zipEntry.compressedSize.coerceAtLeast(0) + val firstPathSegment = zipEntry.name.substringBefore('/') + val category = categories[firstPathSegment] ?: categories.getValue("other") + stats["fileCount"] = stats.getValue("fileCount") + 1 + stats["compressedBytes"] = stats.getValue("compressedBytes") + compressedBytes + stats["uncompressedBytes"] = stats.getValue("uncompressedBytes") + uncompressedBytes + stats["largestUncompressedEntryBytes"] = + maxOf(stats.getValue("largestUncompressedEntryBytes"), uncompressedBytes) + category["fileCount"] = category.getValue("fileCount") + 1 + category["compressedBytes"] = category.getValue("compressedBytes") + compressedBytes + category["uncompressedBytes"] = category.getValue("uncompressedBytes") + uncompressedBytes + + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) + ensureParentDirectory(newFile, createdDirectories) zis.getInputStream(zipEntry).use { inputStream -> FileOutputStream(newFile).use { fos -> inputStream.copyTo(fos, COPY_BUFFER_SIZE) } } } } - promise.resolve(null) + val result = mutableMapOf() + result.putAll(stats) + result["categories"] = categories + result["elapsedMs"] = + (System.nanoTime() - unzipStartedAtNanos) / 1_000_000.0 + promise.resolve(result) } catch (e: Exception) { promise.reject("UNZIP_FAILED", e.message ?: "Unzip failed", e) } @@ -52,7 +138,39 @@ class NativeZipArchiveModule : Module() { Thread { try { FileOutputStream(zipFilePath).use { fos -> - ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos) } + ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos, excludedFile = File(zipFilePath)) } + } + promise.resolve(null) + } catch (e: Exception) { + promise.reject("ZIP_FAILED", e.message ?: "Zip failed", e) + } + }.start() + } + + AsyncFunction("zipDirectories") { sources: List>, zipFilePath: String, promise: Promise -> + Thread { + try { + require(sources.isNotEmpty()) { "ZIP sources must not be empty" } + require(zipFilePath.isNotBlank()) { "ZIP file path must not be blank" } + val normalizedSources = sources.map { source -> + val sourceDirPath = + source["path"]?.takeIf { it.isNotBlank() } + ?: throw IllegalArgumentException("ZIP source path must not be blank") + require(File(sourceDirPath).isDirectory) { + "ZIP source path is not a directory: $sourceDirPath" + } + val archivePrefix = + source["prefix"]?.let(::normalizeArchivePrefix) + ?: throw IllegalArgumentException("ZIP source prefix is missing") + sourceDirPath to archivePrefix + } + val archiveEntries = mutableSetOf() + FileOutputStream(zipFilePath).use { fos -> + ZipOutputStream(fos).use { zos -> + normalizedSources.forEach { (sourceDirPath, archivePrefix) -> + zipProcess(sourceDirPath, zos, archivePrefix, archiveEntries, File(zipFilePath)) + } + } } promise.resolve(null) } catch (e: Exception) { @@ -69,12 +187,13 @@ class NativeZipArchiveModule : Module() { headers.forEach { (key, value) -> connection.setRequestProperty(key, value) } + val createdDirectories = mutableSetOf() ZipInputStream(connection.inputStream).use { zis -> generateSequence { zis.nextEntry } .filterNot { it.isDirectory } .forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) + ensureParentDirectory(newFile, createdDirectories) FileOutputStream(newFile).use { fos -> zis.copyTo(fos, COPY_BUFFER_SIZE) } } } diff --git a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift index 7060acd82a..c36bbd2b6b 100644 --- a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift +++ b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift @@ -13,6 +13,10 @@ public class NativeZipArchiveModule: Module { promise.reject("NOT_IMPLEMENTED", "zip is not implemented on iOS") } + AsyncFunction("zipDirectories") { (sources: [[String: String]], zipFilePath: String, promise: Promise) in + promise.reject("NOT_IMPLEMENTED", "zipDirectories is not implemented on iOS") + } + AsyncFunction("remoteUnzip") { (distDirPath: String, url: String, headers: [String: String], promise: Promise) in promise.reject("NOT_IMPLEMENTED", "remoteUnzip is not implemented on iOS") } diff --git a/modules/native-zip-archive/src/NativeZipArchiveModule.ts b/modules/native-zip-archive/src/NativeZipArchiveModule.ts index cd62ad6fbd..afbe3dfc46 100644 --- a/modules/native-zip-archive/src/NativeZipArchiveModule.ts +++ b/modules/native-zip-archive/src/NativeZipArchiveModule.ts @@ -1,8 +1,27 @@ import { requireNativeModule } from 'expo-modules-core'; +export type UnzipCategoryStats = { + fileCount: number; + compressedBytes: number; + uncompressedBytes: number; +}; + +export type UnzipStats = UnzipCategoryStats & { + largestUncompressedEntryBytes: number; + elapsedMs: number; + categories: Record< + 'NovelAndChapters' | 'Covers' | 'NovelFiles' | 'other', + UnzipCategoryStats + >; +}; + type NativeZipArchiveModule = { - unzip(sourceFilePath: string, distDirPath: string): Promise; + unzip(sourceFilePath: string, distDirPath: string): Promise; zip(sourceDirPath: string, zipFilePath: string): Promise; + zipDirectories( + sources: { path: string; prefix: string }[], + zipFilePath: string, + ): Promise; remoteUnzip( distDirPath: string, urlString: string, diff --git a/src/components/ListView.tsx b/src/components/ListView.tsx index feee0bb221..b240bd3b66 100644 --- a/src/components/ListView.tsx +++ b/src/components/ListView.tsx @@ -16,6 +16,7 @@ interface ListViewProps { onPress: () => void; isSelected?: boolean; onLongPress?: () => void; + continueReadingButton?: React.ReactNode; } const ListView = ({ @@ -27,6 +28,7 @@ const ListView = ({ onPress, isSelected, onLongPress, + continueReadingButton, }: ListViewProps) => { const fadedImage = { opacity: inLibraryBadge ? 0.5 : 1 }; return ( @@ -58,6 +60,7 @@ const ListView = ({ {unreadBadge} {inLibraryBadge} + {continueReadingButton} ); }; diff --git a/src/components/NovelCover.tsx b/src/components/NovelCover.tsx index cdea2b9ec1..a8dcc4913a 100644 --- a/src/components/NovelCover.tsx +++ b/src/components/NovelCover.tsx @@ -8,6 +8,7 @@ import { } from 'react-native'; import { LinearGradient } from 'expo-linear-gradient'; +import Color from 'color'; import ListView from './ListView'; import { DisplayModes } from '@screens/library/constants/constants'; @@ -19,6 +20,7 @@ import { getString } from '@i18n/translations'; import SourceScreenSkeletonLoading from '@screens/browse/loadingAnimation/SourceScreenSkeletonLoading'; import NovelCoverImage from './NovelCoverImage'; import { useNovelCoverLayout } from './NovelCoverLayoutContext'; +import MaterialCommunityIcons from '@react-native-vector-icons/material-design-icons'; interface UnreadBadgeProps { showDownloadBadges: boolean; @@ -61,6 +63,7 @@ interface INovelCover { selectedNovelIds?: number[]; globalSearch?: boolean; imageRequestInit?: ImageRequestInit; + onContinueReading?: () => void; } function isFromDB( @@ -84,6 +87,7 @@ function NovelCover< globalSearch, selectedNovelIds, imageRequestInit, + onContinueReading, }: INovelCover) { const selectionActive = hasSelection ?? (selectedNovelIds != null && selectedNovelIds.length > 0); @@ -108,6 +112,25 @@ function NovelCover< [imageRequestInit], ); + const continueReadingButton = + onContinueReading && !selectionActive ? ( + { + event.stopPropagation(); + onContinueReading(); + }} + style={[ + styles.continueReadingButton, + { backgroundColor: Color(theme.primary).alpha(0.9).string() }, + ]} + > + + + ) : null; + if (item.completeRow) { if (!addSkeletonLoading) { return <>; @@ -197,6 +220,13 @@ function NovelCover< width={coverWidth} /> ) : null} + {continueReadingButton ? ( + + {continueReadingButton} + + ) : null} ) : ( @@ -227,6 +257,7 @@ function NovelCover< onPress={selectionActive ? selectNovel : onPress} onLongPress={selectNovel} isSelected={isSelected} + continueReadingButton={continueReadingButton} /> ); } @@ -373,6 +404,18 @@ const styles = StyleSheet.create({ position: 'absolute', right: 4, }, + continueReadingButton: { + alignItems: 'center', + borderRadius: 8, + height: 36, + justifyContent: 'center', + overflow: 'hidden', + width: 36, + }, + continueReadingOverlay: { + position: 'absolute', + right: 12, + }, downloadBadge: { borderBottomLeftRadius: 4, borderTopLeftRadius: 4, diff --git a/src/components/__tests__/NovelCoverContinueReading.test.tsx b/src/components/__tests__/NovelCoverContinueReading.test.tsx new file mode 100644 index 0000000000..b9f9d66108 --- /dev/null +++ b/src/components/__tests__/NovelCoverContinueReading.test.tsx @@ -0,0 +1,165 @@ +import './mocks'; +import { Fragment, type ReactElement } from 'react'; +import { fireEvent, render, screen } from '@test-utils'; +import NovelCover from '../NovelCover'; +import { NovelCoverLayoutProvider } from '../NovelCoverLayoutContext'; +import { DisplayModes } from '@screens/library/constants/constants'; +import { useTheme } from '@hooks/persisted'; +import { getString } from '@i18n/translations'; +import LibraryNovelItem from '@screens/library/components/LibraryNovelItem'; +import type { History } from '@database/types'; + +jest.mock( + '@screens/browse/loadingAnimation/SourceScreenSkeletonLoading', + () => 'SourceScreenSkeletonLoading', +); +jest.mock('@hooks', () => ({ useDeviceOrientation: () => 'portrait' })); + +const novel = { + id: 1, + name: 'Example Novel', + path: '/example', + pluginId: 'example', +}; + +const onPress = jest.fn(); +const onContinueReading = jest.fn(); +const lastReadChapter = { + id: 7, + novelId: novel.id, + name: 'Chapter 7', +} as History; +const renderCover = (element: ReactElement) => + render(element, { wrapper: Fragment }); + +const Cover = ({ + displayMode, + hasSelection = false, + canContinue = true, +}: { + displayMode: DisplayModes; + hasSelection?: boolean; + canContinue?: boolean; +}) => { + const theme = useTheme(); + + return ( + + + + ); +}; + +const LibraryItem = ({ + showContinueReadingButton, +}: { + showContinueReadingButton: boolean; +}) => { + const theme = useTheme(); + + return ( + + + + ); +}; + +describe('NovelCover continue reading', () => { + beforeEach(() => { + onPress.mockClear(); + onContinueReading.mockClear(); + }); + + it.each([DisplayModes.Compact, DisplayModes.Comfortable, DisplayModes.List])( + 'opens the saved chapter from %s layout', + displayMode => { + renderCover(); + + fireEvent.press( + screen.getByRole('button', { + name: getString('novelScreen.continueReading'), + }), + { stopPropagation: jest.fn() }, + ); + + expect(onContinueReading).toHaveBeenCalledTimes(1); + expect(onPress).not.toHaveBeenCalled(); + }, + ); + + it('hides the shortcut during selection and when there is no reading history', () => { + const { rerender } = renderCover( + , + ); + expect( + screen.queryByRole('button', { + name: getString('novelScreen.continueReading'), + }), + ).toBeNull(); + + rerender( + , + ); + expect( + screen.queryByRole('button', { + name: getString('novelScreen.continueReading'), + }), + ).toBeNull(); + }); + + it('respects the library display setting', () => { + const { rerender } = renderCover( + , + ); + expect( + screen.queryByRole('button', { + name: getString('novelScreen.continueReading'), + }), + ).toBeNull(); + + rerender(); + fireEvent.press( + screen.getByRole('button', { + name: getString('novelScreen.continueReading'), + }), + { stopPropagation: jest.fn() }, + ); + expect(onContinueReading).toHaveBeenCalledWith(novel, lastReadChapter); + }); +}); diff --git a/src/database/__tests__/db.test.ts b/src/database/__tests__/db.test.ts index 155f67f84d..75a1857e56 100644 --- a/src/database/__tests__/db.test.ts +++ b/src/database/__tests__/db.test.ts @@ -809,6 +809,7 @@ describe('production migrations', () => { '20260719143427_long_moondragon', '20260727081855_calm_chimera', '20260811071655_parched_human_torch', + '20260922120323_demonic_rattler', ]); } finally { sqlite.close(); @@ -1210,4 +1211,67 @@ describe('production migrations', () => { sqlite.close(); } }); + it('discards orphan chapters and preserves the AUTOINCREMENT sequence', async () => { + const sqlite = open({ name: ':memory:' }); + sqlite.executeSync('PRAGMA foreign_keys = ON'); + // Drizzle expects async aliases that op-sqlite's DB type does not declare. + const asyncSqlite = sqlite as unknown as { + executeAsync?: typeof sqlite.execute; + executeRawAsync?: typeof sqlite.executeRaw; + }; + asyncSqlite.executeAsync ??= sqlite.execute; + asyncSqlite.executeRawAsync ??= sqlite.executeRaw; + try { + createSchema(sqlite); + sqlite.executeSync('ALTER TABLE Chapter ADD scanlator text'); + sqlite.executeSync('ALTER TABLE Chapter ADD timeSpent integer DEFAULT 0'); + sqlite.executeSync( + `INSERT INTO Novel (id, path, pluginId, name) + VALUES (1, '/novel', 'plugin', 'Novel')`, + ); + sqlite.executeSync(` + INSERT INTO Chapter (id, novelId, path, name) + VALUES + (1, 1, '/valid', 'Valid'), + (100, 1, '/deleted', 'Deleted') + `); + sqlite.executeSync('DELETE FROM Chapter WHERE id = 100'); + sqlite.executeSync( + `INSERT INTO Chapter (id, novelId, path, name) + VALUES (3, 999, '/orphan', 'Orphan')`, + ); + recordAppliedMigrations(sqlite); + + const drizzleDb = drizzle(sqlite, { schema }); + await migrate(drizzleDb, { + migrations: { + '20260922120323_demonic_rattler': + migrations.migrations['20260922120323_demonic_rattler'], + }, + }); + + expect( + sqlite.executeSync('SELECT id, novelId FROM Chapter').rows, + ).toEqual([{ id: 1, novelId: 1 }]); + expect( + sqlite.executeSync( + "SELECT seq FROM sqlite_sequence WHERE name = 'Chapter'", + ).rows, + ).toEqual([{ seq: 100 }]); + + sqlite.executeSync( + `INSERT INTO Chapter (novelId, path, name) + VALUES (1, '/next', 'Next')`, + ); + expect( + sqlite.executeSync('SELECT last_insert_rowid() AS id').rows, + ).toEqual([{ id: 101 }]); + expect(sqlite.executeRawSync('PRAGMA foreign_key_check;')).toEqual([]); + expect(sqlite.executeRawSync('PRAGMA integrity_check;')).toEqual([ + ['ok'], + ]); + } finally { + sqlite.close(); + } + }); }); diff --git a/src/database/db.ts b/src/database/db.ts index acdd5efcc2..c20d424a96 100644 --- a/src/database/db.ts +++ b/src/database/db.ts @@ -66,6 +66,7 @@ export const drizzleDb = drizzle(_db, { schema, logger: __DEV__ ? new MyLogger() : false, }); +export type DrizzleDb = typeof drizzleDb export const dbManager = createDbManager(drizzleDb); diff --git a/src/database/manager/__tests__/manager.test.ts b/src/database/manager/__tests__/manager.test.ts new file mode 100644 index 0000000000..7f9678670c --- /dev/null +++ b/src/database/manager/__tests__/manager.test.ts @@ -0,0 +1,63 @@ +import { + createTestDb, + cleanupTestDb, +} from '@database/queries/__tests__/testDb'; + +describe('DbManager.executeBatch', () => { + const testDb = createTestDb(); + const flushPendingReactiveQueries = jest.fn(); + + afterAll(() => { + cleanupTestDb(testDb); + }); + + beforeEach(() => { + flushPendingReactiveQueries.mockClear(); + Object.defineProperty(testDb.sqlite, 'flushPendingReactiveQueries', { + configurable: true, + value: flushPendingReactiveQueries, + }); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS BatchTest'); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS MissingBatchTable'); + testDb.sqlite.executeSync( + 'CREATE TABLE BatchTest (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL)', + ); + }); + + it('executes heterogeneous and repeated-parameter commands atomically', async () => { + const result = await testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['first'], ['second']]], + ['UPDATE BatchTest SET value = ? WHERE value = ?', ['updated', 'first']], + ['DELETE FROM BatchTest WHERE value = ?', [['second']]], + ]); + + expect(result.rowsAffected).toBe(4); + expect(flushPendingReactiveQueries).toHaveBeenCalledTimes(1); + expect( + testDb.sqlite.executeSync('SELECT value FROM BatchTest').rows, + ).toEqual([{ value: 'updated' }]); + }); + + it('rolls back every command when a later command fails', async () => { + await expect( + testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['rolled back']]], + ['INSERT INTO MissingBatchTable (value) VALUES (?)', [['failure']]], + ]), + ).rejects.toMatchObject({ + message: expect.stringContaining('MissingBatchTable'), + }); + + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + expect(testDb.sqlite.executeSync('SELECT * FROM BatchTest').rows).toEqual( + [], + ); + }); + + it('does not flush for an empty batch', async () => { + const result = await testDb.dbManager.executeBatch([]); + + expect(result).toEqual({ rowsAffected: 0 }); + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + }); +}); diff --git a/src/database/manager/manager.d.ts b/src/database/manager/manager.d.ts index 195bab40bb..95584c8c0d 100644 --- a/src/database/manager/manager.d.ts +++ b/src/database/manager/manager.d.ts @@ -1,27 +1,30 @@ // db-manager.types.ts import type { - SQLiteTransaction, - TablesRelationalConfig, - Placeholder, -} from 'drizzle-orm'; -import { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; + BatchQueryResult, + SQLBatchTuple, +} from '@op-engineering/op-sqlite'; +import type { DrizzleDb } from '@database/db'; +import type { Placeholder } from 'drizzle-orm'; +import type { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; // Define the TransactionParameter type based on your DrizzleDb -export type TransactionParameter = SQLiteTransaction< - 'async', - { changes: number; lastInsertRowid: number } | void, - Record, - TablesRelationalConfig ->; - +export type TransactionParameter = Parameters< + Parameters[0] +>[0]; /** * Interface defining the public API and documentation for the Drizzle database manager. * This contract ensures consistent documentation and type safety across the application. */ export interface IDbManager { + /** + * Executes raw op-sqlite commands as one atomic transaction. + */ + executeBatch(commands: SQLBatchTuple[]): Promise; /** * Efficiently executes a Drizzle query for multiple data rows using - * op-sqlite executeBatch under the hood. + * op-sqlite executeBatch under the hood. The batch owns its transaction; + * it cannot be nested inside write(). The callback argument is only used + * to build the prepared query and is not the surrounding write transaction. */ batch>( data: T[], @@ -29,7 +32,7 @@ export interface IDbManager { tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ): Promise; + ): Promise; /** * Creates a subquery that defines a temporary named result set as a CTE. diff --git a/src/database/manager/manager.ts b/src/database/manager/manager.ts index cc5690d755..6ce98c6ef2 100644 --- a/src/database/manager/manager.ts +++ b/src/database/manager/manager.ts @@ -1,5 +1,9 @@ +import type { + BatchQueryResult, + SQLBatchTuple, + Scalar, +} from '@op-engineering/op-sqlite'; import type { drizzleDb } from '@database/db'; -import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; import { IDbManager } from './manager.d'; import { DbTaskQueue } from './queue'; import { Schema } from '../schema'; @@ -88,15 +92,32 @@ class DbManager implements IDbManager { .rows as Awaited>; } + public async executeBatch( + commands: SQLBatchTuple[], + ): Promise { + if (!commands.length) { + return { rowsAffected: 0 }; + } + + return await this.queue.enqueue({ + id: 'write', + run: async () => { + const result = await this.db.$client.executeBatch(commands); + this.db.$client.flushPendingReactiveQueries(); + return result; + }, + }); + } + public async batch>( data: T[], fn: ( tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ) { + ): Promise { if (!data.length) { - return; + return { rowsAffected: 0 }; } const ph = (arg: Extract) => sql.placeholder(arg); @@ -110,13 +131,7 @@ class DbManager implements IDbManager { }); const commands: SQLBatchTuple[] = [[query.sql, params]]; - await this.queue.enqueue({ - id: 'write', - run: async () => { - await this.db.$client.executeBatch(commands); - this.db.$client?.flushPendingReactiveQueries(); - }, - }); + return await this.executeBatch(commands); } public async write( diff --git a/src/database/queries/CategoryQueries.ts b/src/database/queries/CategoryQueries.ts index 2894c88888..54562a9b88 100644 --- a/src/database/queries/CategoryQueries.ts +++ b/src/database/queries/CategoryQueries.ts @@ -269,16 +269,23 @@ export const _restoreCategory = async ( categoryId = restoredCategory.id; } - // Insert novel-category associations if (category.novelIds && category.novelIds.length > 0) { - for (const backupNovelId of category.novelIds) { - const novelId = novelIdMap?.get(backupNovelId) ?? backupNovelId; + const membershipBatchSize = 500; + for ( + let batchStart = 0; + batchStart < category.novelIds.length; + batchStart += membershipBatchSize + ) { + const memberships = category.novelIds + .slice(batchStart, batchStart + membershipBatchSize) + .map(backupNovelId => ({ + categoryId, + novelId: novelIdMap?.get(backupNovelId) ?? backupNovelId, + })); + await tx .insert(novelCategorySchema) - .values({ - categoryId, - novelId: novelId, - }) + .values(memberships) .onConflictDoNothing() .run(); } diff --git a/src/database/queries/ChapterQueries.ts b/src/database/queries/ChapterQueries.ts index 9b7ca28cf4..3bcddc945b 100644 --- a/src/database/queries/ChapterQueries.ts +++ b/src/database/queries/ChapterQueries.ts @@ -468,14 +468,24 @@ export const getNovelChapters = async ( * a paginated UI query because doing so silently produces incomplete backups. */ export const getAllNovelChaptersForBackup = async ( - novelId: number, -): Promise => - dbManager + novelIds: number | number[], +): Promise => { + const ids = Array.isArray(novelIds) ? novelIds : [novelIds]; + if (ids.length === 0) { + return []; + } + + return dbManager .select() .from(chapterSchema) - .where(eq(chapterSchema.novelId, novelId)) - .orderBy(asc(chapterSchema.id)) + .where( + Array.isArray(novelIds) + ? inArray(chapterSchema.novelId, ids) + : eq(chapterSchema.novelId, ids[0]), + ) + .orderBy(asc(chapterSchema.novelId), asc(chapterSchema.id)) .all(); +}; export const getNovelChaptersSync = ( novelId: number, diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index 3d66c4d298..59c6cfc212 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -6,12 +6,7 @@ import { insertChapters } from './ChapterQueries'; import { showToast } from '@utils/showToast'; import { getString } from '@i18n/translations'; -import { - BackupNovel, - DBNovelInfo, - NovelInfo, - type RestoredNovelMapping, -} from '../types'; +import { DBNovelInfo, NovelInfo } from '../types'; import { SourceNovel } from '@plugins/types'; import { NOVEL_STORAGE } from '@utils/Storages'; import { downloadFile } from '@plugins/helpers/fetch'; @@ -21,7 +16,6 @@ import { novelSchema, novelCategorySchema, categorySchema, - chapterSchema, } from '@database/schema'; import type { TransactionParameter } from '@database/manager/manager.d'; import { getLibraryDefaultCategoryId } from '@hooks/persisted/useSettings'; @@ -75,7 +69,7 @@ const getCategoriesForNewNovel = async ( return getBuiltInDefaultCategory(tx); }; -const getCategoryForNewNovel = async (tx: TransactionParameter) => +export const getCategoryForNewNovel = async (tx: TransactionParameter) => (await getCategoriesForNewNovel(tx))[0]; /** @@ -287,69 +281,6 @@ export const deleteCachedNovels = async () => { showToast(getString('advancedSettingsScreen.cachedNovelsDeletedToast')); }; -/** - * Restore a novel from backup using Drizzle ORM. - */ -export const restoreLibrary = async (novel: NovelInfo) => { - const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { - throw e; - }); - - const novelId = await dbManager.write(async tx => { - const row = await tx - .insert(novelSchema) - .values({ - path: sourceNovel.path, - name: novel.name, - pluginId: novel.pluginId, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - name: novel.name, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }, - }) - .returning() - .get(); - - if (row) { - const defaultCategory = await getCategoryForNewNovel(tx); - - if (defaultCategory) { - await tx - .insert(novelCategorySchema) - .values({ - novelId: row.id, - categoryId: defaultCategory.id, - }) - .onConflictDoNothing() - .run(); - } - } - return row?.id; - }); - - if (novelId && sourceNovel.chapters) { - await insertChapters(novelId, sourceNovel.chapters); - } -}; - export const updateNovelInfo = async (info: NovelInfo) => { await dbManager.write(async tx => { await tx @@ -474,89 +405,3 @@ export const updateNovelCategories = async ( } }); }; - -/** - * Restores novel and chapters from a backup object. - */ -export const _restoreNovelAndChapters = async ( - backupNovel: BackupNovel, -): Promise => { - const { chapters, id: backupNovelId, ...novel } = backupNovel; - return dbManager.write(async tx => { - // Match novels by their stable source identity, not the database-local ID. - const restoredNovel = await tx - .insert(novelSchema) - .values({ - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }, - }) - .returning({ id: novelSchema.id }) - .get(); - - if (novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) { - const cacheSuffix = novel.cover.match(/[?#].*$/)?.[0] ?? ''; - await tx - .update(novelSchema) - .set({ - cover: `file://${NOVEL_STORAGE}/${novel.pluginId}/${restoredNovel.id}/cover.png${cacheSuffix}`, - }) - .where(eq(novelSchema.id, restoredNovel.id)) - .run(); - } - - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); - - const chapterMappings: RestoredNovelMapping['chapters'] = []; - - // Restore chapters in batches - if (chapters.length > 0) { - const BATCH_SIZE = 100; - for (let i = 0; i < chapters.length; i += BATCH_SIZE) { - const batch = chapters.slice(i, i + BATCH_SIZE); - const restoredChapters = await tx - .insert(chapterSchema) - .values( - batch.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ - ...chapter, - novelId: restoredNovel.id, - })), - ) - .returning({ id: chapterSchema.id, path: chapterSchema.path }) - .all(); - const restoredIdsByPath = new Map( - restoredChapters.map(chapter => [chapter.path, chapter.id]), - ); - for (const chapter of batch) { - const restoredChapterId = restoredIdsByPath.get(chapter.path); - if (restoredChapterId !== undefined) { - chapterMappings.push({ - backupChapterId: chapter.id, - restoredChapterId, - }); - } - } - } - } - - return { - pluginId: novel.pluginId, - backupNovelId, - restoredNovelId: restoredNovel.id, - chapters: chapterMappings, - }; - }); -}; diff --git a/src/database/queries/NovelRestoreQueries.ts b/src/database/queries/NovelRestoreQueries.ts new file mode 100644 index 0000000000..89a34afae4 --- /dev/null +++ b/src/database/queries/NovelRestoreQueries.ts @@ -0,0 +1,774 @@ +import { and, eq, inArray, or } from 'drizzle-orm'; +import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; + +import { fetchNovel } from '@services/plugin/fetch'; +import { insertChapters } from './ChapterQueries'; +import { getCategoryForNewNovel } from './NovelQueries'; + +import { dbManager } from '@database/db'; +import { + createNovelTriggerQueryDelete, + createNovelTriggerQueryInsert, + createNovelTriggerQueryUpdate, +} from '@database/queryStrings/triggers'; +import { + novelCategorySchema, + novelSchema, + restoreChapterMappingSchema, +} from '@database/schema'; +import { NOVEL_STORAGE } from '@utils/Storages'; +import type { BackupNovel, NovelInfo, RestoredNovelMapping } from '../types'; + +/** + * Restore a novel from backup using Drizzle ORM. + */ +export const restoreLibrary = async (novel: NovelInfo) => { + const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { + throw e; + }); + + const novelId = await dbManager.write(async tx => { + const row = await tx + .insert(novelSchema) + .values({ + path: sourceNovel.path, + name: novel.name, + pluginId: novel.pluginId, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }) + .onConflictDoUpdate({ + target: [novelSchema.path, novelSchema.pluginId], + set: { + name: novel.name, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }, + }) + .returning() + .get(); + + if (row) { + const defaultCategory = await getCategoryForNewNovel(tx); + + if (defaultCategory) { + await tx + .insert(novelCategorySchema) + .values({ + novelId: row.id, + categoryId: defaultCategory.id, + }) + .onConflictDoNothing() + .run(); + } + } + return row?.id; + }); + + if (novelId && sourceNovel.chapters) { + await insertChapters(novelId, sourceNovel.chapters); + } +}; +const sqliteBoolean = (value: boolean | null | undefined): Scalar => + value == null ? null : value ? 1 : 0; +const RESTORE_NOVEL_BATCH_SIZE = 100; +const RESTORE_CHAPTER_BATCH_SIZE = 10_000; + +export type RestoreNovelProgress = { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; +}; + +export type RestoreNovelOptions = { + includeChapterMappings?: boolean; + restoreRunId?: string; + onProgress?: (progress: RestoreNovelProgress) => void; +}; + +export type RestoreNovelMetrics = { + novelUpsertCalls: number; + novelUpsertRowsAttempted: number; + novelUpsertMs: number; + novelIdentityLookupCalls: number; + novelIdentityLookupRowsReturned: number; + novelIdentityLookupMs: number; + chapterWriteChunkCalls: number; + chapterWriteRowsAttempted: number; + chapterWriteMs: number; + chapterMappingRowsAttempted: number; + statsRefreshCalls: number; + statsRefreshNovels: number; + statsRefreshMs: number; + novelBatchFallbacks: number; + novelFallbackRowsAttempted: number; + chapterBatchFallbacks: number; + chapterFallbackRowsAttempted: number; + fallbackCauses: { name: string; message: string }[]; +}; + +export const recordRestoreFallback = ( + metrics: RestoreNovelMetrics | undefined, + error: unknown, +) => { + if (!metrics || metrics.fallbackCauses.length >= 5) { + return; + } + + const errorName = + error instanceof Error && + [ + 'AggregateError', + 'DatabaseError', + 'Error', + 'EvalError', + 'ReferenceError', + 'RangeError', + 'SQLiteError', + 'SqliteError', + 'SyntaxError', + 'TypeError', + 'URIError', + ].includes(error.name) + ? error.name + : 'Error'; + const message = + error instanceof Error && /constraint/i.test(error.message) + ? 'Database constraint failed' + : error instanceof Error && /locked|busy/i.test(error.message) + ? 'Database is busy' + : error instanceof Error && /no such table/i.test(error.message) + ? 'Database table missing' + : error instanceof Error && /no such column/i.test(error.message) + ? 'Database column missing' + : 'Database operation failed'; + metrics.fallbackCauses.push({ name: errorName, message }); +}; + +const createPlaceholderRows = (rowCount: number, valuesPerRow: number) => + Array.from( + { length: rowCount }, + () => `(${Array(valuesPerRow).fill('?').join(', ')})`, + ).join(',\n'); + +const buildNovelUpsertSql = (rowCount: number) => ` + INSERT INTO Novel ( + path, pluginId, name, cover, summary, author, artist, status, genres, + inLibrary, isLocal, totalPages + ) VALUES ${createPlaceholderRows(rowCount, 12)} + ON CONFLICT(path, pluginId) DO UPDATE SET + path = excluded.path, + pluginId = excluded.pluginId, + name = excluded.name, + cover = excluded.cover, + summary = excluded.summary, + author = excluded.author, + artist = excluded.artist, + status = excluded.status, + genres = excluded.genres, + inLibrary = excluded.inLibrary, + isLocal = excluded.isLocal, + totalPages = excluded.totalPages +`; + +const NOVEL_UPSERT_SQL = buildNovelUpsertSql(RESTORE_NOVEL_BATCH_SIZE); + +const NOVEL_COVER_UPDATE_SQL = ` + UPDATE Novel + SET cover = ? || pluginId || '/' || id || '/cover.png' || ? + WHERE pluginId = ? AND path = ? +`; + +const restoreNovelValues = (novel: BackupNovel): Scalar[] => [ + novel.path, + novel.pluginId, + novel.name, + novel.cover ?? null, + novel.summary ?? null, + novel.author ?? null, + novel.artist ?? null, + novel.status ?? null, + novel.genres ?? null, + sqliteBoolean(novel.inLibrary), + sqliteBoolean(novel.isLocal), + novel.totalPages ?? null, +]; + +const restoreNovelChunk = async ( + backupNovels: BackupNovel[], + metrics?: RestoreNovelMetrics, +): Promise => { + const commands: SQLBatchTuple[] = [ + [ + backupNovels.length === RESTORE_NOVEL_BATCH_SIZE + ? NOVEL_UPSERT_SQL + : buildNovelUpsertSql(backupNovels.length), + backupNovels.flatMap(restoreNovelValues), + ], + ]; + const storedCovers = backupNovels + .filter(novel => novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) + .map( + novel => + [ + `file://${NOVEL_STORAGE}/`, + novel.cover?.match(/[?#].*$/)?.[0] ?? '', + novel.pluginId, + novel.path, + ] as Scalar[], + ); + if (storedCovers.length > 0) { + commands.push([NOVEL_COVER_UPDATE_SQL, storedCovers]); + } + const upsertStartedAt = metrics ? performance.now() : 0; + if (metrics) { + metrics.novelUpsertCalls++; + metrics.novelUpsertRowsAttempted += backupNovels.length; + } + try { + await dbManager.executeBatch(commands); + } finally { + if (metrics) { + metrics.novelUpsertMs += performance.now() - upsertStartedAt; + } + } + + if (metrics) { + metrics.novelIdentityLookupCalls++; + } + const lookupStartedAt = metrics ? performance.now() : 0; + let rows; + try { + rows = await dbManager + .select({ + id: novelSchema.id, + path: novelSchema.path, + pluginId: novelSchema.pluginId, + }) + .from(novelSchema) + .where( + or( + ...backupNovels.map(novel => + and( + eq(novelSchema.pluginId, novel.pluginId), + eq(novelSchema.path, novel.path), + ), + ), + ), + ) + .all(); + if (metrics) { + metrics.novelIdentityLookupRowsReturned += rows.length; + } + } finally { + if (metrics) { + metrics.novelIdentityLookupMs += performance.now() - lookupStartedAt; + } + } + const rowsByIdentity = new Map( + rows.map(row => [`${row.pluginId}\u0000${row.path}`, row]), + ); + return backupNovels.map(backupNovel => { + const row = rowsByIdentity.get( + `${backupNovel.pluginId}\u0000${backupNovel.path}`, + ); + if (!row) { + throw new Error('Restore returned incomplete novel mapping'); + } + return { + pluginId: backupNovel.pluginId, + backupNovelId: backupNovel.id, + restoredNovelId: row.id, + }; + }); +}; + +const restoreNovelChunkWithRetry = async ( + backupNovels: BackupNovel[], + metrics?: RestoreNovelMetrics, +): Promise => { + try { + return await restoreNovelChunk(backupNovels, metrics); + } catch (error) { + if (metrics) { + metrics.novelBatchFallbacks++; + } + recordRestoreFallback(metrics, error); + const mappings: RestoredNovelMapping[] = []; + let failed = false; + for (const backupNovel of backupNovels) { + if (metrics) { + metrics.novelFallbackRowsAttempted++; + } + try { + mappings.push( + await restoreNovelChunk([backupNovel], metrics).then(([mapping]) => { + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; + }), + ); + } catch (rowError) { + recordRestoreFallback(metrics, rowError); + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more novels'); + } + return mappings; + } +}; + +const appendRestoreChapterValues = ( + values: Scalar[], + chapter: BackupNovel['chapters'][number], + restoredNovelId: number, +) => { + values.push( + restoredNovelId, + chapter.path, + chapter.name, + chapter.releaseTime ?? null, + sqliteBoolean(chapter.bookmark), + sqliteBoolean(chapter.unread), + chapter.readTime ?? null, + sqliteBoolean(chapter.isDownloaded), + chapter.updatedTime ?? null, + chapter.chapterNumber ?? null, + chapter.page ?? null, + chapter.position ?? null, + chapter.progress ?? null, + chapter.scanlator ?? null, + chapter.timeSpent ?? null, + ); +}; + +type ChapterRestoreRecord = { + backupNovelId: number; + backupChapterId: number; + restoredNovelId: number; + chapter: BackupNovel['chapters'][number]; +}; + +const RESTORE_CHAPTER_STATEMENT_SIZE = 100; + +const buildChapterUpsertSql = (rowCount: number) => ` + INSERT INTO Chapter ( + novelId, path, name, releaseTime, bookmark, unread, readTime, + isDownloaded, updatedTime, chapterNumber, page, position, progress, + scanlator, timeSpent + ) VALUES ${createPlaceholderRows(rowCount, 15)} + ON CONFLICT(novelId, path) DO UPDATE SET + novelId = excluded.novelId, + path = excluded.path, + name = excluded.name, + releaseTime = excluded.releaseTime, + bookmark = excluded.bookmark, + unread = excluded.unread, + readTime = excluded.readTime, + isDownloaded = excluded.isDownloaded, + updatedTime = excluded.updatedTime, + chapterNumber = excluded.chapterNumber, + page = excluded.page, + position = excluded.position, + progress = excluded.progress, + scanlator = excluded.scanlator, + timeSpent = excluded.timeSpent +`; + +const CHAPTER_UPSERT_SQL = buildChapterUpsertSql( + RESTORE_CHAPTER_STATEMENT_SIZE, +); + +const buildRestoreChapterMappingInsertSql = (rowCount: number) => ` + WITH restoreInputs( + ordinal, restoreRunId, backupNovelId, backupChapterId, + restoredNovelId, chapterPath + ) AS ( + VALUES ${Array.from( + { length: rowCount }, + (_, ordinal) => `(${ordinal}, ?, ?, ?, ?, ?)`, + ).join(',\n')} + ) + INSERT INTO RestoreChapterMapping ( + restoreRunId, backupNovelId, backupChapterId, + restoredNovelId, restoredChapterId + ) + SELECT + restoreInputs.restoreRunId, + restoreInputs.backupNovelId, + restoreInputs.backupChapterId, + restoreInputs.restoredNovelId, + chapter.id + FROM restoreInputs + JOIN Chapter AS chapter + ON chapter.novelId = restoreInputs.restoredNovelId + AND chapter.path = restoreInputs.chapterPath + WHERE 1 + ORDER BY restoreInputs.ordinal + ON CONFLICT(restoreRunId, backupNovelId, backupChapterId) DO UPDATE SET + restoredNovelId = excluded.restoredNovelId, + restoredChapterId = excluded.restoredChapterId +`; +const RESTORE_CHAPTER_MAPPING_INSERT_SQL = buildRestoreChapterMappingInsertSql( + RESTORE_CHAPTER_STATEMENT_SIZE, +); + +const restoreChapterChunk = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, + metrics?: RestoreNovelMetrics, +) => { + if (records.length === 0) { + return; + } + if (includeChapterMappings && !restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const commands: SQLBatchTuple[] = [ + ['DROP TRIGGER IF EXISTS update_novel_stats'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_update'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_delete'], + ]; + for ( + let start = 0; + start < records.length; + start += RESTORE_CHAPTER_STATEMENT_SIZE + ) { + const rowCount = Math.min( + RESTORE_CHAPTER_STATEMENT_SIZE, + records.length - start, + ); + const values: Scalar[] = []; + for (let index = start; index < start + rowCount; index++) { + const record = records[index]; + appendRestoreChapterValues( + values, + record.chapter, + record.restoredNovelId, + ); + } + const chapterUpsertSql = + rowCount === RESTORE_CHAPTER_STATEMENT_SIZE + ? CHAPTER_UPSERT_SQL + : buildChapterUpsertSql(rowCount); + commands.push([chapterUpsertSql, values]); + } + if (includeChapterMappings) { + for ( + let start = 0; + start < records.length; + start += RESTORE_CHAPTER_STATEMENT_SIZE + ) { + const rowCount = Math.min( + RESTORE_CHAPTER_STATEMENT_SIZE, + records.length - start, + ); + const values: Scalar[] = []; + for (let index = start; index < start + rowCount; index++) { + const record = records[index]; + values.push( + restoreRunId as string, + record.backupNovelId, + record.backupChapterId, + record.restoredNovelId, + record.chapter.path, + ); + } + const mappingInsertSql = + rowCount === RESTORE_CHAPTER_STATEMENT_SIZE + ? RESTORE_CHAPTER_MAPPING_INSERT_SQL + : buildRestoreChapterMappingInsertSql(rowCount); + commands.push([mappingInsertSql, values]); + } + } + commands.push( + [createNovelTriggerQueryInsert], + [createNovelTriggerQueryDelete], + [createNovelTriggerQueryUpdate], + ); + if (metrics) { + metrics.chapterWriteChunkCalls++; + metrics.chapterWriteRowsAttempted += records.length; + if (includeChapterMappings) { + metrics.chapterMappingRowsAttempted += records.length; + } + } + const chapterWriteStartedAt = metrics ? performance.now() : 0; + try { + await dbManager.executeBatch(commands); + } finally { + if (metrics) { + metrics.chapterWriteMs += performance.now() - chapterWriteStartedAt; + } + } +}; + +const restoreChapterChunkWithRetry = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, + metrics?: RestoreNovelMetrics, +) => { + try { + await restoreChapterChunk( + records, + includeChapterMappings, + restoreRunId, + metrics, + ); + } catch (error) { + if (metrics) { + metrics.chapterBatchFallbacks++; + } + recordRestoreFallback(metrics, error); + let failed = false; + for (const record of records) { + if (metrics) { + metrics.chapterFallbackRowsAttempted++; + } + try { + await restoreChapterChunk( + [record], + includeChapterMappings, + restoreRunId, + metrics, + ); + } catch (rowError) { + recordRestoreFallback(metrics, rowError); + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more chapters'); + } + } +}; + +const NOVEL_STATS_UPDATE_SQL = ` + UPDATE Novel + SET totalChapters = ( + SELECT COUNT(*) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + chaptersDownloaded = COALESCE(( + SELECT SUM(CASE WHEN Chapter.isDownloaded = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + chaptersUnread = COALESCE(( + SELECT SUM(CASE WHEN Chapter.unread = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + lastReadAt = ( + SELECT MAX(Chapter.readTime) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + lastUpdatedAt = ( + SELECT updatedChapter.updatedTime + FROM Chapter AS updatedChapter + WHERE updatedChapter.novelId = Novel.id + AND updatedChapter.updatedTime IS NOT NULL + ORDER BY julianday(updatedChapter.updatedTime) DESC + LIMIT 1 + ) + WHERE Novel.id = ? +`; + +const refreshRestoredNovelStats = async ( + novelIds: number[], + metrics?: RestoreNovelMetrics, +) => { + if (novelIds.length === 0) { + return; + } + if (metrics) { + metrics.statsRefreshCalls++; + metrics.statsRefreshNovels += novelIds.length; + } + const statsRefreshStartedAt = metrics ? performance.now() : 0; + try { + await dbManager.executeBatch([ + [NOVEL_STATS_UPDATE_SQL, novelIds.map(id => [id] as Scalar[])], + ]); + } finally { + if (metrics) { + metrics.statsRefreshMs += performance.now() - statsRefreshStartedAt; + } + } +}; + +export const _restoreNovelsAndChapters = async ( + backupNovels: BackupNovel[], + options: RestoreNovelOptions = {}, + metrics?: RestoreNovelMetrics, +): Promise => { + if (backupNovels.length === 0) { + return []; + } + const includeChapterMappings = options.includeChapterMappings ?? true; + if (includeChapterMappings && !options.restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const mappings: RestoredNovelMapping[] = []; + for ( + let start = 0; + start < backupNovels.length; + start += RESTORE_NOVEL_BATCH_SIZE + ) { + const chunkMappings = await restoreNovelChunkWithRetry( + backupNovels.slice(start, start + RESTORE_NOVEL_BATCH_SIZE), + metrics, + ); + mappings.push(...chunkMappings); + options.onProgress?.({ + stage: 'novels', + completed: mappings.length, + total: backupNovels.length, + }); + } + + const restoredNovelIds = new Map( + mappings.map(mapping => [mapping.backupNovelId, mapping.restoredNovelId]), + ); + const totalChapters = backupNovels.reduce( + (total, novel) => total + novel.chapters.length, + 0, + ); + let completedChapters = 0; + const chapterChunk: ChapterRestoreRecord[] = []; + for (const backupNovel of backupNovels) { + const restoredNovelId = restoredNovelIds.get(backupNovel.id); + if (restoredNovelId === undefined) { + throw new Error('Missing restored novel mapping'); + } + for (const chapter of backupNovel.chapters) { + chapterChunk.push({ + backupNovelId: backupNovel.id, + backupChapterId: chapter.id, + restoredNovelId, + chapter, + }); + if (chapterChunk.length === RESTORE_CHAPTER_BATCH_SIZE) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + metrics, + ); + completedChapters += chapterChunk.length; + chapterChunk.length = 0; + options.onProgress?.({ + stage: 'chapters', + completed: completedChapters, + total: totalChapters, + }); + } + } + } + if (chapterChunk.length > 0) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + metrics, + ); + completedChapters += chapterChunk.length; + options.onProgress?.({ + stage: 'chapters', + completed: completedChapters, + total: totalChapters, + }); + } else if (totalChapters === 0) { + options.onProgress?.({ + stage: 'chapters', + completed: 0, + total: 0, + }); + } + const restoredIds = [...restoredNovelIds.values()]; + options.onProgress?.({ + stage: 'stats', + completed: 0, + total: restoredIds.length, + }); + await refreshRestoredNovelStats(restoredIds, metrics); + options.onProgress?.({ + stage: 'stats', + completed: restoredIds.length, + total: restoredIds.length, + }); + return mappings; +}; + +/** + * Restores a novel and its chapters from a backup object. + */ +export const _restoreNovelAndChapters = async ( + backupNovel: BackupNovel, + options: RestoreNovelOptions = {}, + metrics?: RestoreNovelMetrics, +): Promise => { + const [mapping] = await _restoreNovelsAndChapters( + [backupNovel], + options, + metrics, + ); + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; +}; + +export const getRestoreChapterMappings = async ( + restoreRunId: string, + backupNovelId: number, + backupChapterIds: number[], +) => { + if (backupChapterIds.length === 0) { + return []; + } + return dbManager + .select({ + backupChapterId: restoreChapterMappingSchema.backupChapterId, + restoredChapterId: restoreChapterMappingSchema.restoredChapterId, + }) + .from(restoreChapterMappingSchema) + .where( + and( + eq(restoreChapterMappingSchema.restoreRunId, restoreRunId), + eq(restoreChapterMappingSchema.backupNovelId, backupNovelId), + inArray(restoreChapterMappingSchema.backupChapterId, backupChapterIds), + ), + ) + .all(); +}; + +export const clearRestoreChapterMappings = async (restoreRunId: string) => { + await dbManager.write(tx => + tx + .delete(restoreChapterMappingSchema) + .where(eq(restoreChapterMappingSchema.restoreRunId, restoreRunId)) + .run(), + ); +}; diff --git a/src/database/queries/__tests__/CategoryQueries.test.ts b/src/database/queries/__tests__/CategoryQueries.test.ts index ae3cb2f5eb..ceb3c58e86 100644 --- a/src/database/queries/__tests__/CategoryQueries.test.ts +++ b/src/database/queries/__tests__/CategoryQueries.test.ts @@ -12,7 +12,11 @@ import { insertTestNovelCategory, clearAllTables, } from './testData'; -import { categorySchema, novelCategorySchema } from '@database/schema'; +import { + categorySchema, + novelCategorySchema, + novelSchema, +} from '@database/schema'; import { eq, sql } from 'drizzle-orm'; import { @@ -86,6 +90,56 @@ describe('CategoryQueries', () => { .map(membership => membership.novelId), ).toEqual(expect.arrayContaining([existingNovelId, restoredNovelId])); }); + it('restores memberships in bounded batches with remapping and conflict-ignore semantics', async () => { + const testDb = getTestDb(); + const existingNovelId = await insertTestNovel(testDb, { + inLibrary: true, + }); + const secondNovelId = await insertTestNovel(testDb, { + inLibrary: true, + }); + const novelIds = [ + ...Array.from({ length: 501 }, (_, index) => index + 1), + 1, + 501, + 987654321, + ]; + const novelIdMap = new Map([ + [1, existingNovelId], + [2, secondNovelId], + ]); + const categoryId = await _restoreCategory( + { + id: 3, + name: 'Batch restored', + sort: 3, + novelIds, + }, + novelIdMap, + ); + + const memberships = await testDb.drizzleDb + .select({ novelId: novelCategorySchema.novelId }) + .from(novelCategorySchema) + .where(eq(novelCategorySchema.categoryId, categoryId)) + .all(); + const membershipIds = memberships.map(membership => membership.novelId); + const expectedMembershipIds = Array.from( + new Set(novelIds.map(id => novelIdMap.get(id) ?? id)), + ).sort((a, b) => a - b); + + expect(membershipIds.sort((a, b) => a - b)).toEqual( + expectedMembershipIds, + ); + expect( + await testDb.drizzleDb + .select({ id: novelSchema.id }) + .from(novelSchema) + .where(eq(novelSchema.id, 987654321)) + .get(), + ).toBeUndefined(); + expect(membershipIds).toContain(987654321); + }); it('does not overwrite a custom category with a colliding backup ID', async () => { const testDb = getTestDb(); diff --git a/src/database/queries/__tests__/ChapterQueries.test.ts b/src/database/queries/__tests__/ChapterQueries.test.ts index 687f833db3..263d790888 100644 --- a/src/database/queries/__tests__/ChapterQueries.test.ts +++ b/src/database/queries/__tests__/ChapterQueries.test.ts @@ -153,9 +153,7 @@ describe('ChapterQueries', () => { describe('getAllNovelChaptersForBackup', () => { it('returns chapters beyond the 1000-row UI query limit', async () => { const testDb = getTestDb(); - // No Novel row is needed for this selector test. Keeping the id orphaned - // also avoids running aggregate-stat triggers 1001 times during setup. - const novelId = 123456; + const novelId = await insertTestNovel(testDb); const values = Array.from( { length: 1001 }, @@ -185,6 +183,33 @@ describe('ChapterQueries', () => { expect(backupChapters).toHaveLength(1001); expect(backupChapters.at(-1)?.name).toBe('Chapter 1001'); }); + + it('returns chapters for multiple novels in novel and chapter order', async () => { + const testDb = getTestDb(); + const firstNovelId = await insertTestNovel(testDb); + const secondNovelId = await insertTestNovel(testDb); + testDb.sqlite.executeSync(` + INSERT INTO Chapter + (novelId, path, name, chapterNumber, page, position) + VALUES + (${firstNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${firstNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2), + (${secondNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${secondNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2) + `); + + const chapters = await getAllNovelChaptersForBackup([ + secondNovelId, + firstNovelId, + ]); + + expect(chapters.map(chapter => [chapter.novelId, chapter.name])).toEqual([ + [firstNovelId, 'Chapter 1'], + [firstNovelId, 'Chapter 2'], + [secondNovelId, 'Chapter 1'], + [secondNovelId, 'Chapter 2'], + ]); + }); }); describe('markChaptersRead', () => { diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index 916daad39b..7185332849 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -14,7 +14,6 @@ import { } from './testData'; import { categorySchema, - chapterSchema, novelCategorySchema, novelSchema, } from '@database/schema'; @@ -30,12 +29,10 @@ import { removeNovelsFromLibrary, getCachedNovels, deleteCachedNovels, - restoreLibrary, updateNovelInfo, pickCustomNovelCover, updateNovelCategoryById, updateNovelCategories, - _restoreNovelAndChapters, } from '../NovelQueries'; const mockGetLibraryDefaultCategoryId = jest.fn(); @@ -421,112 +418,6 @@ describe('NovelQueries', () => { }); }); - describe('restoreLibrary', () => { - it('should restore novel from backup', async () => { - const novel = { - id: 999, - path: '/test/novel', - pluginId: 'test-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 1, - chaptersDownloaded: 0, - chaptersUnread: 0, - totalChapters: 0, - lastReadAt: null, - lastUpdatedAt: null, - }; - - // Mock fetchNovel to return a valid SourceNovel - const { fetchNovel } = require('@services/plugin/fetch'); - jest.mocked(fetchNovel).mockResolvedValueOnce({ - id: undefined, - path: '/test/novel', - name: 'Restored Novel', - chapters: [], - }); - - await restoreLibrary(novel); - - const restored = await getNovelByPath('/test/novel', 'test-plugin'); - expect(restored?.name).toBe('Restored Novel'); - }); - }); - - describe('_restoreNovelAndChapters', () => { - it('does not replace an unrelated novel when backup IDs collide', async () => { - const testDb = getTestDb(); - await insertTestNovel(testDb, { - path: '/existing/novel', - pluginId: 'existing-plugin', - name: 'Existing Novel', - inLibrary: true, - }); - - const mapping = await _restoreNovelAndChapters({ - id: 1, - path: '/restored/novel', - pluginId: 'restored-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 10, - novelId: 1, - path: '/restored/chapter-1', - name: 'Chapter 1', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: null, - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }); - - expect(mapping.restoredNovelId).not.toBe(1); - expect( - (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, - ).toBe('Existing Novel'); - expect( - (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, - ).toBe(mapping.restoredNovelId); - const restoredChapters = await testDb.drizzleDb - .select() - .from(chapterSchema) - .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) - .all(); - expect(mapping.chapters).toEqual([ - { - backupChapterId: 10, - restoredChapterId: restoredChapters[0].id, - }, - ]); - }); - }); - describe('updateNovelInfo', () => { it('should update novel information', async () => { const testDb = getTestDb(); diff --git a/src/database/queries/__tests__/NovelRestoreQueries.test.ts b/src/database/queries/__tests__/NovelRestoreQueries.test.ts new file mode 100644 index 0000000000..e33d7bad23 --- /dev/null +++ b/src/database/queries/__tests__/NovelRestoreQueries.test.ts @@ -0,0 +1,1035 @@ +import './mockDb'; +import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; +import { insertTestNovel, insertTestChapter, clearAllTables } from './testData'; +import { chapterSchema, novelSchema } from '@database/schema'; +import { and, eq } from 'drizzle-orm'; +import type { SQLBatchTuple } from '@op-engineering/op-sqlite'; +import type { BackupNovel, ChapterInfo } from '../../types'; + +import { getNovelByPath } from '../NovelQueries'; +import { + clearRestoreChapterMappings, + getRestoreChapterMappings, + restoreLibrary, + _restoreNovelAndChapters, + _restoreNovelsAndChapters, +} from '../NovelRestoreQueries'; + +const mockGetLibraryDefaultCategoryId = jest.fn(); + +jest.mock('@hooks/persisted/useSettings', () => ({ + getLibraryDefaultCategoryId: () => mockGetLibraryDefaultCategoryId(), +})); + +const createBackupChapter = ( + id: number, + novelId: number, + path: string, + values: Partial = {}, +): ChapterInfo => ({ + id, + novelId, + path, + name: `Chapter ${id}`, + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + ...values, +}); + +const createBackupNovel = ( + id: number, + path: string, + chapters: ChapterInfo[] = [], + pluginId = 'restore-plugin', +): BackupNovel => ({ + id, + path, + pluginId, + name: `Novel ${id}`, + cover: null, + summary: null, + author: null, + artist: null, + status: 'Unknown', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters, +}); + +describe('NovelRestoreQueries', () => { + beforeEach(() => { + const testDb = setupTestDatabase(); + clearAllTables(testDb); + mockGetLibraryDefaultCategoryId.mockReturnValue(undefined); + }); + + afterAll(() => { + teardownTestDatabase(); + }); + describe('restoreLibrary', () => { + it('should restore novel from backup', async () => { + const novel = { + id: 999, + path: '/test/novel', + pluginId: 'test-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 1, + chaptersDownloaded: 0, + chaptersUnread: 0, + totalChapters: 0, + lastReadAt: null, + lastUpdatedAt: null, + }; + + // Mock fetchNovel to return a valid SourceNovel + const { fetchNovel } = require('@services/plugin/fetch'); + jest.mocked(fetchNovel).mockResolvedValueOnce({ + id: undefined, + path: '/test/novel', + name: 'Restored Novel', + chapters: [], + }); + + await restoreLibrary(novel); + + const restored = await getNovelByPath('/test/novel', 'test-plugin'); + expect(restored?.name).toBe('Restored Novel'); + }); + }); + + describe('_restoreNovelAndChapters', () => { + it('does not replace an unrelated novel when backup IDs collide', async () => { + const testDb = getTestDb(); + await insertTestNovel(testDb, { + path: '/existing/novel', + pluginId: 'existing-plugin', + name: 'Existing Novel', + inLibrary: true, + }); + + const mapping = await _restoreNovelAndChapters( + { + id: 1, + path: '/restored/novel', + pluginId: 'restored-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 10, + novelId: 1, + path: '/restored/chapter-1', + name: 'Chapter 1', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { restoreRunId: 'restore-collision' }, + ); + + expect(mapping.restoredNovelId).not.toBe(1); + expect( + (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, + ).toBe('Existing Novel'); + expect( + (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, + ).toBe(mapping.restoredNovelId); + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-1'); + }); + it('reports a completed chapter checkpoint for a batch with no chapters', async () => { + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const mappings = await _restoreNovelsAndChapters( + [ + createBackupNovel(1, '/empty/one'), + createBackupNovel(2, '/empty/two'), + ], + { + includeChapterMappings: false, + onProgress: checkpoint => progress.push(checkpoint), + }, + ); + + expect(mappings).toHaveLength(2); + expect(progress).toEqual([ + { stage: 'novels', completed: 2, total: 2 }, + { stage: 'chapters', completed: 0, total: 0 }, + { stage: 'stats', completed: 0, total: 2 }, + { stage: 'stats', completed: 2, total: 2 }, + ]); + }); + it('restores multiple novels in one batch', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 100, + path: '/bulk/one', + pluginId: 'bulk-plugin', + name: 'Bulk One', + chapters: [], + }, + { + id: 101, + path: '/bulk/two', + pluginId: 'bulk-plugin', + name: 'Bulk Two', + chapters: [], + }, + ], + { includeChapterMappings: false }, + ); + + expect(mappings).toHaveLength(2); + expect(mappings.map(mapping => mapping.pluginId)).toEqual([ + 'bulk-plugin', + 'bulk-plugin', + ]); + expect(await getNovelByPath('/bulk/one', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk One' }), + ); + expect(await getNovelByPath('/bulk/two', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk Two' }), + ); + }); + it('restores 101 novels across a full statement and tail', async () => { + const testDb = getTestDb(); + const unrelatedNovelId = await insertTestNovel(testDb, { + path: '/collision/unrelated', + pluginId: 'unrelated-plugin', + name: 'Unrelated Novel', + }); + const matchingNovelId = await insertTestNovel(testDb, { + path: '/bulk/novel-1', + pluginId: 'restore-plugin', + name: 'Before Restore', + }); + const firstNovelChapters = [ + createBackupChapter(51_001, 1, '/bulk/chapter-1-1', { + updatedTime: '2024-02-01T00:00:00.000Z', + }), + createBackupChapter(51_002, 1, '/bulk/chapter-1-2', { + updatedTime: '2024-02-01 01:00:00', + }), + ]; + const backupNovels = Array.from({ length: 101 }, (_, index) => + createBackupNovel( + index + 1, + `/bulk/novel-${index + 1}`, + index === 0 ? firstNovelChapters : [], + ), + ); + + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const mappings = await _restoreNovelsAndChapters(backupNovels, { + restoreRunId: 'restore-101-novels', + onProgress: checkpoint => progress.push(checkpoint), + }); + expect(progress.filter(({ stage }) => stage === 'novels')).toEqual([ + { stage: 'novels', completed: 100, total: 101 }, + { stage: 'novels', completed: 101, total: 101 }, + ]); + expect(progress.filter(({ stage }) => stage === 'chapters')).toEqual([ + { stage: 'chapters', completed: 2, total: 2 }, + ]); + expect(progress.filter(({ stage }) => stage === 'stats')).toEqual([ + { stage: 'stats', completed: 0, total: 101 }, + { stage: 'stats', completed: 101, total: 101 }, + ]); + + expect(mappings.map(mapping => mapping.backupNovelId)).toEqual( + Array.from({ length: 101 }, (_, index) => index + 1), + ); + expect(mappings[0].restoredNovelId).toBe(matchingNovelId); + expect(mappings[0].restoredNovelId).not.toBe(unrelatedNovelId); + const restoredNovels = await testDb.drizzleDb + .select() + .from(novelSchema) + .all(); + const restoredIdsByIdentity = new Map( + restoredNovels.map(novel => [ + `${novel.pluginId}\u0000${novel.path}`, + novel.id, + ]), + ); + expect( + mappings.map(mapping => + restoredIdsByIdentity.get( + `restore-plugin\u0000/bulk/novel-${mapping.backupNovelId}`, + ), + ), + ).toEqual(mappings.map(mapping => mapping.restoredNovelId)); + expect( + await getNovelByPath('/collision/unrelated', 'unrelated-plugin'), + ).toEqual( + expect.objectContaining({ + id: unrelatedNovelId, + name: 'Unrelated Novel', + }), + ); + expect(await getNovelByPath('/bulk/novel-1', 'restore-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + lastUpdatedAt: '2024-02-01 01:00:00', + }), + ); + expect(await getNovelByPath('/bulk/novel-101', 'restore-plugin')).toEqual( + expect.objectContaining({ totalChapters: 0 }), + ); + }); + it('reports each completed 10,000-chapter database chunk', async () => { + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const chapters = Array.from({ length: 10_001 }, (_, index) => + createBackupChapter( + index + 1, + 1, + `/restore/checkpoints/chapter-${index + 1}`, + ), + ); + + const [mapping] = await _restoreNovelsAndChapters( + [createBackupNovel(1, '/restore/checkpoints', chapters)], + { + includeChapterMappings: false, + onProgress: checkpoint => progress.push(checkpoint), + }, + ); + + expect(progress.filter(({ stage }) => stage === 'chapters')).toEqual([ + { stage: 'chapters', completed: 10_000, total: 10_001 }, + { stage: 'chapters', completed: 10_001, total: 10_001 }, + ]); + expect( + await getNovelByPath('/restore/checkpoints', 'restore-plugin'), + ).toEqual(expect.objectContaining({ totalChapters: 10_001 })); + const persistedChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(persistedChapters).toHaveLength(10_001); + }); + it('restores 201 chapters with stable identities and ordered mappings', async () => { + const testDb = getTestDb(); + const restoredNovelId = await insertTestNovel(testDb, { + path: '/restore/large', + pluginId: 'large-plugin', + name: 'Before Restore', + }); + const stableChapterId = await insertTestChapter(testDb, restoredNovelId, { + path: '/restore/large/chapter-0', + name: 'Before Chapter', + readTime: '2023-01-01T00:00:00.000Z', + updatedTime: '2022-01-01T00:00:00.000Z', + }); + const existingOnlyChapterId = await insertTestChapter( + testDb, + restoredNovelId, + { + path: '/restore/large/existing-only', + name: 'Existing Only', + readTime: '2024-01-01T00:00:00.000Z', + updatedTime: '2023-01-01T00:00:00.000Z', + }, + ); + const chapters = Array.from({ length: 201 }, (_, index) => { + const backupChapterId = + index === 1 || index === 2 + ? 7000 + : index === 99 || index === 100 + ? 8000 + : 10_000 + index; + return createBackupChapter( + backupChapterId, + 999, + `/restore/large/chapter-${index}`, + { + unread: index % 2 === 0, + isDownloaded: index % 3 === 0, + ...(index === 0 + ? { + name: 'Restored Chapter', + releaseTime: '2024-03-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 10, + page: '8', + progress: 45, + position: 7, + scanlator: 'Backup Scanlator', + timeSpent: 12, + } + : {}), + ...(index === 1 ? { updatedTime: '2024-02-01 01:00:00' } : {}), + ...(index === 2 ? { name: 'Same Statement Last' } : {}), + ...(index === 100 ? { name: 'Across Boundary Last' } : {}), + }, + ); + }); + const backupNovel = createBackupNovel( + 999, + '/restore/large', + chapters, + 'large-plugin', + ); + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const options = { + restoreRunId: 'restore-large', + includeChapterMappings: true, + onProgress: (checkpoint: (typeof progress)[number]) => + progress.push(checkpoint), + }; + + const firstMapping = await _restoreNovelAndChapters(backupNovel, options); + expect(progress).toEqual([ + { stage: 'novels', completed: 1, total: 1 }, + { stage: 'chapters', completed: 201, total: 201 }, + { stage: 'stats', completed: 0, total: 1 }, + { stage: 'stats', completed: 1, total: 1 }, + ]); + expect(firstMapping.restoredNovelId).toBe(restoredNovelId); + + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(202); + const chaptersByPath = new Map( + restoredChapters.map(chapter => [chapter.path, chapter]), + ); + expect(chaptersByPath.get('/restore/large/chapter-0')).toEqual( + expect.objectContaining({ + id: stableChapterId, + name: 'Restored Chapter', + releaseTime: '2024-03-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: true, + isDownloaded: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 10, + page: '8', + progress: 45, + position: 7, + scanlator: 'Backup Scanlator', + timeSpent: 12, + }), + ); + expect(chaptersByPath.get('/restore/large/existing-only')).toEqual( + expect.objectContaining({ + id: existingOnlyChapterId, + name: 'Existing Only', + }), + ); + const mappingRows = await getRestoreChapterMappings( + 'restore-large', + 999, + [10_000, 7000, 8000], + ); + expect(mappingRows).toHaveLength(3); + const mappingsByBackupId = new Map( + mappingRows.map(row => [row.backupChapterId, row]), + ); + expect(mappingsByBackupId.get(10_000)?.restoredChapterId).toBe( + stableChapterId, + ); + expect(mappingsByBackupId.get(7000)?.restoredChapterId).toBe( + chaptersByPath.get('/restore/large/chapter-2')?.id, + ); + expect(mappingsByBackupId.get(8000)?.restoredChapterId).toBe( + chaptersByPath.get('/restore/large/chapter-100')?.id, + ); + expect(await getNovelByPath('/restore/large', 'large-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 202, + chaptersDownloaded: 67, + chaptersUnread: 102, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-02-01 01:00:00', + }), + ); + + await _restoreNovelAndChapters(backupNovel, options); + + const chaptersAfterReplay = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovelId)) + .all(); + expect(chaptersAfterReplay).toHaveLength(202); + expect( + chaptersAfterReplay.map(chapter => chapter.id).sort((a, b) => a - b), + ).toEqual( + restoredChapters.map(chapter => chapter.id).sort((a, b) => a - b), + ); + expect( + await getRestoreChapterMappings('restore-large', 999, [7000, 8000]), + ).toEqual( + expect.arrayContaining([ + { + backupChapterId: 7000, + restoredChapterId: chaptersByPath.get('/restore/large/chapter-2') + ?.id, + }, + { + backupChapterId: 8000, + restoredChapterId: chaptersByPath.get('/restore/large/chapter-100') + ?.id, + }, + ]), + ); + expect( + testDb.sqlite.executeSync('PRAGMA foreign_key_check').rows, + ).toEqual([]); + }); + it('preserves mappings and aggregate stats for multiple novels', async () => { + await _restoreNovelsAndChapters( + [ + { + id: 200, + path: '/bulk/mapped-one', + pluginId: 'bulk-plugin', + name: 'Mapped One', + chapters: [ + { + id: 2001, + novelId: 200, + path: '/bulk/chapter-one', + name: 'Chapter One', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + { + id: 2002, + novelId: 200, + path: '/bulk/chapter-two', + name: 'Chapter Two', + releaseTime: null, + readTime: '2025-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: false, + updatedTime: '2023-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + progress: null, + position: 1, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 201, + path: '/bulk/mapped-two', + pluginId: 'bulk-plugin', + name: 'Mapped Two', + chapters: [ + { + id: 2011, + novelId: 201, + path: '/bulk/chapter-three', + name: 'Chapter Three', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ], + { includeChapterMappings: true, restoreRunId: 'restore-mappings' }, + ); + + const stagedMappingsOne = await getRestoreChapterMappings( + 'restore-mappings', + 200, + [2001, 2002], + ); + const stagedMappingsTwo = await getRestoreChapterMappings( + 'restore-mappings', + 201, + [2011], + ); + expect( + stagedMappingsOne.map(row => row.backupChapterId).sort((a, b) => a - b), + ).toEqual([2001, 2002]); + expect(stagedMappingsTwo.map(row => row.backupChapterId)).toEqual([2011]); + expect( + new Set( + [...stagedMappingsOne, ...stagedMappingsTwo].map( + row => row.restoredChapterId, + ), + ).size, + ).toBe(3); + expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-01-01T00:00:00.000Z', + }), + ); + expect(await getNovelByPath('/bulk/mapped-two', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: null, + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + }); + it('restores chapters without allocating ID mappings when requested', async () => { + const mapping = await _restoreNovelAndChapters( + { + id: 2, + path: '/restored/without-mappings', + pluginId: 'restored-plugin', + name: 'Restored Without Mappings', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 20, + novelId: 2, + path: '/restored/chapter-2', + name: 'Chapter 2', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 2, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + const restoredChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-2'); + expect( + await getRestoreChapterMappings('without-mappings', 2, [20]), + ).toEqual([]); + }); + it('requires a run ID when chapter mappings are enabled', async () => { + const backupNovel = createBackupNovel(1, '/restore/missing-run', [ + createBackupChapter(10, 1, '/restore/missing-run/chapter'), + ]); + + await expect(_restoreNovelAndChapters(backupNovel)).rejects.toThrow( + 'Restore run ID is required for chapter mappings', + ); + expect( + await getNovelByPath('/restore/missing-run', 'restore-plugin'), + ).toBeUndefined(); + }); + it('falls back after a failed chapter batch and restores triggers', async () => { + const testDb = getTestDb(); + const restoredNovelId = await insertTestNovel(testDb, { + path: '/restore/fallback', + pluginId: 'fallback-plugin', + name: 'Before Restore', + }); + const existingChapterId = await insertTestChapter( + testDb, + restoredNovelId, + { + path: '/restore/fallback/existing', + name: 'Before Existing', + }, + ); + const backupNovel = createBackupNovel( + 40, + '/restore/fallback', + [ + createBackupChapter(4001, 40, '/restore/fallback/existing', { + name: 'Restored Existing', + }), + createBackupChapter(4002, 40, '/restore/fallback/new', { + name: 'New Chapter', + }), + ], + 'fallback-plugin', + ); + const executeBatch = testDb.dbManager.executeBatch.bind(testDb.dbManager); + let batchFailureInjected = false; + let triggersRestoredAfterRollback = false; + const executeBatchSpy = jest + .spyOn(testDb.dbManager, 'executeBatch') + .mockImplementation(async commands => { + if ( + !batchFailureInjected && + commands.some(([sql]) => + sql.includes('DROP TRIGGER IF EXISTS update_novel_stats'), + ) + ) { + batchFailureInjected = true; + const failedCommands = commands.slice(); + failedCommands.splice(3, 0, [ + 'INSERT INTO MissingRestoreTable VALUES (1)', + ] as SQLBatchTuple); + try { + return await executeBatch(failedCommands); + } catch (error) { + const triggerRows = testDb.sqlite.executeSync( + `SELECT name FROM sqlite_master + WHERE type = 'trigger' + AND name IN ( + 'update_novel_stats', + 'update_novel_stats_on_update', + 'update_novel_stats_on_delete' + )`, + ).rows; + triggersRestoredAfterRollback = triggerRows.length === 3; + throw error; + } + } + return executeBatch(commands); + }); + + try { + await _restoreNovelAndChapters(backupNovel, { + restoreRunId: 'restore-fallback', + }); + } finally { + executeBatchSpy.mockRestore(); + } + + expect(batchFailureInjected).toBe(true); + expect(triggersRestoredAfterRollback).toBe(true); + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(2); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/fallback/existing', + )?.id, + ).toBe(existingChapterId); + expect( + await getRestoreChapterMappings('restore-fallback', 40, [4001, 4002]), + ).toHaveLength(2); + + await insertTestChapter(testDb, restoredNovelId, { + path: '/restore/fallback/trigger-check', + }); + expect( + await getNovelByPath('/restore/fallback', 'fallback-plugin'), + ).toEqual(expect.objectContaining({ totalChapters: 3 })); + }); + it('merges restored identities without replacing existing rows', async () => { + const testDb = getTestDb(); + const existingNovelId = await insertTestNovel(testDb, { + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Before Restore', + cover: 'before-cover', + summary: 'before-summary', + author: 'Before Author', + artist: 'Before Artist', + status: 'Paused', + genres: 'before', + inLibrary: false, + isLocal: false, + totalPages: 1, + }); + const conflictingChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/chapter', + name: 'Before Chapter', + bookmark: false, + unread: true, + readTime: '2023-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2022-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + position: 0, + progress: 1, + timeSpent: 2, + }, + ); + const unrelatedChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/unrelated', + name: 'Unrelated Chapter', + bookmark: false, + unread: true, + readTime: '2024-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + position: 1, + progress: 0, + timeSpent: 3, + }, + ); + expect(existingNovelId).not.toBe(900); + expect(conflictingChapterId).not.toBe(9001); + + const restoreRunId = 'restore-merge'; + const backupNovel = { + id: 900, + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Restored Novel', + cover: 'file:///mock/novel/storage/cache-key?size=large', + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: true, + isLocal: true, + totalPages: 123, + chapters: [ + { + id: 9001, + novelId: 900, + path: '/restore/merge/chapter', + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }, + ], + }; + + const firstMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(firstMapping).toEqual({ + pluginId: 'merge-plugin', + backupNovelId: 900, + restoredNovelId: existingNovelId, + }); + + const restoredNovel = await getNovelByPath( + '/restore/merge', + 'merge-plugin', + ); + expect(restoredNovel).toEqual( + expect.objectContaining({ + id: existingNovelId, + name: 'Restored Novel', + cover: `file:///mock/novel/storage/merge-plugin/${existingNovelId}/cover.png?size=large`, + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: 1, + isLocal: 1, + totalPages: 123, + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(restoredChapters).toHaveLength(2); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/chapter', + ), + ).toEqual( + expect.objectContaining({ + id: conflictingChapterId, + novelId: existingNovelId, + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }), + ); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/unrelated', + ), + ).toEqual( + expect.objectContaining({ + id: unrelatedChapterId, + name: 'Unrelated Chapter', + }), + ); + + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([ + { + backupChapterId: 9001, + restoredChapterId: conflictingChapterId, + }, + ]); + + const secondMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(secondMapping.restoredNovelId).toBe(existingNovelId); + const novelsAfterReplay = await testDb.drizzleDb + .select() + .from(novelSchema) + .where( + and( + eq(novelSchema.path, '/restore/merge'), + eq(novelSchema.pluginId, 'merge-plugin'), + ), + ) + .all(); + const chaptersAfterReplay = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(novelsAfterReplay).toHaveLength(1); + expect(chaptersAfterReplay).toHaveLength(2); + expect( + chaptersAfterReplay.map(chapter => chapter.id).sort((a, b) => a - b), + ).toEqual( + [conflictingChapterId, unrelatedChapterId].sort((a, b) => a - b), + ); + + await clearRestoreChapterMappings(restoreRunId); + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([]); + expect( + testDb.sqlite.executeSync('PRAGMA foreign_key_check').rows, + ).toEqual([]); + }); + }); +}); diff --git a/src/database/queries/__tests__/index.ts b/src/database/queries/__tests__/index.ts index fe7903cab7..2b9fec59f9 100644 --- a/src/database/queries/__tests__/index.ts +++ b/src/database/queries/__tests__/index.ts @@ -122,12 +122,18 @@ export const testModules = { 'removeNovelsFromLibrary', 'getCachedNovels', 'deleteCachedNovels', - 'restoreLibrary', 'updateNovelInfo', 'pickCustomNovelCover', 'updateNovelCategoryById', 'updateNovelCategories', + ], + }, + NovelRestoreQueries: { + description: 'Novel restore operations', + functions: [ + 'restoreLibrary', '_restoreNovelAndChapters', + '_restoreNovelsAndChapters', ], }, RepositoryQueries: { diff --git a/src/database/queries/__tests__/testData.ts b/src/database/queries/__tests__/testData.ts index 834b2f5d7e..cd2761495d 100644 --- a/src/database/queries/__tests__/testData.ts +++ b/src/database/queries/__tests__/testData.ts @@ -22,6 +22,7 @@ import { export function clearAllTables(testDb: TestDb) { const { sqlite } = testDb; sqlite.executeSync('DELETE FROM NovelCategory'); + sqlite.executeSync('DELETE FROM RestoreChapterMapping'); sqlite.executeSync('DELETE FROM Chapter'); sqlite.executeSync('DELETE FROM Novel'); sqlite.executeSync('DELETE FROM Repository'); diff --git a/src/database/queries/__tests__/testDb.ts b/src/database/queries/__tests__/testDb.ts index 0232c41bdd..dcc43981fd 100644 --- a/src/database/queries/__tests__/testDb.ts +++ b/src/database/queries/__tests__/testDb.ts @@ -30,7 +30,7 @@ const MIGRATION_STATEMENTS = [ `CREATE INDEX IF NOT EXISTS category_sort_idx ON Category (sort)`, `CREATE TABLE IF NOT EXISTS Chapter ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, - novelId integer NOT NULL, + novelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, path text NOT NULL, name text NOT NULL, releaseTime text, @@ -70,6 +70,17 @@ const MIGRATION_STATEMENTS = [ )`, `CREATE UNIQUE INDEX IF NOT EXISTS novel_path_plugin_unique ON Novel (path, pluginId)`, `CREATE INDEX IF NOT EXISTS NovelIndex ON Novel (pluginId, path, id, inLibrary)`, + `CREATE TABLE IF NOT EXISTS RestoreChapterMapping ( + restoreRunId text NOT NULL, + backupNovelId integer NOT NULL, + backupChapterId integer NOT NULL, + restoredNovelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, + restoredChapterId integer NOT NULL REFERENCES Chapter(id) ON DELETE CASCADE +)`, + `CREATE UNIQUE INDEX IF NOT EXISTS restore_chapter_mapping_unique + ON RestoreChapterMapping (restoreRunId, backupNovelId, backupChapterId)`, + `CREATE INDEX IF NOT EXISTS restore_chapter_mapping_novel_index + ON RestoreChapterMapping (restoreRunId, backupNovelId)`, `CREATE TABLE IF NOT EXISTS NovelCategory ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, novelId integer NOT NULL, diff --git a/src/database/schema/chapter.ts b/src/database/schema/chapter.ts index 08354a1282..c4720deb9f 100644 --- a/src/database/schema/chapter.ts +++ b/src/database/schema/chapter.ts @@ -6,12 +6,15 @@ import { text, uniqueIndex, } from 'drizzle-orm/sqlite-core'; +import { novel } from './novel'; export const chapter = sqliteTable( 'Chapter', { id: integer('id').primaryKey({ autoIncrement: true }), - novelId: integer('novelId').notNull(), + novelId: integer('novelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), path: text('path').notNull(), name: text('name').notNull(), releaseTime: text('releaseTime'), diff --git a/src/database/schema/index.ts b/src/database/schema/index.ts index 744c84cd73..2fa0c4edea 100644 --- a/src/database/schema/index.ts +++ b/src/database/schema/index.ts @@ -3,6 +3,7 @@ import { novel } from './novel'; import { chapter } from './chapter'; import { novelCategory } from './novelCategory'; import { repository } from './repository'; +import { restoreChapterMapping } from './restoreChapterMapping'; export { category as categorySchema, @@ -25,6 +26,11 @@ export { type RepositoryRow, type RepositoryInsert, } from './repository'; +export { + restoreChapterMapping as restoreChapterMappingSchema, + type RestoreChapterMappingRow, + type RestoreChapterMappingInsert, +} from './restoreChapterMapping'; /** * Unified schema object containing all database tables @@ -36,6 +42,7 @@ export const schema = { chapter, novelCategory, repository, + restoreChapterMapping, } as const; export type Schema = typeof schema; diff --git a/src/database/schema/restoreChapterMapping.ts b/src/database/schema/restoreChapterMapping.ts new file mode 100644 index 0000000000..c47f1f68f0 --- /dev/null +++ b/src/database/schema/restoreChapterMapping.ts @@ -0,0 +1,40 @@ +import { + index, + integer, + sqliteTable, + text, + uniqueIndex, +} from 'drizzle-orm/sqlite-core'; +import { chapter } from './chapter'; +import { novel } from './novel'; + +export const restoreChapterMapping = sqliteTable( + 'RestoreChapterMapping', + { + restoreRunId: text('restoreRunId').notNull(), + backupNovelId: integer('backupNovelId').notNull(), + backupChapterId: integer('backupChapterId').notNull(), + restoredNovelId: integer('restoredNovelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), + restoredChapterId: integer('restoredChapterId') + .notNull() + .references(() => chapter.id, { onDelete: 'cascade' }), + }, + table => [ + uniqueIndex('restore_chapter_mapping_unique').on( + table.restoreRunId, + table.backupNovelId, + table.backupChapterId, + ), + index('restore_chapter_mapping_novel_index').on( + table.restoreRunId, + table.backupNovelId, + ), + ], +); + +export type RestoreChapterMappingRow = + typeof restoreChapterMapping.$inferSelect; +export type RestoreChapterMappingInsert = + typeof restoreChapterMapping.$inferInsert; diff --git a/src/database/types/index.ts b/src/database/types/index.ts index 349bfaf0bb..5c260c7601 100644 --- a/src/database/types/index.ts +++ b/src/database/types/index.ts @@ -125,10 +125,6 @@ export interface RestoredNovelMapping { pluginId: string; backupNovelId: number; restoredNovelId: number; - chapters: { - backupChapterId: number; - restoredChapterId: number; - }[]; } export interface BackupCategory extends Category { diff --git a/src/hooks/persisted/__mocks__/useSettings.ts b/src/hooks/persisted/__mocks__/useSettings.ts index 8e10d2dd22..34ccf81249 100644 --- a/src/hooks/persisted/__mocks__/useSettings.ts +++ b/src/hooks/persisted/__mocks__/useSettings.ts @@ -44,6 +44,7 @@ export const defaultLibrarySettings = { displayMode: 'comfortable', showDownloadBadges: true, showUnreadBadges: true, + showContinueReadingButton: false, novelsPerRow: 3, sortOrder: 'DateAdded_DESC', }; diff --git a/src/hooks/persisted/useDownload.ts b/src/hooks/persisted/useDownload.ts index b50b1d0630..9c6b63a84e 100644 --- a/src/hooks/persisted/useDownload.ts +++ b/src/hooks/persisted/useDownload.ts @@ -38,9 +38,14 @@ export default function useDownload() { const downloadingNovelIds = useMemo( () => new Set( - downloadQueue.flatMap(item => - item.task.data.novelId === undefined ? [] : [item.task.data.novelId], - ), + downloadQueue.flatMap(item => [ + ...(item.task.data.novelId === undefined + ? [] + : [item.task.data.novelId]), + ...item.task.data.chapters.flatMap(chapter => + chapter.novelId === undefined ? [] : [chapter.novelId], + ), + ]), ), [downloadQueue], ); diff --git a/src/hooks/persisted/useSettings.ts b/src/hooks/persisted/useSettings.ts index fc59ea7701..113d2bf386 100644 --- a/src/hooks/persisted/useSettings.ts +++ b/src/hooks/persisted/useSettings.ts @@ -124,6 +124,7 @@ export interface LibrarySettings { filter?: LibraryFilter; showDownloadBadges?: boolean; showUnreadBadges?: boolean; + showContinueReadingButton?: boolean; showNumberOfNovels?: boolean; displayMode?: DisplayModes; novelsPerRow?: number; @@ -385,6 +386,7 @@ const defaultLibrarySettings: LibrarySettings = { displayMode: DisplayModes.Comfortable, showDownloadBadges: true, showUnreadBadges: true, + showContinueReadingButton: false, novelsPerRow: 3, sortOrder: LibrarySortOrder.DateAdded_DESC, }; diff --git a/src/i18n/languages/en/strings.json b/src/i18n/languages/en/strings.json index 7c39c7bb34..ea003d16f2 100644 --- a/src/i18n/languages/en/strings.json +++ b/src/i18n/languages/en/strings.json @@ -122,9 +122,12 @@ "categoryFileWriteFailed": "Failed to write category file: %{error}", "createBackup": "Create backup", "createBackupDesc": "Can be used to restore current library", + "copyingBackup": "Copying backup", "downloadingData": "Downloading Data", "downloadingDownloadedFiles": "Downloading Downloaded Files", + "extractingBackup": "Extracting backup", "failed": "Backup failed", + "finalizingRestore": "Finishing restore", "novelsRestored": "Restored %{count} novels", "novelsRestoredWithErrors": "Restored %{count} novels (%{failedCount} failed)", "novelsBackupFailedSummary": { @@ -141,8 +144,12 @@ "novelDirectoryReadFailed": "Failed to read novel directory: %{error}", "restoringCategories": "Restoring Categories", "restoringCategoriesProgress": "Restoring categories (%{current}/%{total})", + "validatingNovels": "Validating Novels", + "validatingNovelsProgress": "Validating novels (%{current}/%{total})", "restoringNovels": "Restoring Novels", "restoringNovelsProgress": "Restoring novels (%{current}/%{total})", + "restoringNovelsCount": "Restoring novels (%{current})", + "restoringPlugins": "Restoring plugins", "restoringSettings": "Restoring Settings", "missingPluginsAfterRestore": "Missing plugins: %{plugins}", "sectionsRestoreFailedSummary": { @@ -194,6 +201,7 @@ "remoteBackup": "Remote Backup", "restoreBackup": "Restore backup", "restoreBackupDesc": "Restore library from backup file", + "restoreMayBePartial": "Changes already restored will remain if you pause, cancel, or the restore fails.", "restorinBackup": "Restoring backup", "restoringData": "Restoring Data", "restoringSelectedFiles": "Restoring selected files", @@ -443,6 +451,7 @@ "jumpToLastReadChapter": "Jump to last read chapter in list", "novel": "Novel", "novelBadges": "Novel Badges", + "showContinueReadingButton": "Show continue reading button", "novelSort": "Novel Sort", "refreshMetadata": "Automatically refresh metadata", "refreshMetadataDescription": "Check for new cover and details when updating library", @@ -454,7 +463,8 @@ "updateLibrary": "Update library on launch", "updateLibraryDesc": "Not recommended for low-end devices", "updateTime": "Show last update time", - "useFAB": "Use FAB in Library", + "useFAB": "Show library resume button", + "useFABDescription": "Show a floating button that opens the most recently read chapter.", "timeTracking": "Time Tracking", "enableTimeTracking": "Enable time tracking", "enableTimeTrackingDesc": "Tracks time spent reading and displays it in the statistics screen", @@ -486,12 +496,14 @@ "badges": "Badges", "comfortable": "Comfortable grid", "compact": "Compact grid", + "continueReadingButton": "Continue reading button", "displayMode": "Display mode", "download": "Download", "downloadBadges": "Download badges", "list": "List", "noTitle": "Cover only grid", "numberOfItems": "Number of Items", + "overlays": "Overlays", "showNoOfItems": "Show number of items", "unread": "Unread", "unreadBadges": "Unread badges" diff --git a/src/i18n/types/index.ts b/src/i18n/types/index.ts index 6c9d9322c9..287df5dc67 100644 --- a/src/i18n/types/index.ts +++ b/src/i18n/types/index.ts @@ -109,9 +109,12 @@ export interface StringMap { 'backupScreen.categoryFileWriteFailed': 'string'; 'backupScreen.createBackup': 'string'; 'backupScreen.createBackupDesc': 'string'; + 'backupScreen.copyingBackup': 'string'; 'backupScreen.downloadingData': 'string'; 'backupScreen.downloadingDownloadedFiles': 'string'; + 'backupScreen.extractingBackup': 'string'; 'backupScreen.failed': 'string'; + 'backupScreen.finalizingRestore': 'string'; 'backupScreen.novelsRestored': 'string'; 'backupScreen.novelsRestoredWithErrors': 'string'; 'backupScreen.novelsBackupFailedSummary.one': 'string'; @@ -124,8 +127,12 @@ export interface StringMap { 'backupScreen.novelDirectoryReadFailed': 'string'; 'backupScreen.restoringCategories': 'string'; 'backupScreen.restoringCategoriesProgress': 'string'; + 'backupScreen.validatingNovels': 'string'; + 'backupScreen.validatingNovelsProgress': 'string'; 'backupScreen.restoringNovels': 'string'; 'backupScreen.restoringNovelsProgress': 'string'; + 'backupScreen.restoringNovelsCount': 'string'; + 'backupScreen.restoringPlugins': 'string'; 'backupScreen.restoringSettings': 'string'; 'backupScreen.missingPluginsAfterRestore': 'string'; 'backupScreen.sectionsRestoreFailedSummary.one': 'string'; @@ -167,6 +174,7 @@ export interface StringMap { 'backupScreen.remoteBackup': 'string'; 'backupScreen.restoreBackup': 'string'; 'backupScreen.restoreBackupDesc': 'string'; + 'backupScreen.restoreMayBePartial': 'string'; 'backupScreen.restorinBackup': 'string'; 'backupScreen.restoringData': 'string'; 'backupScreen.restoringSelectedFiles': 'string'; @@ -386,6 +394,7 @@ export interface StringMap { 'generalSettingsScreen.jumpToLastReadChapter': 'string'; 'generalSettingsScreen.novel': 'string'; 'generalSettingsScreen.novelBadges': 'string'; + 'generalSettingsScreen.showContinueReadingButton': 'string'; 'generalSettingsScreen.novelSort': 'string'; 'generalSettingsScreen.refreshMetadata': 'string'; 'generalSettingsScreen.refreshMetadataDescription': 'string'; @@ -398,6 +407,7 @@ export interface StringMap { 'generalSettingsScreen.updateLibraryDesc': 'string'; 'generalSettingsScreen.updateTime': 'string'; 'generalSettingsScreen.useFAB': 'string'; + 'generalSettingsScreen.useFABDescription': 'string'; 'generalSettingsScreen.timeTracking': 'string'; 'generalSettingsScreen.enableTimeTracking': 'string'; 'generalSettingsScreen.enableTimeTrackingDesc': 'string'; @@ -421,12 +431,14 @@ export interface StringMap { 'libraryScreen.bottomSheet.display.badges': 'string'; 'libraryScreen.bottomSheet.display.comfortable': 'string'; 'libraryScreen.bottomSheet.display.compact': 'string'; + 'libraryScreen.bottomSheet.display.continueReadingButton': 'string'; 'libraryScreen.bottomSheet.display.displayMode': 'string'; 'libraryScreen.bottomSheet.display.download': 'string'; 'libraryScreen.bottomSheet.display.downloadBadges': 'string'; 'libraryScreen.bottomSheet.display.list': 'string'; 'libraryScreen.bottomSheet.display.noTitle': 'string'; 'libraryScreen.bottomSheet.display.numberOfItems': 'string'; + 'libraryScreen.bottomSheet.display.overlays': 'string'; 'libraryScreen.bottomSheet.display.showNoOfItems': 'string'; 'libraryScreen.bottomSheet.display.unread': 'string'; 'libraryScreen.bottomSheet.display.unreadBadges': 'string'; diff --git a/src/screens/library/LibraryScreen.tsx b/src/screens/library/LibraryScreen.tsx index c2e68c0ab8..88635affa2 100644 --- a/src/screens/library/LibraryScreen.tsx +++ b/src/screens/library/LibraryScreen.tsx @@ -47,7 +47,7 @@ import { useSafeAreaInsets } from 'react-native-safe-area-context'; import SourceScreenSkeletonLoading from '@screens/browse/loadingAnimation/SourceScreenSkeletonLoading'; import { Row } from '@components/Common'; import { LibraryScreenProps } from '@navigators/types'; -import { NovelInfo } from '@database/types'; +import { History, NovelInfo } from '@database/types'; import * as DocumentPicker from 'expo-document-picker'; import { backgroundTasks } from '@services/backgroundTasks'; import useImport from '@hooks/persisted/useImport'; @@ -91,6 +91,7 @@ const LibraryScreen = ({ navigation }: LibraryScreenProps) => { error: libraryError, settings: { showNumberOfNovels, + showContinueReadingButton = false, downloadedOnlyMode, incognitoMode, lastUsedCategoryId, @@ -102,6 +103,11 @@ const LibraryScreen = ({ navigation }: LibraryScreenProps) => { const { useLibraryFAB = false } = useAppSettings(); const { isLoading: isHistoryLoading, history, error } = useHistory(); + const historyByNovelId = useMemo(() => { + const chapters = new Map(); + history.forEach(chapter => chapters.set(chapter.novelId, chapter)); + return chapters; + }, [history]); const layout = useWindowDimensions(); @@ -282,12 +288,16 @@ const LibraryScreen = ({ navigation }: LibraryScreenProps) => { novels={novels} pickAndImport={pickAndImport} navigation={navigation} + historyByNovelId={historyByNovelId} + showContinueReadingButton={showContinueReadingButton} /> ); }, [ library, + historyByNovelId, + showContinueReadingButton, navigation, pickAndImport, searchText, diff --git a/src/screens/library/components/LibraryBottomSheet/LibraryBottomSheet.tsx b/src/screens/library/components/LibraryBottomSheet/LibraryBottomSheet.tsx index dddde0e403..1b25f9d8e0 100644 --- a/src/screens/library/components/LibraryBottomSheet/LibraryBottomSheet.tsx +++ b/src/screens/library/components/LibraryBottomSheet/LibraryBottomSheet.tsx @@ -110,6 +110,7 @@ const ThirdRoute = () => { showDownloadBadges = true, showNumberOfNovels = false, showUnreadBadges = true, + showContinueReadingButton = false, displayMode = DisplayModes.Comfortable, setLibrarySettings, } = useLibrarySettings(); @@ -117,7 +118,7 @@ const ThirdRoute = () => { return ( - {getString('libraryScreen.bottomSheet.display.badges')} + {getString('libraryScreen.bottomSheet.display.overlays')} { } theme={theme} /> + + setLibrarySettings({ + showContinueReadingButton: !showContinueReadingButton, + }) + } + theme={theme} + /> {getString('libraryScreen.bottomSheet.display.displayMode')} diff --git a/src/screens/library/components/LibraryListView.tsx b/src/screens/library/components/LibraryListView.tsx index 5ccfe7ec54..65755b2f2e 100644 --- a/src/screens/library/components/LibraryListView.tsx +++ b/src/screens/library/components/LibraryListView.tsx @@ -5,7 +5,7 @@ import { EmptyView } from '@components/index'; import NovelList, { NovelListRenderItem } from '@components/NovelList'; import LibraryNovelItem from './LibraryNovelItem'; -import { NovelInfo } from '@database/types'; +import { History, NovelInfo } from '@database/types'; import { getString } from '@i18n/translations'; import { useTheme } from '@hooks/persisted'; @@ -19,6 +19,8 @@ interface Props { categoryId: number; categoryName: string; novels: NovelInfo[]; + historyByNovelId: Map; + showContinueReadingButton: boolean; navigation: LibraryScreenProps['navigation']; pickAndImport: () => void; } @@ -29,6 +31,8 @@ export const LibraryView: React.FC = ({ pickAndImport, navigation, novels, + historyByNovelId, + showContinueReadingButton, }) => { const theme = useTheme(); const { selectedIdsSet, hasSelection, toggleSelection } = @@ -44,6 +48,16 @@ export const LibraryView: React.FC = ({ [navigation], ); + const onContinueReading = useCallback( + (item: NovelInfo, chapter: History) => { + navigation.navigate('ReaderStack', { + screen: 'Chapter', + params: { novel: item, chapter }, + }); + }, + [navigation], + ); + const imageRequestInitMap = useMemo(() => { const map = new Map(); for (const novel of novels) { @@ -54,6 +68,11 @@ export const LibraryView: React.FC = ({ return map; }, [novels]); + const extraData = useMemo( + () => ({ selectedIdsSet, historyByNovelId }), + [selectedIdsSet, historyByNovelId], + ); + const renderItem = useCallback( ({ item }: { item: NovelInfo }) => ( = ({ hasSelection={hasSelection} onSelect={toggleSelection} onNavigate={onNavigate} + onContinueReading={onContinueReading} + lastReadChapter={historyByNovelId.get(item.id)} + showContinueReadingButton={showContinueReadingButton} imageRequestInit={imageRequestInitMap.get(item.pluginId)} /> ), @@ -72,6 +94,9 @@ export const LibraryView: React.FC = ({ hasSelection, toggleSelection, onNavigate, + onContinueReading, + historyByNovelId, + showContinueReadingButton, imageRequestInitMap, ], ); @@ -126,7 +151,7 @@ export const LibraryView: React.FC = ({ void; onNavigate: (item: NovelInfo) => void; + onContinueReading: (item: NovelInfo, chapter: History) => void; + lastReadChapter?: History; + showContinueReadingButton: boolean; imageRequestInit: ImageRequestInit | undefined; } @@ -21,6 +24,9 @@ const LibraryNovelItem = memo(function LibraryNovelItem_({ hasSelection, onSelect, onNavigate, + onContinueReading, + lastReadChapter, + showContinueReadingButton, imageRequestInit, }: LibraryNovelItemProps) { const handleLongPress = useCallback(() => { @@ -35,6 +41,12 @@ const LibraryNovelItem = memo(function LibraryNovelItem_({ } }, [hasSelection, item, onSelect, onNavigate]); + const handleContinueReading = useCallback(() => { + if (lastReadChapter) { + onContinueReading(item, lastReadChapter); + } + }, [item, lastReadChapter, onContinueReading]); + return ( diff --git a/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx b/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx index 64613a7b1a..a49cbef933 100644 --- a/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx +++ b/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx @@ -10,7 +10,10 @@ import { showToast } from '@utils/showToast'; import { getString } from '@i18n/translations'; import { exists, getBackups, makeDir } from '@api/drive'; import { DriveFile } from '@api/drive/types'; -import { backgroundTasks } from '@services/backgroundTasks'; +import { + backgroundTasks, + type BackgroundTask, +} from '@services/backgroundTasks'; import { useAppSettings } from '@hooks/persisted'; import { formatDate } from '@utils/dateFormat'; import { @@ -20,6 +23,11 @@ import { } from '@services/backup/options'; import { BackupOptionsList } from './BackupOptions'; +type RestoreTask = Extract< + BackgroundTask, + { name: 'LOCAL_RESTORE' | 'DRIVE_RESTORE' | 'SELF_HOST_RESTORE' } +>; + enum BackupModal { UNAUTHORIZED, AUTHORIZED, @@ -174,10 +182,12 @@ function RestoreBackup({ theme, setBackupModal, closeModal, + onRestoreSelected, }: { theme: ThemeColors; setBackupModal: (backupModal: BackupModal) => void; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; }) { const [backupList, setBackupList] = useState([]); const { dateFormat = 'default', relativeTimestamps = true } = @@ -212,10 +222,7 @@ function RestoreBackup({ style={styles.btnOutline} onPress={() => { closeModal(); - backgroundTasks.enqueue({ - name: 'DRIVE_RESTORE', - data: item, - }); + onRestoreSelected({ name: 'DRIVE_RESTORE', data: item }); }} > @@ -245,12 +252,14 @@ interface GoogleDriveModalProps { visible: boolean; theme: ThemeColors; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; } export default function GoogleDriveModal({ visible, theme, closeModal, + onRestoreSelected, }: GoogleDriveModalProps) { const [backupModal, setBackupModal] = useState(BackupModal.UNAUTHORIZED); const [user, setUser] = useState(null); @@ -303,6 +312,7 @@ export default function GoogleDriveModal({ theme={theme} setBackupModal={setBackupModal} closeModal={closeModal} + onRestoreSelected={onRestoreSelected} /> ); } diff --git a/src/screens/settings/SettingsBackupScreen/Components/RestoreConfirmationDialog.tsx b/src/screens/settings/SettingsBackupScreen/Components/RestoreConfirmationDialog.tsx new file mode 100644 index 0000000000..2709e28548 --- /dev/null +++ b/src/screens/settings/SettingsBackupScreen/Components/RestoreConfirmationDialog.tsx @@ -0,0 +1,33 @@ +import { Dialog } from '@components'; +import { getString } from '@i18n/translations'; + +interface RestoreConfirmationDialogProps { + visible: boolean; + onCancel: () => void; + onRestore: () => void; +} + +const RestoreConfirmationDialog = ({ + visible, + onCancel, + onRestore, +}: RestoreConfirmationDialogProps) => ( + + + {getString('backupScreen.restoreBackup')} + + {getString('backupScreen.restoreMayBePartial')} + + + + + {getString('common.cancel')} + + + {getString('common.restore')} + + + +); + +export default RestoreConfirmationDialog; diff --git a/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx b/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx index d3ae2a8a25..bde005063c 100644 --- a/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx +++ b/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx @@ -1,7 +1,10 @@ import { list } from '@api/remote'; import { Button, Dialog, EmptyView } from '@components'; import { useSelfHost } from '@hooks/persisted/useSelfHost'; -import { backgroundTasks } from '@services/backgroundTasks'; +import { + backgroundTasks, + type BackgroundTask, +} from '@services/backgroundTasks'; import { getString } from '@i18n/translations'; import { ThemeColors } from '@theme/types'; import { fetchTimeout } from '@utils/fetch/fetch'; @@ -16,6 +19,11 @@ import { } from '@services/backup/options'; import { BackupOptionsList } from './BackupOptions'; +type RestoreTask = Extract< + BackgroundTask, + { name: 'LOCAL_RESTORE' | 'DRIVE_RESTORE' | 'SELF_HOST_RESTORE' } +>; + enum BackupModal { SET_HOST, CONNECTED, @@ -27,6 +35,7 @@ interface SelfHostModalProps { visible: boolean; theme: ThemeColors; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; } function CreateBackup({ @@ -93,11 +102,13 @@ function RestoreBackup({ theme, setBackupModal, closeModal, + onRestoreSelected, }: { host: string; theme: ThemeColors; setBackupModal: (backupModal: BackupModal) => void; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; }) { const [backupList, setBackupList] = useState([]); useEffect(() => { @@ -128,7 +139,7 @@ function RestoreBackup({ style={styles.btnOutline} onPress={() => { closeModal(); - backgroundTasks.enqueue({ + onRestoreSelected({ name: 'SELF_HOST_RESTORE', data: { host, @@ -249,6 +260,7 @@ export default function SelfHostModal({ visible, theme, closeModal, + onRestoreSelected, }: SelfHostModalProps) { const [backupModal, setBackupModal] = useState(BackupModal.SET_HOST); const { host, setHost } = useSelfHost(); @@ -282,6 +294,7 @@ export default function SelfHostModal({ closeModal={closeModal} setBackupModal={setBackupModal} theme={theme} + onRestoreSelected={onRestoreSelected} /> ); } diff --git a/src/screens/settings/SettingsBackupScreen/__tests__/SettingsBackupScreen.test.tsx b/src/screens/settings/SettingsBackupScreen/__tests__/SettingsBackupScreen.test.tsx new file mode 100644 index 0000000000..f4150857ab --- /dev/null +++ b/src/screens/settings/SettingsBackupScreen/__tests__/SettingsBackupScreen.test.tsx @@ -0,0 +1,428 @@ +import { fireEvent, render, screen } from '@test-utils'; +import { type ElementType, type ReactElement, type ReactNode } from 'react'; +import { backgroundTasks } from '@services/backgroundTasks'; +import SettingsBackupScreen from '../index'; + +type MockReactModule = { + Fragment: ElementType; + createElement: ( + type: ElementType, + props?: Record | null, + ...children: ReactNode[] + ) => ReactElement; + useState(initialState: T): [T, (value: T | ((previous: T) => T)) => void]; +}; + +type MockReactNativeModule = { + Pressable: ElementType; + Text: ElementType; + TextInput: ElementType; + View: ElementType; +}; + +type MockChildrenProps = { children?: ReactNode }; +type MockButtonProps = MockChildrenProps & { + disabled?: boolean; + onPress?: () => void; + title?: string; +}; +type MockDialogRootProps = MockChildrenProps & { visible: boolean }; +type MockListItemProps = { onPress?: () => void; title: string }; +type MockListInfoItemProps = { title: string }; +type MockFlatListProps = { + data: readonly Item[]; + renderItem: (info: { item: Item; index: number }) => ReactNode; +}; + +jest.mock('@screens/novel/NovelContext', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const NovelContextProvider = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(ReactRuntime.Fragment, null, children); + }; + return { NovelContextProvider }; +}); + +const partialRestoreWarning = + 'Changes already restored will remain if you pause, cancel, or the restore fails.'; + +jest.mock('@hooks/persisted', () => ({ + useAppSettings() { + 'use no memo'; + return { + automaticBackupIntervalHours: 0, + automaticBackupDirectoryName: undefined, + automaticBackupDirectoryUri: undefined, + dateFormat: 'default', + lastAutomaticBackupAt: undefined, + relativeTimestamps: true, + setAppSettings: jest.fn(), + }; + }, + useTheme() { + 'use no memo'; + return { + background: '#111111', + error: '#cc0000', + onSurface: '#222222', + onSurfaceDisabled: '#666666', + onSurfaceVariant: '#444444', + outline: '#555555', + primary: '#0000cc', + rippleColor: '#dddddd', + secondary: '#777777', + }; + }, +})); + +jest.mock('@hooks/persisted/useTheme', () => { + 'use no memo'; + return { + ThemeProvider: ({ children }: MockChildrenProps) => { + 'use no memo'; + return children; + }, + useTheme() { + 'use no memo'; + return { + background: '#111111', + error: '#cc0000', + onSurface: '#222222', + onSurfaceDisabled: '#666666', + onSurfaceVariant: '#444444', + outline: '#555555', + primary: '#0000cc', + rippleColor: '#dddddd', + secondary: '#777777', + }; + }, + }; +}); + +jest.mock('@hooks', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const useBoolean = () => { + 'use no memo'; + const [value, setValue] = ReactRuntime.useState(false); + return { + value, + setFalse() { + setValue(false); + }, + setTrue() { + setValue(true); + }, + }; + }; + + return { useBoolean }; +}); + +jest.mock('@hooks/persisted/useSelfHost', () => ({ + useSelfHost() { + 'use no memo'; + return { + host: 'https://backup.example', + setHost: jest.fn(), + }; + }, +})); + +jest.mock('@services/backgroundTasks', () => ({ + backgroundTasks: { enqueue: jest.fn() }, + configureAutomaticBackups: jest.fn().mockResolvedValue(undefined), +})); + +jest.mock('@modules/native-file', () => ({ + __esModule: true, + default: { + ExternalDirectoryPath: '/backups', + createDocument: jest.fn(), + pickDirectory: jest.fn(), + pickDocument: jest.fn().mockResolvedValue('file://local-backup.zip'), + }, +})); + +jest.mock('@api/drive', () => ({ + exists: jest.fn().mockResolvedValue({ id: 'drive-root' }), + getBackups: jest.fn().mockResolvedValue([ + { + id: 'drive-archive-id', + name: 'drive-archive.backup', + createdTime: '2026-09-24T00:00:00.000Z', + }, + ]), + makeDir: jest.fn(), +})); + +jest.mock('@api/remote', () => ({ + list: jest.fn().mockResolvedValue(['selfhost-archive.backup']), +})); + +jest.mock('@utils/fetch/fetch', () => ({ + fetchTimeout: jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ name: 'LNReader' }), + }), +})); + +jest.mock('@utils/dateFormat', () => ({ + formatDate: jest.fn(() => 'Sep 24'), +})); + +jest.mock('@utils/showToast', () => ({ showToast: jest.fn() })); + +jest.mock('@i18n/translations', () => { + 'use no memo'; + return { + getString(key: string) { + return key === 'backupScreen.restoreMayBePartial' + ? 'Changes already restored will remain if you pause, cancel, or the restore fails.' + : key; + }, + }; +}); + +jest.mock('@react-native-google-signin/google-signin', () => ({ + GoogleSignin: { + configure: jest.fn(), + getCurrentUser: jest.fn(() => ({ user: { email: 'reader@example.com' } })), + hasPreviousSignIn: jest.fn(() => true), + }, +})); + +jest.mock('expo-clipboard', () => ({ + setStringAsync: jest.fn().mockResolvedValue(true), +})); + +jest.mock('@components', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const { Pressable, Text, View } = + jest.requireActual('react-native'); + + const ViewContent = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(View, null, children); + }; + const TextContent = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(Text, null, children); + }; + const Button = ({ children, disabled, onPress, title }: MockButtonProps) => { + 'use no memo'; + return ReactRuntime.createElement( + Pressable, + { accessibilityRole: 'button', disabled, onPress }, + title + ? ReactRuntime.createElement(Text, null, title) + : ReactRuntime.createElement(Text, null, children), + ); + }; + const Dialog = { + Root: ({ children, visible }: MockDialogRootProps) => { + 'use no memo'; + return visible ? ReactRuntime.createElement(View, null, children) : null; + }, + Header: ViewContent, + Title: TextContent, + Description: TextContent, + Content: ViewContent, + Actions: ViewContent, + Action: Button, + ScrollArea: ViewContent, + }; + const List = { + Section: ViewContent, + SubHeader: TextContent, + Item: ({ onPress, title }: MockListItemProps) => { + 'use no memo'; + return ReactRuntime.createElement( + Pressable, + { accessibilityRole: 'button', onPress }, + ReactRuntime.createElement(Text, null, title), + ); + }, + InfoItem: ({ title }: MockListInfoItemProps) => { + 'use no memo'; + return ReactRuntime.createElement(Text, null, title); + }, + }; + + return { + Appbar: () => { + 'use no memo'; + return null; + }, + Button, + Dialog, + EmptyView: () => { + 'use no memo'; + return null; + }, + List, + SafeAreaView: ViewContent, + }; +}); +jest.mock('react-native-gesture-handler', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const { Pressable, View } = + jest.requireActual('react-native'); + const FlatList = ({ data, renderItem }: MockFlatListProps) => { + 'use no memo'; + return ReactRuntime.createElement( + View, + null, + data.map((item, index) => + ReactRuntime.createElement( + ReactRuntime.Fragment, + { key: String(index) }, + renderItem({ item, index }), + ), + ), + ); + }; + + return { + FlatList, + GestureHandlerRootView: View, + ScrollView: View, + TouchableOpacity: Pressable, + }; +}); + +jest.mock('react-native-safe-area-context', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const SafeAreaProvider = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(ReactRuntime.Fragment, null, children); + }; + return { SafeAreaProvider }; +}); +jest.mock('react-native-paper', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const { TextInput } = + jest.requireActual('react-native'); + const PassThrough = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(ReactRuntime.Fragment, null, children); + }; + return { Portal: PassThrough, Provider: PassThrough, TextInput }; +}); +jest.mock('../Components/BackupOptions', () => ({ + BackupOptionsDialog: () => { + 'use no memo'; + return null; + }, + BackupOptionsList: () => { + 'use no memo'; + return null; + }, +})); + +jest.mock('../Components/AutomaticBackupDialog', () => ({ + __esModule: true, + AUTOMATIC_BACKUP_LABELS: { 0: 'backupScreen.automaticBackupFrequency' }, + default: () => { + 'use no memo'; + return null; + }, +})); + +const driveBackup = { + id: 'drive-archive-id', + name: 'drive-archive.backup', + createdTime: '2026-09-24T00:00:00.000Z', +}; + +const renderScreen = () => + render( + , + ); + +const selectLocalBackup = async () => { + fireEvent.press(screen.getByText('backupScreen.restoreBackup')); + await screen.findByText(partialRestoreWarning); +}; +const selectDriveBackup = async () => { + fireEvent.press(screen.getByText('backupScreen.googeDrive')); + fireEvent.press(await screen.findByText('common.restore')); + fireEvent.press(await screen.findByText(/drive-archive/)); + await screen.findByText(partialRestoreWarning); +}; + +const selectSelfHostBackup = async () => { + fireEvent.press(screen.getByText('backupScreen.selfHost')); + fireEvent.press(await screen.findByText('common.ok')); + fireEvent.press(await screen.findByText('common.restore')); + fireEvent.press(await screen.findByText(/selfhost-archive/)); + await screen.findByText(partialRestoreWarning); +}; + +const restoreSelections = [ + { + source: 'local', + select: selectLocalBackup, + task: { + name: 'LOCAL_RESTORE', + data: { sourceUri: 'file://local-backup.zip' }, + }, + }, + { + source: 'Drive', + select: selectDriveBackup, + task: { name: 'DRIVE_RESTORE', data: driveBackup }, + }, + { + source: 'self-host', + select: selectSelfHostBackup, + task: { + name: 'SELF_HOST_RESTORE', + data: { + host: 'https://backup.example', + backupFolder: 'selfhost-archive.backup', + }, + }, + }, +]; + +describe('SettingsBackupScreen restore confirmation', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each(restoreSelections)( + 'waits for Restore before enqueueing the $source backup and shows the partial warning', + async ({ select, task }) => { + renderScreen(); + + await select(); + + expect(screen.getByText(partialRestoreWarning)).toBeTruthy(); + expect(backgroundTasks.enqueue).not.toHaveBeenCalled(); + + fireEvent.press(screen.getByText('common.restore')); + + expect(backgroundTasks.enqueue).toHaveBeenCalledTimes(1); + expect(backgroundTasks.enqueue).toHaveBeenCalledWith(task); + }, + ); + + it('discards a selected restore when the user cancels', async () => { + renderScreen(); + + await selectLocalBackup(); + fireEvent.press(screen.getByText('common.cancel')); + + expect(backgroundTasks.enqueue).not.toHaveBeenCalled(); + expect(screen.queryByText(partialRestoreWarning)).toBeNull(); + expect(screen.queryByText('common.restore')).toBeNull(); + }); +}); diff --git a/src/screens/settings/SettingsBackupScreen/index.tsx b/src/screens/settings/SettingsBackupScreen/index.tsx index 274b453d6b..1a650107ea 100644 --- a/src/screens/settings/SettingsBackupScreen/index.tsx +++ b/src/screens/settings/SettingsBackupScreen/index.tsx @@ -3,11 +3,13 @@ import { Appbar, List, SafeAreaView } from '@components'; import { useBoolean } from '@hooks'; import { BackupSettingsScreenProps } from '@navigators/types'; import GoogleDriveModal from './Components/GoogleDriveModal'; +import RestoreConfirmationDialog from './Components/RestoreConfirmationDialog'; import SelfHostModal from './Components/SelfHostModal'; import { backgroundTasks, configureAutomaticBackups, type AutomaticBackupInterval, + type BackgroundTask, } from '@services/backgroundTasks'; import { ScrollView } from 'react-native-gesture-handler'; import { getString } from '@i18n/translations'; @@ -26,6 +28,11 @@ import AutomaticBackupDialog, { } from './Components/AutomaticBackupDialog'; import { showToast } from '@utils/showToast'; +type RestoreTask = Extract< + BackgroundTask, + { name: 'LOCAL_RESTORE' | 'DRIVE_RESTORE' | 'SELF_HOST_RESTORE' } +>; + const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { const theme = useTheme(); const { @@ -38,6 +45,8 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { const [backupOptions, setBackupOptions] = useState({ ...DEFAULT_BACKUP_OPTIONS, }); + const [pendingRestoreTask, setPendingRestoreTask] = + useState(null); const { value: backupOptionsVisible, setFalse: closeBackupOptions, @@ -117,7 +126,7 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { const restoreLocalBackup = async () => { try { const sourceUri = await NativeFile.pickDocument('application/zip'); - backgroundTasks.enqueue({ + setPendingRestoreTask({ name: 'LOCAL_RESTORE', data: { sourceUri }, }); @@ -126,6 +135,13 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { } }; + const confirmRestore = () => { + if (pendingRestoreTask) { + backgroundTasks.enqueue(pendingRestoreTask); + setPendingRestoreTask(null); + } + }; + const { value: selfHostModalVisible, setFalse: closeSelfHostModal, @@ -203,11 +219,18 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { visible={googleDriveModalVisible} theme={theme} closeModal={closeGoogleDriveModal} + onRestoreSelected={setPendingRestoreTask} /> + setPendingRestoreTask(null)} + onRestore={confirmRestore} /> { showDownloadBadges = true, showNumberOfNovels = false, showUnreadBadges = true, + showContinueReadingButton = false, sortOrder = LibrarySortOrder.DateAdded_DESC, defaultCategoryId, promptForCategoryOnAdd = false, @@ -278,6 +279,16 @@ const SettingsLibraryScreen = ({ navigation }: LibrarySettingsScreenProps) => { onPress={novelBadgesModal.setTrue} theme={theme} /> + + setLibrarySettings({ + showContinueReadingButton: !showContinueReadingButton, + }) + } + theme={theme} + /> { /> setAppSettings({ useLibraryFAB: !useLibraryFAB })} theme={theme} diff --git a/src/services/backgroundTasks/BackgroundTaskQueue.ts b/src/services/backgroundTasks/BackgroundTaskQueue.ts index fd95fd27db..3b44293773 100644 --- a/src/services/backgroundTasks/BackgroundTaskQueue.ts +++ b/src/services/backgroundTasks/BackgroundTaskQueue.ts @@ -50,10 +50,20 @@ export class BackgroundTaskQueue { } async refresh() { - const records = await NativeBackgroundTasks.getTasks(); - const queue = records - .filter(record => ACTIVE_BACKGROUND_TASK_STATES.has(record.state)) - .map(fromNativeTaskRecord); + const summaries = await NativeBackgroundTasks.getTasks(); + const queue: QueuedBackgroundTask[] = []; + + for (const summary of summaries) { + const record = await NativeBackgroundTasks.getTask(summary.id); + if ( + record && + ACTIVE_BACKGROUND_TASK_STATES.has(record.state) && + record.payload + ) { + queue.push(fromNativeTaskRecord(record)); + } + } + this.store(queue); return queue; } @@ -156,11 +166,17 @@ export class BackgroundTaskQueue { getString('notifications.taskCompleted'), ); } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + const failureMessage = + task.name === 'LOCAL_RESTORE' || + task.name === 'DRIVE_RESTORE' || + task.name === 'SELF_HOST_RESTORE' + ? `${errorMessage}\n${getString('backupScreen.restoreMayBePartial')}` + : errorMessage; await NativeBackgroundTasks.fail( taskId, - getString('notifications.taskFailed', { - error: error instanceof Error ? error.message : String(error), - }), + getString('notifications.taskFailed', { error: failureMessage }), false, ); if (!this.interruptedTasks.has(taskId)) { diff --git a/src/services/backgroundTasks/__tests__/BackgroundTaskQueue.test.ts b/src/services/backgroundTasks/__tests__/BackgroundTaskQueue.test.ts index 7c3a703719..1f0272b553 100644 --- a/src/services/backgroundTasks/__tests__/BackgroundTaskQueue.test.ts +++ b/src/services/backgroundTasks/__tests__/BackgroundTaskQueue.test.ts @@ -12,6 +12,8 @@ jest.mock('@modules/native-background-tasks', () => ({ complete: jest.fn(), enqueue: jest.fn().mockResolvedValue('native-task-1'), fail: jest.fn(), + getTask: jest.fn(), + getTasks: jest.fn().mockResolvedValue([]), updateProgress: jest.fn().mockResolvedValue(undefined), }, })); @@ -43,6 +45,8 @@ jest.mock('@i18n/translations', () => ({ ? 'Download' : key === 'common.preparing' ? 'Preparing' + : key === 'backupScreen.restoreMayBePartial' + ? 'Changes already restored will remain if you pause, cancel, or the restore fails.' : 'Completed', })); @@ -79,7 +83,7 @@ describe('BackgroundTaskQueue completion notifications', () => { ); }); - it('localizes failure text before handing it to the native notification', async () => { + it('appends the partial-restore warning to restore failure notifications', async () => { jest .mocked(executeBackgroundTask) .mockRejectedValueOnce(new Error('Invalid backup')); @@ -89,7 +93,31 @@ describe('BackgroundTaskQueue completion notifications', () => { ).rejects.toThrow('Invalid backup'); expect(NativeBackgroundTasks.fail).toHaveBeenCalledWith( 'restore-2', - 'Failed: Invalid backup', + 'Failed: Invalid backup\nChanges already restored will remain if you pause, cancel, or the restore fails.', + false, + ); + }); + + it('leaves unrelated task failure notifications unchanged', async () => { + const downloadTask = { + name: 'DOWNLOAD_CHAPTER' as const, + data: { + novelName: 'Example Novel', + novelId: 42, + pluginId: 'source-a', + chapters: [{ chapterId: 1, chapterName: 'Chapter 1' }], + }, + }; + jest + .mocked(executeBackgroundTask) + .mockRejectedValueOnce(new Error('Network unavailable')); + + await expect( + new BackgroundTaskQueue().run('download-1', downloadTask), + ).rejects.toThrow('Network unavailable'); + expect(NativeBackgroundTasks.fail).toHaveBeenCalledWith( + 'download-1', + 'Failed: Network unavailable', false, ); }); @@ -229,4 +257,36 @@ describe('BackgroundTaskQueue completion notifications', () => { resolvers.forEach(resolve => resolve()); await Promise.all([firstRun, secondRun]); }); + it('hydrates active task payloads individually during refresh', async () => { + const summary = { + id: 'active-task', + type: 'LOCAL_RESTORE', + title: 'Restore', + state: 'queued', + attempt: 0, + createdAt: 1, + updatedAt: 1, + }; + const activeTask = { + ...summary, + payload: JSON.stringify(task), + checkpoint: undefined, + }; + jest.mocked(NativeBackgroundTasks.getTasks).mockResolvedValue([summary]); + jest + .mocked(NativeBackgroundTasks.getTask) + .mockResolvedValueOnce(activeTask); + + const queue = new BackgroundTaskQueue(); + await queue.refresh(); + + expect(NativeBackgroundTasks.getTask).toHaveBeenCalledWith('active-task'); + expect(mockStoredTasks).toEqual([ + expect.objectContaining({ + id: 'active-task', + task, + state: 'queued', + }), + ]); + }); }); diff --git a/src/services/backgroundTasks/__tests__/taskDefinitions.test.ts b/src/services/backgroundTasks/__tests__/taskDefinitions.test.ts index 43ba17f37c..17b77f8c4e 100644 --- a/src/services/backgroundTasks/__tests__/taskDefinitions.test.ts +++ b/src/services/backgroundTasks/__tests__/taskDefinitions.test.ts @@ -1,5 +1,5 @@ import type { NativeBackgroundTaskRecord } from '@modules/native-background-tasks'; -import type { BackgroundTask } from '../contracts'; +import type { BackgroundTask, QueuedBackgroundTask } from '../contracts'; import { allowsDuplicateTask, createBackgroundTaskMetadata, @@ -155,36 +155,49 @@ describe('background task definitions', () => { }); }); - it('derives progress keys only for the requested novel downloads', () => { + it('derives progress keys from task-level or chapter-level novel ids', () => { const createDownload = ( id: string, - novelId: number, + taskNovelId?: number, + chapterNovelId?: number, progress?: number, ) => ({ id, task: { name: 'DOWNLOAD_CHAPTER' as const, data: { - novelName: `Novel ${novelId}`, - novelId, - chapters: [{ chapterId: novelId, chapterName: 'Chapter 1' }], + novelName: `Novel ${taskNovelId ?? chapterNovelId}`, + ...(taskNovelId === undefined ? {} : { novelId: taskNovelId }), + chapters: [ + { + chapterId: taskNovelId ?? chapterNovelId ?? 0, + chapterName: 'Chapter 1', + ...(chapterNovelId === undefined + ? {} + : { novelId: chapterNovelId }), + }, + ], }, }, state: 'running' as const, meta: { - name: `Novel ${novelId}`, + name: `Novel ${taskNovelId ?? chapterNovelId}`, isRunning: true, progress, progressText: 'Chapter 1', }, }); - const tasks: (ReturnType | BackgroundTask)[] = [ + const tasks: (QueuedBackgroundTask | BackgroundTask)[] = [ { name: 'UPDATE_LIBRARY' }, - createDownload('first', 1, 0.5), - createDownload('second', 2), + createDownload('legacy', 1, undefined, 0.5), + createDownload('task-level', 2), + createDownload('chapter-level', undefined, 3), ]; - expect(getDownloadProgressKey(tasks, 1)).toBe('first:running:0.5'); - expect(getDownloadProgressKey(tasks, 2)).toBe('second:running:pending'); + expect(getDownloadProgressKey(tasks, 1)).toBe('legacy:running:0.5'); + expect(getDownloadProgressKey(tasks, 2)).toBe('task-level:running:pending'); + expect(getDownloadProgressKey(tasks, 3)).toBe( + 'chapter-level:running:pending', + ); }); }); diff --git a/src/services/backgroundTasks/contracts.ts b/src/services/backgroundTasks/contracts.ts index d7abc922e7..4098bf23fb 100644 --- a/src/services/backgroundTasks/contracts.ts +++ b/src/services/backgroundTasks/contracts.ts @@ -55,6 +55,7 @@ export type EpubExportData = { export type ChapterDownload = { chapterId: number; chapterName: string; + novelId?: number; }; export type BackgroundTask = diff --git a/src/services/backgroundTasks/executeTask.ts b/src/services/backgroundTasks/executeTask.ts index b2fd3041ad..be5e213467 100644 --- a/src/services/backgroundTasks/executeTask.ts +++ b/src/services/backgroundTasks/executeTask.ts @@ -1,6 +1,9 @@ -import { createDriveBackup, driveRestore } from '../backup/drive'; -import { createBackup, restoreBackup } from '../backup/local'; -import { createSelfHostBackup, selfHostRestore } from '../backup/selfhost'; +import { createDriveBackup } from '../backup/drive'; +import { driveRestore } from '../backup/restore/drive'; +import { createBackup } from '../backup/local'; +import { restoreBackup } from '../backup/restore/local'; +import { createSelfHostBackup } from '../backup/selfhost'; +import { selfHostRestore } from '../backup/restore/selfhost'; import { downloadChapters } from '../download/downloadChapter'; import { exportEpub } from '../epub/export'; import { importEpubBatch } from '../epub/import'; diff --git a/src/services/backgroundTasks/taskDefinitions.ts b/src/services/backgroundTasks/taskDefinitions.ts index ba86e45746..654285a6fe 100644 --- a/src/services/backgroundTasks/taskDefinitions.ts +++ b/src/services/backgroundTasks/taskDefinitions.ts @@ -34,7 +34,9 @@ export const getDownloadProgressKey = ( .filter( task => task.task.name === 'DOWNLOAD_CHAPTER' && - (novelId === undefined || task.task.data.novelId === novelId), + (novelId === undefined || + task.task.data.novelId === novelId || + task.task.data.chapters.some(chapter => chapter.novelId === novelId)), ) .map( task => diff --git a/src/services/backup/__tests__/create.test.ts b/src/services/backup/__tests__/create.test.ts new file mode 100644 index 0000000000..4a45774ce0 --- /dev/null +++ b/src/services/backup/__tests__/create.test.ts @@ -0,0 +1,529 @@ +import { getAllNovels } from '@database/queries/NovelQueries'; +import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; +import { + getAllNovelCategories, + getCategoriesFromDb, +} from '@database/queries/CategoryQueries'; +import NativeFile from '@modules/native-file'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { prepareBackupData } from '../create'; +import { decodeNovelBatch, encodeNovelBatch } from '../novelPayload'; +import type { BackupNovel, ChapterInfo } from '@database/types'; +import type { BackupOptions } from '../options'; + +jest.mock('@database/queries/NovelQueries', () => ({ + getAllNovels: jest.fn(), +})); + +jest.mock('@database/queries/ChapterQueries', () => ({ + getAllNovelChaptersForBackup: jest.fn(), +})); + +jest.mock('@database/queries/CategoryQueries', () => ({ + getAllNovelCategories: jest.fn(), + getCategoriesFromDb: jest.fn(), +})); + +jest.mock('@hooks/persisted/useSelfHost', () => ({ + SELF_HOST_BACKUP: 'SELF_HOST_BACKUP', +})); + +jest.mock('@hooks/persisted/migrations/trackerMigration', () => ({ + OLD_TRACKED_NOVEL_PREFIX: 'OLD_TRACKED_NOVEL_PREFIX', +})); + +jest.mock('@hooks/persisted/useUpdates', () => ({ + LAST_UPDATE_TIME: 'LAST_UPDATE_TIME', +})); + +jest.mock('@utils/mmkv/mmkv', () => ({ + MMKVStorage: { + getAllKeys: jest.fn(() => []), + getBoolean: jest.fn(), + getString: jest.fn(), + set: jest.fn(), + }, +})); + +jest.mock('@plugins/pluginManager', () => ({ + INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + ROOT_STORAGE: '/storage', +})); + +const pluginOnlyOptions: BackupOptions = { + library: false, + settings: false, + plugins: true, + downloadedFiles: false, +}; + +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: null, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); + +describe('selective backup creation', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(NativeFile.exists).mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(getAllNovels).mockResolvedValue([]); + jest.mocked(getAllNovelChaptersForBackup).mockResolvedValue([]); + jest.mocked(getCategoriesFromDb).mockResolvedValue([]); + jest.mocked(getAllNovelCategories).mockResolvedValue([]); + }); + + it('writes the selected sections to the v2 manifest', async () => { + await prepareBackupData('/cache', pluginOnlyOptions); + + expect(NativeFile.writeFile).toHaveBeenCalledTimes(2); + expect(NativeFile.writeFile).toHaveBeenCalledWith( + '/cache/Version.json', + expect.stringContaining( + '"sections":{"library":false,"settings":false,"plugins":true,"downloadedFiles":false}', + ), + ); + expect(getAllNovels).not.toHaveBeenCalled(); + expect(getCategoriesFromDb).not.toHaveBeenCalled(); + expect(NativeFile.writeFile).toHaveBeenCalledWith( + '/cache/Plugins.json', + '[]', + ); + }); + it('stores the total novel count in the v3 manifest', async () => { + const novels = [makeTestNovel(1), makeTestNovel(2), makeTestNovel(3)]; + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + + await prepareBackupData( + '/cache', + { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + 3, + ); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1] ?? '{}'), + ).toMatchObject({ + formatVersion: 3, + novelCount: 3, + }); + }); + + it('counts only validated novels in the v3 manifest', async () => { + const novels = [ + makeTestNovel(1), + { ...makeTestNovel(2), name: 2 as unknown as string }, + ]; + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + + const result = await prepareBackupData( + '/cache', + { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + 3, + ); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1]).novelCount, + ).toBe(1); + expect(result.failedNovelCount).toBe(1); + }); + + it('does not count novels from a failed batch write', async () => { + jest + .mocked(getAllNovels) + .mockResolvedValueOnce([makeTestNovel(1), makeTestNovel(2)]); + jest.mocked(NativeFile.writeFile).mockImplementation(async path => { + if (path.includes('/NovelAndChapters/batch-')) { + throw new Error('Novel batch write failed'); + } + }); + + const result = await prepareBackupData( + '/cache', + { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + 3, + ); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1] ?? '{}') + .novelCount, + ).toBe(0); + expect(result.failedNovelCount).toBe(2); + }); + + it('includes stored covers with library data when downloads are omitted', async () => { + jest.mocked(getAllNovels).mockResolvedValueOnce([ + { + ...makeTestNovel(1), + cover: 'file:///storage/Novels/source/1/cover.png?123', + }, + ]); + jest.mocked(getAllNovelChaptersForBackup).mockResolvedValueOnce([ + { + ...makeTestChapter(1), + id: 10, + }, + ]); + + await prepareBackupData('/cache', { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }); + + const novelWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', + }); + expect(compactNovel.c).toHaveLength(1); + expect(compactNovel.c[0]).toHaveLength(15); + expect(compactNovel.c[0][0]).toBe(10); + expect(compactNovel.c[0][7]).toBe(false); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + 'file:///storage/Novels/source/1/cover.png', + '/cache/Covers/1', + ); + }); + + it('does not duplicate covers when downloaded files are included', async () => { + jest.mocked(getAllNovels).mockResolvedValueOnce([ + { + ...makeTestNovel(1), + cover: 'file:///storage/Novels/source/1/cover.png?123', + }, + ]); + jest.mocked(NativeFile.copyFile).mockClear(); + jest.mocked(NativeFile.mkdir).mockClear(); + + await prepareBackupData('/cache', { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }); + + const novelWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', + }); + expect(NativeFile.copyFile).not.toHaveBeenCalledWith( + 'file:///storage/Novels/source/1/cover.png', + '/cache/Covers/1', + ); + expect(NativeFile.mkdir).not.toHaveBeenCalledWith('/cache/Covers'); + }); + + it('round-trips a fully populated novel through the compact codec', () => { + const novel: BackupNovel & { + chaptersDownloaded: number; + chaptersUnread: number; + totalChapters: number; + lastReadAt: string; + lastUpdatedAt: string; + } = { + id: 7, + name: 'The Compact Novel', + path: '/novels/compact', + pluginId: 'source', + cover: '/covers/compact.png?cache=1', + summary: null, + author: 'Author', + artist: null, + status: 'ongoing', + genres: null, + inLibrary: null, + isLocal: false, + totalPages: 2048, + chaptersDownloaded: 1, + chaptersUnread: 1, + totalChapters: 2, + lastReadAt: '2024-03-01T10:20:30.000Z', + lastUpdatedAt: '2024-03-02T10:20:30.000Z', + chapters: [ + { + id: 701, + novelId: 7, + path: '/novels/compact/1', + name: 'First chapter', + releaseTime: '2024-01-01T00:00:00.000Z', + readTime: '2024-03-01T10:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 1, + page: '4', + position: 2, + progress: 0.75, + scanlator: 'Team A', + timeSpent: 90, + }, + { + id: 702, + novelId: 7, + path: '/novels/compact/2', + name: 'Second chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: null, + }, + ], + }; + + const [compactNovel] = encodeNovelBatch([novel]); + + expect(Object.keys(compactNovel).sort()).toEqual( + [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', + ].sort(), + ); + expect(compactNovel).toMatchObject({ + id: 7, + p: '/novels/compact', + pi: 'source', + n: 'The Compact Novel', + co: '/covers/compact.png?cache=1', + d: 1, + u: 1, + tc: 2, + lr: '2024-03-01T10:20:30.000Z', + lu: '2024-03-02T10:20:30.000Z', + }); + expect(compactNovel.c).toEqual([ + [ + 701, + '/novels/compact/1', + 'First chapter', + '2024-01-01T00:00:00.000Z', + true, + false, + '2024-03-01T10:00:00.000Z', + true, + '2024-02-01T00:00:00.000Z', + 1, + '4', + 2, + 0.75, + 'Team A', + 90, + ], + [ + 702, + '/novels/compact/2', + 'Second chapter', + null, + null, + true, + null, + false, + null, + null, + null, + null, + null, + null, + null, + ], + ]); + expect(Object.keys(compactNovel.c[0])).not.toContain('novelId'); + expect(compactNovel).not.toHaveProperty('chapters'); + expect(decodeNovelBatch([compactNovel])).toEqual([novel]); + }); + + it('writes 101 novels as ordered compact batches with a format marker', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels = Array.from({ length: 101 }, (_, index) => { + const id = index + 1; + return makeTestNovel(id, 'source', [ + makeTestChapter(id, 1), + makeTestChapter(id, 2), + ]); + }); + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + jest.mocked(getAllNovelChaptersForBackup).mockImplementation(async ids => { + const requestedIds = Array.isArray(ids) ? ids : [ids]; + return novels + .flatMap(novel => novel.chapters) + .filter(chapter => requestedIds.includes(chapter.novelId)); + }); + + await prepareBackupData('/cache', options); + + const batchWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.includes('/NovelAndChapters/batch-')); + expect(batchWrites.map(([path]) => path)).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + ]); + expect(batchWrites.some(([path]) => /\/\d+\.json$/.test(path))).toBe(false); + + const batches = batchWrites.map(([, content]) => JSON.parse(content)); + expect(batches.map(batch => batch.length)).toEqual([100, 1]); + const records = batches.flat(); + expect(records.map(record => record.id)).toEqual( + novels.map(novel => novel.id), + ); + expect( + records.map(record => + record.c.map((chapter: [number, ...unknown[]]) => chapter[0]), + ), + ).toEqual(novels.map(novel => novel.chapters.map(chapter => chapter.id))); + expect( + records.every( + record => + Array.isArray(record.c) && + record.c.every((chapter: unknown[]) => chapter.length === 15), + ), + ).toBe(true); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1] ?? '{}'), + ).toMatchObject({ + formatVersion: 2, + novelDataFormat: 2, + novelCount: 101, + }); + }); + + it('omits the installed-plugin registry when plugin files are excluded', async () => { + jest + .mocked(MMKVStorage.getAllKeys) + .mockReturnValueOnce(['INSTALL_PLUGINS', 'OTHER_SETTING']); + jest + .mocked(MMKVStorage.getString) + .mockImplementation(key => + key === 'INSTALL_PLUGINS' + ? '[{"id":"source"}]' + : key === 'OTHER_SETTING' + ? 'kept' + : undefined, + ); + + await prepareBackupData('/cache', { + library: false, + settings: true, + plugins: false, + downloadedFiles: false, + }); + + const settingsWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => path.endsWith('/Setting.json')); + expect(JSON.parse(settingsWrite?.[1] ?? '{}')).toEqual({ + OTHER_SETTING: 'kept', + }); + }); +}); diff --git a/src/services/backup/__tests__/fileSections.test.ts b/src/services/backup/__tests__/fileSections.test.ts index f7d5bac972..58058dbc50 100644 --- a/src/services/backup/__tests__/fileSections.test.ts +++ b/src/services/backup/__tests__/fileSections.test.ts @@ -1,70 +1,34 @@ -import NativeFile from '@modules/native-file'; -import { restoreNovelFiles } from '../fileSections'; +import { getSelectedBackupFileSections } from '../fileSections'; +import { ZipBackupName } from '../types'; jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', })); +describe('selected backup file sections', () => { + const options = { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }; -describe('restoreNovelFiles', () => { - it('moves downloaded files to remapped novel and chapter IDs', async () => { - jest.mocked(NativeFile.exists).mockResolvedValue(true); - jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); - jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); - jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); - jest.mocked(NativeFile.readDir).mockImplementation(async path => { - if (path === '/staging/source/1') { - return [ - { - name: 'cover.png', - path: '/staging/source/1/cover.png', - isDirectory: false, - }, - { - name: '10', - path: '/staging/source/1/10', - isDirectory: true, - }, - { - name: '20', - path: '/staging/source/1/20', - isDirectory: true, - }, - ]; - } - if (path === '/staging/source/1/10') { - return [ - { - name: 'index.html', - path: '/staging/source/1/10/index.html', - isDirectory: false, - }, - ]; - } - return []; - }); - - await restoreNovelFiles('/staging', [ + it('keeps novel files in the legacy v2 section only', () => { + expect(getSelectedBackupFileSections(options)).toEqual([ { - pluginId: 'source', - backupNovelId: 1, - restoredNovelId: 7, - chapters: [{ backupChapterId: 10, restoredChapterId: 99 }], + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + ]); + expect(getSelectedBackupFileSections(options, 2)).toEqual([ + { + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + { + archiveName: ZipBackupName.NOVEL_FILES, + storagePath: '/storage/Novels', }, ]); - - expect(NativeFile.moveFile).toHaveBeenCalledWith( - '/staging/source/1/cover.png', - '/storage/Novels/source/7/cover.png', - ); - expect(NativeFile.moveFile).toHaveBeenCalledWith( - '/staging/source/1/10/index.html', - '/storage/Novels/source/7/99/index.html', - ); - expect(NativeFile.moveFile).not.toHaveBeenCalledWith( - expect.stringContaining('/20/'), - expect.any(String), - ); - expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); }); }); diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index 5601615128..6c85e67bbe 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -1,19 +1,15 @@ import NativeFile from '@modules/native-file'; import NativeZipArchive from '@modules/native-zip-archive'; -import { createBackup, restoreBackup } from '../local'; -import { finalizeRestoredPlugins } from '../restoreResult'; -import { prepareBackupData, restoreData } from '../utils'; +import { createBackup } from '../local'; +import { prepareBackupData } from '../create'; -jest.mock('../utils', () => ({ +jest.mock('../cache', () => ({ CACHE_DIR_PATH: '/cache/BackupData', clearBackupCache: jest.fn(), - prepareBackupData: jest.fn(), - restoreData: jest.fn(), })); -jest.mock('../restoreResult', () => ({ - finalizeRestoredPlugins: jest.fn(), - getRestoreCompletionText: jest.fn(), +jest.mock('../create', () => ({ + prepareBackupData: jest.fn(), })); jest.mock('../backupResult', () => ({ @@ -23,7 +19,6 @@ jest.mock('../backupResult', () => ({ jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', - ROOT_STORAGE: '/storage', })); jest.mock('@utils/sleep', () => ({ @@ -34,14 +29,23 @@ jest.mock('@i18n/translations', () => ({ getString: (key: string) => key, })); -describe('local selective backup', () => { +describe('local backup creation', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(prepareBackupData).mockReset(); + jest.mocked(NativeZipArchive.zip).mockReset().mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.zipDirectories) + .mockReset() + .mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockReset().mockResolvedValue(undefined); + }); + it('creates archives only for selected file sections', async () => { jest.mocked(prepareBackupData).mockResolvedValue({ failedNovelCount: 0, failedSectionCount: 0, }); - jest.mocked(NativeZipArchive.zip).mockResolvedValue(undefined); - jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); await createBackup({ destinationUri: 'content://backup.zip', @@ -53,12 +57,16 @@ describe('local selective backup', () => { }, }); - expect(prepareBackupData).toHaveBeenCalledWith('/cache/BackupData', { - library: true, - settings: true, - plugins: true, - downloadedFiles: false, - }); + expect(prepareBackupData).toHaveBeenCalledWith( + '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }, + 3, + ); expect(NativeZipArchive.zip).toHaveBeenCalledWith( '/storage/Plugins', '/cache/BackupData/plugins.zip', @@ -67,50 +75,52 @@ describe('local selective backup', () => { '/storage/Novels', expect.any(String), ); - expect(NativeZipArchive.zip).toHaveBeenCalledWith( - '/cache/BackupData', + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [{ path: '/cache/BackupData', prefix: '' }], '/cache/BackupData.zip', ); }); - it('loads restored plugins after their archive is extracted', async () => { - const restoreResult = { - novelCount: 1, + it('adds novel files to the v3 outer archive without a nested archive', async () => { + jest.mocked(prepareBackupData).mockResolvedValue({ failedNovelCount: 0, - categoryCount: 0, - failedCategoryCount: 0, - settingsRestored: true, failedSectionCount: 0, - pluginIds: ['restored'], - novelMappings: [], - manifest: { - appVersion: '2.1.0', - formatVersion: 2 as const, - sections: { - library: true, - settings: true, - plugins: true, - downloadedFiles: false, - }, - }, - }; - jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); - jest.mocked(NativeFile.exists).mockResolvedValue(true); - jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); - jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); - jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + }); - await restoreBackup({ sourceUri: 'content://backup.zip' }); + await createBackup({ + destinationUri: 'content://backup.zip', + options: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }); - expect(NativeZipArchive.unzip).toHaveBeenCalledWith( - '/cache/BackupData/plugins.zip', + expect(prepareBackupData).toHaveBeenCalledWith( + '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + 3, + ); + expect(NativeZipArchive.zip).toHaveBeenCalledWith( '/storage/Plugins', + '/cache/BackupData/plugins.zip', ); - expect(finalizeRestoredPlugins).toHaveBeenCalledWith(restoreResult); - expect( - jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], - ).toBeGreaterThan( - jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], + expect(NativeZipArchive.zip).not.toHaveBeenCalledWith( + '/storage/Novels', + expect.any(String), + ); + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [ + { path: '/cache/BackupData', prefix: '' }, + { path: '/storage/Novels', prefix: 'NovelFiles' }, + ], + '/cache/BackupData.zip', ); }); }); diff --git a/src/services/backup/__tests__/options.test.ts b/src/services/backup/__tests__/options.test.ts index 33fff6b8bf..edf8fe6cb5 100644 --- a/src/services/backup/__tests__/options.test.ts +++ b/src/services/backup/__tests__/options.test.ts @@ -7,6 +7,10 @@ import { import { getSelectedBackupFileSections } from '../fileSections'; import { ZipBackupName } from '../types'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts deleted file mode 100644 index 7c018e283a..0000000000 --- a/src/services/backup/__tests__/utils.test.ts +++ /dev/null @@ -1,388 +0,0 @@ -import { - _restoreNovelAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; -import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; -import { - _restoreCategory, - getAllNovelCategories, - getCategoriesFromDb, -} from '@database/queries/CategoryQueries'; -import NativeFile from '@modules/native-file'; -import { MMKVStorage } from '@utils/mmkv/mmkv'; -import { prepareBackupData, restoreData } from '../utils'; -import type { BackupOptions } from '../options'; - -jest.mock('@database/queries/NovelQueries', () => ({ - _restoreNovelAndChapters: jest.fn(), - getAllNovels: jest.fn(), -})); - -jest.mock('@database/queries/ChapterQueries', () => ({ - getAllNovelChaptersForBackup: jest.fn(), -})); - -jest.mock('@database/queries/CategoryQueries', () => ({ - _restoreCategory: jest.fn(), - getAllNovelCategories: jest.fn(), - getCategoriesFromDb: jest.fn(), -})); - -jest.mock('@hooks/persisted/useSelfHost', () => ({ - SELF_HOST_BACKUP: 'SELF_HOST_BACKUP', -})); - -jest.mock('@hooks/persisted/migrations/trackerMigration', () => ({ - OLD_TRACKED_NOVEL_PREFIX: 'OLD_TRACKED_NOVEL_PREFIX', -})); - -jest.mock('@hooks/persisted/useUpdates', () => ({ - LAST_UPDATE_TIME: 'LAST_UPDATE_TIME', -})); - -jest.mock('@utils/mmkv/mmkv', () => ({ - MMKVStorage: { - getAllKeys: jest.fn(() => []), - getBoolean: jest.fn(), - getString: jest.fn(), - set: jest.fn(), - }, -})); - -jest.mock('@i18n/translations', () => ({ - getString: (key: string) => key, -})); - -jest.mock('@plugins/pluginManager', () => ({ - INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', -})); - -jest.mock('@utils/Storages', () => ({ - NOVEL_STORAGE: '/storage/Novels', - ROOT_STORAGE: '/storage', -})); - -const pluginOnlyOptions: BackupOptions = { - library: false, - settings: false, - plugins: true, - downloadedFiles: false, -}; - -describe('selective backup data', () => { - beforeEach(() => { - jest.mocked(NativeFile.exists).mockResolvedValue(false); - jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); - jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); - jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); - jest.mocked(getAllNovels).mockResolvedValue([]); - jest.mocked(getAllNovelChaptersForBackup).mockResolvedValue([]); - jest.mocked(getCategoriesFromDb).mockResolvedValue([]); - jest.mocked(getAllNovelCategories).mockResolvedValue([]); - jest.mocked(_restoreNovelAndChapters).mockImplementation(async novel => ({ - pluginId: novel.pluginId, - backupNovelId: novel.id, - restoredNovelId: novel.id, - chapters: novel.chapters.map(chapter => ({ - backupChapterId: chapter.id, - restoredChapterId: chapter.id, - })), - })); - }); - - it('writes the selected sections to the v2 manifest', async () => { - await prepareBackupData('/cache', pluginOnlyOptions); - - expect(NativeFile.writeFile).toHaveBeenCalledTimes(2); - expect(NativeFile.writeFile).toHaveBeenCalledWith( - '/cache/Version.json', - expect.stringContaining( - '"sections":{"library":false,"settings":false,"plugins":true,"downloadedFiles":false}', - ), - ); - expect(getAllNovels).not.toHaveBeenCalled(); - expect(getCategoriesFromDb).not.toHaveBeenCalled(); - expect(NativeFile.writeFile).toHaveBeenCalledWith( - '/cache/Plugins.json', - '[]', - ); - }); - - it('does not warn about sections intentionally omitted by the manifest', async () => { - jest - .mocked(NativeFile.readFile) - .mockResolvedValueOnce( - JSON.stringify({ - appVersion: '2.1.0', - formatVersion: 2, - sections: pluginOnlyOptions, - }), - ) - .mockResolvedValueOnce('[]'); - jest - .mocked(NativeFile.exists) - .mockImplementation(async path => path.endsWith('/Plugins.json')); - - const result = await restoreData('/cache'); - - expect(result).toMatchObject({ - failedNovelCount: 0, - failedCategoryCount: 0, - failedSectionCount: 0, - settingsRestored: true, - manifest: { - formatVersion: 2, - sections: pluginOnlyOptions, - }, - }); - expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); - expect(_restoreCategory).not.toHaveBeenCalled(); - expect(MMKVStorage.set).toHaveBeenCalledWith('INSTALL_PLUGINS', '[]'); - }); - - it('merges restored plugins with the existing registry', async () => { - const options: BackupOptions = { - library: false, - settings: true, - plugins: true, - downloadedFiles: false, - }; - jest - .mocked(NativeFile.exists) - .mockImplementation( - async path => - path.endsWith('/Version.json') || - path.endsWith('/Setting.json') || - path.endsWith('/Plugins.json'), - ); - jest.mocked(NativeFile.readFile).mockImplementation(async path => { - if (path.endsWith('/Version.json')) { - return JSON.stringify({ - appVersion: '2.1.0', - formatVersion: 2, - sections: options, - }); - } - if (path.endsWith('/Setting.json')) { - return JSON.stringify({ - INSTALL_PLUGINS: JSON.stringify([ - { id: 'restored', name: 'Restored' }, - ]), - THEME: 'dark', - }); - } - return JSON.stringify([{ id: 'restored', name: 'Restored' }]); - }); - jest - .mocked(MMKVStorage.getString) - .mockReturnValueOnce( - JSON.stringify([{ id: 'existing', name: 'Existing' }]), - ); - - await restoreData('/cache'); - - expect(MMKVStorage.set).toHaveBeenCalledWith('THEME', 'dark'); - expect(MMKVStorage.set).toHaveBeenCalledWith( - 'INSTALL_PLUGINS', - JSON.stringify([ - { id: 'existing', name: 'Existing' }, - { id: 'restored', name: 'Restored' }, - ]), - ); - }); - - it('merges the plugin registry from legacy settings', async () => { - jest - .mocked(NativeFile.exists) - .mockImplementation(async path => path.endsWith('/Setting.json')); - jest.mocked(NativeFile.readFile).mockImplementation(async path => { - if (path.endsWith('/Version.json')) { - return JSON.stringify({ version: '2.0.0' }); - } - return JSON.stringify({ - INSTALL_PLUGINS: JSON.stringify([{ id: 'legacy', name: 'Legacy' }]), - }); - }); - jest - .mocked(MMKVStorage.getString) - .mockReturnValueOnce( - JSON.stringify([{ id: 'existing', name: 'Existing' }]), - ); - - await restoreData('/cache'); - - expect(MMKVStorage.set).toHaveBeenCalledWith( - 'INSTALL_PLUGINS', - JSON.stringify([ - { id: 'existing', name: 'Existing' }, - { id: 'legacy', name: 'Legacy' }, - ]), - ); - }); - - it('includes stored covers with library data when downloads are omitted', async () => { - jest.mocked(getAllNovels).mockResolvedValueOnce([ - { - id: 1, - name: 'Example', - path: '/example', - pluginId: 'source', - cover: 'file:///storage/Novels/source/1/cover.png?123', - }, - ]); - jest.mocked(getAllNovelChaptersForBackup).mockResolvedValueOnce([ - { - id: 10, - novelId: 1, - path: '/chapter-1', - name: 'Chapter 1', - isDownloaded: true, - }, - ] as Awaited>); - - await prepareBackupData('/cache', { - library: true, - settings: false, - plugins: false, - downloadedFiles: false, - }); - - const novelWrite = jest - .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/1.json')); - expect(JSON.parse(novelWrite?.[1] ?? '{}')).toMatchObject({ - cover: '/Novels/source/1/cover.png?123', - chapters: [{ id: 10, isDownloaded: false }], - }); - expect(NativeFile.copyFile).toHaveBeenCalledWith( - 'file:///storage/Novels/source/1/cover.png', - '/cache/Covers/1', - ); - }); - - it('restores stored covers from library data and preserves missing covers', async () => { - const options: BackupOptions = { - library: true, - settings: false, - plugins: false, - downloadedFiles: false, - }; - jest.mocked(NativeFile.readFile).mockImplementation(async path => { - if (path.endsWith('/Version.json')) { - return JSON.stringify({ - appVersion: '2.1.2', - formatVersion: 2, - sections: options, - }); - } - if (path.endsWith('/1.json')) { - return JSON.stringify({ - id: 1, - name: 'Stored cover', - path: '/stored-cover', - pluginId: 'source', - cover: '/Novels/source/1/cover.png?123', - chapters: [], - }); - } - if (path.endsWith('/2.json')) { - return JSON.stringify({ - id: 2, - name: 'Missing cover', - path: '/missing-cover', - pluginId: 'source', - cover: null, - chapters: [], - }); - } - return '[]'; - }); - jest - .mocked(NativeFile.exists) - .mockImplementation(async path => - [ - '/cache/NovelAndChapters', - '/cache/Covers/1', - '/cache/Category.json', - ].includes(path), - ); - jest.mocked(NativeFile.readDir).mockResolvedValue([ - { - name: '1.json', - path: '/cache/NovelAndChapters/1.json', - isDirectory: false, - }, - { - name: '2.json', - path: '/cache/NovelAndChapters/2.json', - isDirectory: false, - }, - ]); - - await restoreData('/cache'); - - expect(NativeFile.copyFile).toHaveBeenCalledWith( - '/cache/Covers/1', - '/storage/Novels/source/1/cover.png', - ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - id: 1, - cover: 'file:///storage/Novels/source/1/cover.png?123', - }), - ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ id: 2, cover: null }), - ); - }); - - it('omits the installed-plugin registry when plugin files are excluded', async () => { - jest - .mocked(MMKVStorage.getAllKeys) - .mockReturnValueOnce(['INSTALL_PLUGINS', 'OTHER_SETTING']); - jest - .mocked(MMKVStorage.getString) - .mockImplementation(key => - key === 'INSTALL_PLUGINS' - ? '[{"id":"source"}]' - : key === 'OTHER_SETTING' - ? 'kept' - : undefined, - ); - - await prepareBackupData('/cache', { - library: false, - settings: true, - plugins: false, - downloadedFiles: false, - }); - - const settingsWrite = jest - .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/Setting.json')); - expect(JSON.parse(settingsWrite?.[1] ?? '{}')).toEqual({ - OTHER_SETTING: 'kept', - }); - }); - - it('treats backups without a section manifest as legacy full backups', async () => { - jest - .mocked(NativeFile.readFile) - .mockResolvedValueOnce(JSON.stringify({ version: '2.0.0' })); - - const result = await restoreData('/cache'); - - expect(result.manifest).toMatchObject({ - appVersion: '2.0.0', - formatVersion: 1, - sections: { - library: true, - settings: true, - plugins: true, - downloadedFiles: true, - }, - }); - }); -}); diff --git a/src/services/backup/cache.ts b/src/services/backup/cache.ts new file mode 100644 index 0000000000..5f4a9a4543 --- /dev/null +++ b/src/services/backup/cache.ts @@ -0,0 +1,10 @@ +import NativeFile from '@modules/native-file'; + +export const CACHE_DIR_PATH = + NativeFile.ExternalCachesDirectoryPath + '/BackupData'; + +export const clearBackupCache = async (cacheDirPath = CACHE_DIR_PATH) => { + if (await NativeFile.exists(cacheDirPath)) { + await NativeFile.unlink(cacheDirPath); + } +}; diff --git a/src/services/backup/create.ts b/src/services/backup/create.ts new file mode 100644 index 0000000000..d5e1397460 --- /dev/null +++ b/src/services/backup/create.ts @@ -0,0 +1,250 @@ +import { SELF_HOST_BACKUP } from '@hooks/persisted/useSelfHost'; +import { OLD_TRACKED_NOVEL_PREFIX } from '@hooks/persisted/migrations/trackerMigration'; +import { LAST_UPDATE_TIME } from '@hooks/persisted/useUpdates'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { version } from '../../../package.json'; +import { getAllNovels } from '@database/queries/NovelQueries'; +import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; +import { + getAllNovelCategories, + getCategoriesFromDb, +} from '@database/queries/CategoryQueries'; +import type { BackupNovel } from '@database/types'; +import { encodeNovelBatch, validateBackupNovel } from './novelPayload'; +import { BackupEntryName, type BackupManifest } from './types'; +import { BACKGROUND_TASKS_STORE_KEY } from '@services/backgroundTasks/constants'; +import NativeFile from '@modules/native-file'; +import type { BackupResult } from './backupResult'; +import { resolveBackupOptions, type BackupOptions } from './options'; +import { INSTALLED_PLUGINS_KEY } from '@plugins/pluginManager'; +import { ROOT_STORAGE } from '@utils/Storages'; +import { clearBackupCache } from './cache'; + +const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; + +const BACKUP_NOVEL_BATCH_SIZE = 100; + +const BACKUP_FILE_CONCURRENCY = 8; + +const backupMMKVData = () => { + const excludeKeys = [ + BACKGROUND_TASKS_STORE_KEY, + OLD_TRACKED_NOVEL_PREFIX, + SELF_HOST_BACKUP, + LAST_UPDATE_TIME, + INSTALLED_PLUGINS_KEY, + ]; + const keys = MMKVStorage.getAllKeys().filter( + key => !excludeKeys.includes(key), + ); + const data: Record = {}; + for (const key of keys) { + let value: number | string | boolean | undefined = + MMKVStorage.getString(key); + if (!value) { + value = MMKVStorage.getBoolean(key); + } + if (key && value) { + data[key] = value; + } + } + return data; +}; + +export const prepareBackupData = async ( + cacheDirPath: string, + requestedOptions?: BackupOptions, + formatVersion: BackupManifest['formatVersion'] = 2, +): Promise => { + const options = resolveBackupOptions(requestedOptions); + const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; + const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; + let failedNovelCount = 0; + let failedSectionCount = 0; + + await clearBackupCache(cacheDirPath); + await NativeFile.mkdir(cacheDirPath); + + const manifest: BackupManifest = { + appVersion: version, + formatVersion, + novelDataFormat: 2, + sections: options, + }; + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.VERSION, + JSON.stringify(manifest), + ); + + // novels + if (options.library) { + await NativeFile.mkdir(novelDirPath); + if (!options.downloadedFiles) { + await NativeFile.mkdir(coversDirPath); + } + let backedUpNovelCount = 0; + const novels = await getAllNovels(); + for ( + let start = 0; + start < novels.length; + start += BACKUP_NOVEL_BATCH_SIZE + ) { + const novelBatch = novels.slice(start, start + BACKUP_NOVEL_BATCH_SIZE); + let chapters; + try { + chapters = await getAllNovelChaptersForBackup( + novelBatch.map(novel => novel.id), + ); + } catch { + failedNovelCount += novelBatch.length; + continue; + } + + const chaptersByNovel = new Map(); + for (const chapter of chapters) { + const novelChapters = chaptersByNovel.get(chapter.novelId); + if (novelChapters) { + novelChapters.push(chapter); + } else { + chaptersByNovel.set(chapter.novelId, [chapter]); + } + } + + const preparedNovels: BackupNovel[] = []; + for ( + let fileStart = 0; + fileStart < novelBatch.length; + fileStart += BACKUP_FILE_CONCURRENCY + ) { + const fileBatch = novelBatch.slice( + fileStart, + fileStart + BACKUP_FILE_CONCURRENCY, + ); + const prepared = await Promise.all( + fileBatch.map(async (novel): Promise => { + try { + const novelChapters = chaptersByNovel.get(novel.id) ?? []; + const backedUpChapters = options.downloadedFiles + ? novelChapters + : novelChapters.map(chapter => ({ + ...chapter, + isDownloaded: false, + })); + let cover = novel.cover; + if (cover?.startsWith(APP_STORAGE_URI)) { + if (options.downloadedFiles) { + cover = cover.replace(APP_STORAGE_URI, ''); + } else { + try { + await NativeFile.copyFile( + cover.split(/[?#]/, 1)[0], + coversDirPath + '/' + novel.id, + ); + cover = cover.replace(APP_STORAGE_URI, ''); + } catch { + cover = null; + } + } + } + const preparedNovel = { + ...novel, + chapters: backedUpChapters, + cover, + }; + return validateBackupNovel(preparedNovel); + } catch { + failedNovelCount++; + return null; + } + }), + ); + preparedNovels.push( + ...prepared.filter((novel): novel is BackupNovel => novel !== null), + ); + } + + if (preparedNovels.length > 0) { + const batchName = `batch-${String( + Math.floor(start / BACKUP_NOVEL_BATCH_SIZE) + 1, + ).padStart(6, '0')}.json`; + const batchPath = novelDirPath + '/' + batchName; + try { + await NativeFile.writeFile( + batchPath, + JSON.stringify(encodeNovelBatch(preparedNovels)), + ); + backedUpNovelCount += preparedNovels.length; + } catch { + failedNovelCount += preparedNovels.length; + try { + await NativeFile.unlink(batchPath); + } catch { + // Best effort cleanup prevents a failed write from restoring partial data. + } + } + } + } + manifest.novelCount = backedUpNovelCount; + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.VERSION, + JSON.stringify(manifest), + ); + + // categories + try { + const categories = await getCategoriesFromDb(); + const novelCategories = await getAllNovelCategories(); + const novelIdsByCategory = new Map(); + for (const novelCategory of novelCategories) { + const novelIds = novelIdsByCategory.get(novelCategory.categoryId); + if (novelIds) { + novelIds.push(novelCategory.novelId); + } else { + novelIdsByCategory.set(novelCategory.categoryId, [ + novelCategory.novelId, + ]); + } + } + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.CATEGORY, + JSON.stringify( + categories.map(category => ({ + ...category, + novelIds: novelIdsByCategory.get(category.id) ?? [], + })), + ), + ); + } catch { + failedSectionCount++; + } + } + + // settings + if (options.settings) { + try { + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.SETTING, + JSON.stringify(backupMMKVData()), + ); + } catch { + failedSectionCount++; + } + } + + // installed plugin registry + if (options.plugins) { + try { + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA, + MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', + ); + } catch { + failedSectionCount++; + } + } + + return { + failedNovelCount, + failedSectionCount, + }; +}; diff --git a/src/services/backup/drive/index.ts b/src/services/backup/drive/index.ts index d00e13041c..ae3c475d54 100644 --- a/src/services/backup/drive/index.ts +++ b/src/services/backup/drive/index.ts @@ -1,31 +1,16 @@ import { DriveFile } from '@api/drive/types'; import { sleep } from '@utils/sleep'; -import { exists } from '@api/drive'; import { getString } from '@i18n/translations'; -import { - CACHE_DIR_PATH, - clearBackupCache, - prepareBackupData, - restoreData, -} from '../utils'; -import { - finalizeRestoredPlugins, - getRestoreCompletionText, -} from '../restoreResult'; +import { CACHE_DIR_PATH } from '../cache'; +import { prepareBackupData } from '../create'; import { getBackupCompletionText } from '../backupResult'; -import { download, updateMetadata, uploadMedia } from '@api/drive/request'; +import { updateMetadata, uploadMedia } from '@api/drive/request'; import { ZipBackupName } from '../types'; import type { DriveBackupData, TaskProgressUpdater, } from '@services/backgroundTasks/contracts'; -import { - getLegacyFilesRestorePath, - getNovelFilesRestorePath, - getSelectedBackupFileSections, - restoreLegacyFiles, - restoreNovelFiles, -} from '../fileSections'; +import { getSelectedBackupFileSections } from '../fileSections'; import { resolveBackupOptions } from '../options'; const uploadBackupSection = async ( @@ -77,7 +62,7 @@ export const createDriveBackup = async ( progressText: getString('backupScreen.uploadingSelectedFiles'), })); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await uploadBackupSection( section.storagePath, section.archiveName, @@ -94,91 +79,3 @@ export const createDriveBackup = async ( completionText, })); }; - -export const driveRestore = async ( - backupFolder: DriveFile, - setMeta: TaskProgressUpdater, -) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - const zipDataFile = await exists(ZipBackupName.DATA, false, backupFolder.id); - if (!zipDataFile) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - - await clearBackupCache(); - await download(zipDataFile, CACHE_DIR_PATH); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFile = await exists( - ZipBackupName.DOWNLOAD, - false, - backupFolder.id, - ); - if (!legacyFile) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download(legacyFile, legacyFilesRestorePath); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - const file = await exists(section.archiveName, false, backupFolder.id); - if (!file) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - await download( - file, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, - restoreResult.novelMappings, - ); - } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); - - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); -}; diff --git a/src/services/backup/fileSections.ts b/src/services/backup/fileSections.ts index ece55687ad..dad9a6e2df 100644 --- a/src/services/backup/fileSections.ts +++ b/src/services/backup/fileSections.ts @@ -1,9 +1,6 @@ import { NOVEL_STORAGE, PLUGIN_STORAGE } from '@utils/Storages'; import type { BackupOptions } from './options'; import { ZipBackupName } from './types'; -import NativeFile from '@modules/native-file'; -import type { RestoredNovelMapping } from '@database/types'; - export type BackupFileSection = { archiveName: ZipBackupName; storagePath: string; @@ -11,6 +8,7 @@ export type BackupFileSection = { export const getSelectedBackupFileSections = ( options: BackupOptions, + formatVersion: 2 | 3 = 3, ): BackupFileSection[] => { const sections: BackupFileSection[] = []; @@ -20,7 +18,7 @@ export const getSelectedBackupFileSections = ( storagePath: PLUGIN_STORAGE, }); } - if (options.downloadedFiles) { + if (formatVersion === 2 && options.downloadedFiles) { sections.push({ archiveName: ZipBackupName.NOVEL_FILES, storagePath: NOVEL_STORAGE, @@ -29,78 +27,3 @@ export const getSelectedBackupFileSections = ( return sections; }; - -export const getNovelFilesRestorePath = (cacheDirPath: string) => - `${cacheDirPath}/RestoredNovelFiles`; - -export const getLegacyFilesRestorePath = (cacheDirPath: string) => - `${cacheDirPath}/RestoredLegacyFiles`; - -const moveDirectoryContents = async (source: string, destination: string) => { - if (!(await NativeFile.exists(source))) { - return; - } - await NativeFile.mkdir(destination); - for (const item of await NativeFile.readDir(source)) { - const destinationPath = `${destination}/${item.name}`; - if (item.isDirectory) { - await moveDirectoryContents(item.path, destinationPath); - } else { - await NativeFile.moveFile(item.path, destinationPath); - } - } -}; - -export const restoreNovelFiles = async ( - stagingPath: string, - novelMappings: RestoredNovelMapping[], -) => { - for (const mapping of novelMappings) { - const sourceNovelPath = `${stagingPath}/${mapping.pluginId}/${mapping.backupNovelId}`; - if (!(await NativeFile.exists(sourceNovelPath))) { - continue; - } - - const destinationNovelPath = `${NOVEL_STORAGE}/${mapping.pluginId}/${mapping.restoredNovelId}`; - await NativeFile.mkdir(destinationNovelPath); - const chapterIds = new Map( - mapping.chapters.map(chapter => [ - String(chapter.backupChapterId), - chapter.restoredChapterId, - ]), - ); - - for (const item of await NativeFile.readDir(sourceNovelPath)) { - if (!item.isDirectory) { - await NativeFile.moveFile( - item.path, - `${destinationNovelPath}/${item.name}`, - ); - continue; - } - - const restoredChapterId = chapterIds.get(item.name); - if (restoredChapterId !== undefined) { - await moveDirectoryContents( - item.path, - `${destinationNovelPath}/${restoredChapterId}`, - ); - } - } - } - - if (await NativeFile.exists(stagingPath)) { - await NativeFile.unlink(stagingPath); - } -}; - -export const restoreLegacyFiles = async ( - stagingPath: string, - novelMappings: RestoredNovelMapping[], -) => { - await moveDirectoryContents(`${stagingPath}/Plugins`, PLUGIN_STORAGE); - await restoreNovelFiles(`${stagingPath}/Novels`, novelMappings); - if (await NativeFile.exists(stagingPath)) { - await NativeFile.unlink(stagingPath); - } -}; diff --git a/src/services/backup/local/index.ts b/src/services/backup/local/index.ts index 85e7822c32..2ede9d2776 100644 --- a/src/services/backup/local/index.ts +++ b/src/services/backup/local/index.ts @@ -1,27 +1,14 @@ -import { - CACHE_DIR_PATH, - clearBackupCache, - prepareBackupData, - restoreData, -} from '../utils'; -import { - finalizeRestoredPlugins, - getRestoreCompletionText, -} from '../restoreResult'; +import { CACHE_DIR_PATH } from '../cache'; +import { prepareBackupData } from '../create'; import { getBackupCompletionText } from '../backupResult'; import NativeZipArchive from '@modules/native-zip-archive'; -import { ZipBackupName } from '../types'; +import { BackupEntryName } from '../types'; import NativeFile from '@modules/native-file'; import { getString } from '@i18n/translations'; import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; import { sleep } from '@utils/sleep'; -import { - getLegacyFilesRestorePath, - getNovelFilesRestorePath, - getSelectedBackupFileSections, - restoreLegacyFiles, - restoreNovelFiles, -} from '../fileSections'; +import { NOVEL_STORAGE } from '@utils/Storages'; +import { getSelectedBackupFileSections } from '../fileSections'; import { resolveBackupOptions, type BackupOptions } from '../options'; export const createBackup = async ( @@ -40,7 +27,7 @@ export const createBackup = async ( progressText: getString('backupScreen.preparingData'), })); - const backupResult = await prepareBackupData(CACHE_DIR_PATH, options); + const backupResult = await prepareBackupData(CACHE_DIR_PATH, options, 3); setMeta?.(meta => ({ ...meta, @@ -65,7 +52,15 @@ export const createBackup = async ( await sleep(200); - await NativeZipArchive.zip(CACHE_DIR_PATH, CACHE_DIR_PATH + '.zip'); + await NativeZipArchive.zipDirectories( + [ + { path: CACHE_DIR_PATH, prefix: '' }, + ...(options.downloadedFiles + ? [{ path: NOVEL_STORAGE, prefix: BackupEntryName.NOVEL_FILES }] + : []), + ], + CACHE_DIR_PATH + '.zip', + ); setMeta?.(meta => ({ ...meta, @@ -83,107 +78,7 @@ export const createBackup = async ( progressText: completionText, completionText, })); - } catch (error: any) { - setMeta?.(meta => ({ - ...meta, - isRunning: false, - })); - throw error; - } -}; - -export const restoreBackup = async ( - { sourceUri }: { sourceUri: string }, - setMeta?: TaskProgressUpdater, -) => { - try { - setMeta?.(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 4, - progressText: getString('backupScreen.downloadingData'), - })); - - await clearBackupCache(); - const localPath = CACHE_DIR_PATH + '-source.zip'; - await NativeFile.copyFile(sourceUri, localPath); - - setMeta?.(meta => ({ - ...meta, - progress: 1 / 4, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - await NativeZipArchive.unzip(localPath, CACHE_DIR_PATH); - - setMeta?.(meta => ({ - ...meta, - progress: 2 / 4, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta?.(meta => ({ - ...meta, - progress: 3 / 4, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyArchive = CACHE_DIR_PATH + '/' + ZipBackupName.DOWNLOAD; - if (!(await NativeFile.exists(legacyArchive))) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await NativeZipArchive.unzip(legacyArchive, legacyFilesRestorePath); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - const archivePath = `${CACHE_DIR_PATH}/${section.archiveName}`; - if (!(await NativeFile.exists(archivePath))) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - await NativeZipArchive.unzip( - archivePath, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, - restoreResult.novelMappings, - ); - } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); - - setMeta?.(meta => ({ - ...meta, - progress: 4 / 4, - isRunning: false, - progressText: completionText, - completionText, - })); - } catch (error: any) { + } catch (error) { setMeta?.(meta => ({ ...meta, isRunning: false, diff --git a/src/services/backup/novelPayload.ts b/src/services/backup/novelPayload.ts new file mode 100644 index 0000000000..b8c00c2102 --- /dev/null +++ b/src/services/backup/novelPayload.ts @@ -0,0 +1,444 @@ +import type { BackupNovel, ChapterInfo } from '@database/types'; + +type NullableString = string | null; +type NullableNumber = number | null; +type NullableBoolean = boolean | null; + +type CompactChapter = [ + number, + string, + string, + NullableString, + NullableBoolean, + NullableBoolean, + NullableString, + NullableBoolean, + NullableString, + NullableNumber, + NullableString, + NullableNumber, + NullableNumber, + NullableString, + NullableNumber, +]; + +type CompactNovel = { + c: CompactChapter[]; + id: number; + p: string; + pi: string; + n: string; + co: NullableString; + s: NullableString; + a: NullableString; + ar: NullableString; + st: NullableString; + g: NullableString; + l: NullableBoolean; + lo: NullableBoolean; + t: NullableNumber; + d: NullableNumber; + u: NullableNumber; + tc: NullableNumber; + lr: NullableString; + lu: NullableString; +}; + +type BackupNovelWithAggregates = BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; +}; + +const NOVEL_KEYS = [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', +] as const; +const EXPECTED_NOVEL_KEYS = [...NOVEL_KEYS].sort(); + +const isNullable = ( + value: unknown, + predicate: (value: unknown) => value is T, +): value is T | null => value === null || predicate(value); + +const isString = (value: unknown): value is string => typeof value === 'string'; +const isNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); +const isId = (value: unknown): value is number => + isNumber(value) && Number.isInteger(value) && value > 0; +const isBoolean = (value: unknown): value is boolean => + typeof value === 'boolean'; + +const nullableString = (value: unknown): value is NullableString => + isNullable(value, isString); +const nullableNumber = (value: unknown): value is NullableNumber => + isNullable(value, isNumber); +const nullableBoolean = (value: unknown): value is NullableBoolean => + isNullable(value, isBoolean); +const nonEmptyString = (value: unknown): value is string => + isString(value) && value.trim().length > 0; + +const nullableLegacyString = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isString(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyNumber = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isNumber(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyBoolean = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isBoolean(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const positiveId = (value: unknown): value is number => + isId(value) && value > 0; + +const normalizeLegacyChapter = ( + value: unknown, + containingNovelId: number, +): ChapterInfo => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy chapter'); + } + const chapter = value as Record; + if ( + !positiveId(chapter.id) || + !nonEmptyString(chapter.path) || + !isString(chapter.name) || + (chapter.novelId !== undefined && + chapter.novelId !== null && + !positiveId(chapter.novelId)) + ) { + throw new Error('Invalid legacy chapter fields'); + } + return { + id: chapter.id, + novelId: containingNovelId, + path: chapter.path, + name: chapter.name, + releaseTime: nullableLegacyString(chapter, 'releaseTime'), + readTime: nullableLegacyString(chapter, 'readTime'), + bookmark: nullableLegacyBoolean(chapter, 'bookmark'), + unread: nullableLegacyBoolean(chapter, 'unread'), + isDownloaded: nullableLegacyBoolean(chapter, 'isDownloaded'), + updatedTime: nullableLegacyString(chapter, 'updatedTime'), + chapterNumber: nullableLegacyNumber(chapter, 'chapterNumber'), + page: nullableLegacyString(chapter, 'page'), + progress: nullableLegacyNumber(chapter, 'progress'), + position: nullableLegacyNumber(chapter, 'position'), + scanlator: nullableLegacyString(chapter, 'scanlator'), + timeSpent: nullableLegacyNumber(chapter, 'timeSpent'), + }; +}; + +export const validateBackupNovel = (novel: BackupNovel): BackupNovel => { + if ( + !positiveId(novel.id) || + !nonEmptyString(novel.path) || + !nonEmptyString(novel.pluginId) || + !isString(novel.name) || + !Array.isArray(novel.chapters) + ) { + throw new Error('Invalid novel fields'); + } + const stringFields = [ + novel.cover, + novel.summary, + novel.author, + novel.artist, + novel.status, + novel.genres, + ]; + if (stringFields.some(value => value !== null && !isString(value))) { + throw new Error('Invalid novel string field'); + } + const booleanFields = [novel.inLibrary, novel.isLocal]; + if (booleanFields.some(value => value !== null && !isBoolean(value))) { + throw new Error('Invalid novel boolean field'); + } + if (novel.totalPages !== null && !isNumber(novel.totalPages)) { + throw new Error('Invalid novel number field'); + } + for (const chapter of novel.chapters) { + if ( + !positiveId(chapter.id) || + !positiveId(chapter.novelId) || + !nonEmptyString(chapter.path) || + !isStringOrNull(chapter.releaseTime) || + !isStringOrNull(chapter.readTime) || + !isBooleanOrNull(chapter.bookmark) || + !isBooleanOrNull(chapter.unread) || + !isBooleanOrNull(chapter.isDownloaded) || + !isStringOrNull(chapter.updatedTime) || + !isNumberOrNull(chapter.chapterNumber) || + !isStringOrNull(chapter.page) || + !isNumberOrNull(chapter.progress) || + !isNumberOrNull(chapter.position) || + !isStringOrNull(chapter.scanlator) || + !isNumberOrNull(chapter.timeSpent) + ) { + throw new Error('Invalid chapter fields'); + } + } + return novel; +}; + +const isStringOrNull = (value: unknown): value is string | null => + value === null || isString(value); +const isNumberOrNull = (value: unknown): value is number | null => + value === null || isNumber(value); +const isBooleanOrNull = (value: unknown): value is boolean | null => + value === null || isBoolean(value); + +export const normalizeLegacyNovel = (value: unknown): BackupNovel => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy novel'); + } + const source = value as Record; + if ( + !positiveId(source.id) || + !nonEmptyString(source.path) || + !nonEmptyString(source.pluginId) || + !isString(source.name) || + !Array.isArray(source.chapters) + ) { + throw new Error('Invalid legacy novel fields'); + } + const novelId = source.id; + const novel: BackupNovel = { + id: novelId, + path: source.path, + pluginId: source.pluginId, + name: source.name, + cover: nullableLegacyString(source, 'cover'), + summary: nullableLegacyString(source, 'summary'), + author: nullableLegacyString(source, 'author'), + artist: nullableLegacyString(source, 'artist'), + status: nullableLegacyString(source, 'status'), + genres: nullableLegacyString(source, 'genres'), + inLibrary: nullableLegacyBoolean(source, 'inLibrary'), + isLocal: nullableLegacyBoolean(source, 'isLocal'), + totalPages: nullableLegacyNumber(source, 'totalPages'), + chapters: source.chapters.map(chapter => + normalizeLegacyChapter(chapter, novelId), + ), + }; + return novel; +}; + +const assertNovelKeys = (novel: Record) => { + const keys = Object.keys(novel).sort(); + const expected = EXPECTED_NOVEL_KEYS; + if ( + keys.length !== expected.length || + keys.some((key, index) => key !== expected[index]) + ) { + throw new Error('Invalid compact novel keys'); + } +}; + +const isCompactChapter = (value: unknown): value is CompactChapter => { + if (!Array.isArray(value) || value.length !== 15) { + return false; + } + + return ( + isId(value[0]) && + nonEmptyString(value[1]) && + isString(value[2]) && + nullableString(value[3]) && + nullableBoolean(value[4]) && + nullableBoolean(value[5]) && + nullableString(value[6]) && + nullableBoolean(value[7]) && + nullableString(value[8]) && + nullableNumber(value[9]) && + nullableString(value[10]) && + nullableNumber(value[11]) && + nullableNumber(value[12]) && + nullableString(value[13]) && + nullableNumber(value[14]) + ); +}; + +type CompactNovelHeader = Omit & { c: unknown[] }; + +const isCompactNovelHeader = (value: unknown): value is CompactNovelHeader => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const novel = value as Record; + assertNovelKeys(novel); + return ( + Array.isArray(novel.c) && + isId(novel.id) && + nonEmptyString(novel.p) && + nonEmptyString(novel.pi) && + isString(novel.n) && + nullableString(novel.co) && + nullableString(novel.s) && + nullableString(novel.a) && + nullableString(novel.ar) && + nullableString(novel.st) && + nullableString(novel.g) && + nullableBoolean(novel.l) && + nullableBoolean(novel.lo) && + nullableNumber(novel.t) && + nullableNumber(novel.d) && + nullableNumber(novel.u) && + nullableNumber(novel.tc) && + nullableString(novel.lr) && + nullableString(novel.lu) + ); +}; + +const nullable = (value: T | undefined | null): T | null => value ?? null; + +const encodeChapter = (chapter: ChapterInfo): CompactChapter => [ + chapter.id, + chapter.path, + chapter.name, + nullable(chapter.releaseTime), + nullable(chapter.bookmark), + nullable(chapter.unread), + nullable(chapter.readTime), + nullable(chapter.isDownloaded), + nullable(chapter.updatedTime), + nullable(chapter.chapterNumber), + nullable(chapter.page), + nullable(chapter.position), + nullable(chapter.progress), + nullable(chapter.scanlator), + nullable(chapter.timeSpent), +]; + +const encodeNovel = (novel: BackupNovel): CompactNovel => { + const novelWithAggregates = novel as BackupNovelWithAggregates; + return { + c: novel.chapters.map(encodeChapter), + id: novel.id, + p: novel.path, + pi: novel.pluginId, + n: novel.name, + co: nullable(novel.cover), + s: nullable(novel.summary), + a: nullable(novel.author), + ar: nullable(novel.artist), + st: nullable(novel.status), + g: nullable(novel.genres), + l: nullable(novel.inLibrary), + lo: nullable(novel.isLocal), + t: nullable(novel.totalPages), + d: nullable(novelWithAggregates.chaptersDownloaded), + u: nullable(novelWithAggregates.chaptersUnread), + tc: nullable(novelWithAggregates.totalChapters), + lr: nullable(novelWithAggregates.lastReadAt), + lu: nullable(novelWithAggregates.lastUpdatedAt), + }; +}; + +const decodeNovel = (novel: CompactNovelHeader): BackupNovel => { + const decoded: BackupNovelWithAggregates = { + id: novel.id, + path: novel.p, + pluginId: novel.pi, + name: novel.n, + cover: novel.co, + summary: novel.s, + author: novel.a, + artist: novel.ar, + status: novel.st, + genres: novel.g, + inLibrary: novel.l, + isLocal: novel.lo, + totalPages: novel.t, + chaptersDownloaded: novel.d, + chaptersUnread: novel.u, + totalChapters: novel.tc, + lastReadAt: novel.lr, + lastUpdatedAt: novel.lu, + chapters: novel.c.map(chapter => { + if (!isCompactChapter(chapter)) { + throw new Error('Invalid compact novel record'); + } + return { + id: chapter[0], + novelId: novel.id, + path: chapter[1], + name: chapter[2], + releaseTime: chapter[3], + bookmark: chapter[4], + unread: chapter[5], + isDownloaded: chapter[7], + updatedTime: chapter[8], + chapterNumber: chapter[9], + page: chapter[10], + position: chapter[11], + progress: chapter[12], + scanlator: chapter[13], + timeSpent: chapter[14], + readTime: chapter[6], + }; + }), + }; + return decoded; +}; + +export const encodeNovelBatch = (novels: BackupNovel[]): CompactNovel[] => { + if (novels.length > 100) { + throw new Error('Compact novel batch exceeds 100 novels'); + } + return novels.map(encodeNovel); +}; + +export const decodeNovelBatch = (payload: unknown): BackupNovel[] => { + if (!Array.isArray(payload) || payload.length > 100) { + throw new Error('Invalid compact novel batch'); + } + return payload.map(value => { + if (!isCompactNovelHeader(value)) { + throw new Error('Invalid compact novel record'); + } + return decodeNovel(value); + }); +}; diff --git a/src/services/backup/restore/__tests__/drive.test.ts b/src/services/backup/restore/__tests__/drive.test.ts new file mode 100644 index 0000000000..fb115c0222 --- /dev/null +++ b/src/services/backup/restore/__tests__/drive.test.ts @@ -0,0 +1,240 @@ +import { exists } from '@api/drive'; +import type { DriveFile } from '@api/drive/types'; +import { download } from '@api/drive/request'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { CACHE_DIR_PATH, clearBackupCache } from '../../cache'; +import { ZipBackupName } from '../../types'; +import { restoreNovelFiles } from '../files'; + +import { driveRestore } from '../drive'; +import { restoreData } from '../index'; +import { + finalizeRestoredPlugins, + getRestoreCompletionText, + type RestoreResult, +} from '../result'; +import type { RestoreProgressReporter } from '../progress'; + +jest.mock('@api/drive', () => ({ exists: jest.fn() })); +jest.mock('@api/drive/request', () => ({ download: jest.fn() })); +jest.mock('../../cache', () => ({ + CACHE_DIR_PATH: '/cache/BackupData', + clearBackupCache: jest.fn(), +})); +jest.mock('../files', () => ({ + getLegacyFilesRestorePath: jest.fn(() => '/cache/legacy-files'), + getNovelFilesRestorePath: jest.fn(() => '/cache/novel-files'), + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), +})); +jest.mock('../index', () => ({ + clearRestoreChapterMappingsSafely: jest.fn(), + restoreData: jest.fn(), +})); +jest.mock('../result', () => ({ + finalizeRestoredPlugins: jest.fn(), + getRestoreCompletionText: jest.fn(() => 'completed'), +})); +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', +})); + +const dataFile = { id: 'data-file', name: 'data.zip' } as DriveFile; +const backupFolder = { id: 'backup-folder', name: 'backup' } as DriveFile; + +const createRestoreResult = ( + plugins: boolean, + downloadedFiles = false, + formatVersion: 1 | 2 | 3 = 3, + novelMappings: RestoreResult['novelMappings'] = [], +): RestoreResult => ({ + novelCount: 0, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: plugins ? ['restored-plugin'] : [], + novelMappings, + restoreRunId: 'drive-restore-run', + manifest: { + appVersion: '2.1.3', + formatVersion, + novelDataFormat: 2, + sections: { + library: downloadedFiles, + settings: false, + plugins, + downloadedFiles, + }, + }, +}); + +const reportRestoreStages = (reporter: RestoreProgressReporter | undefined) => { + reporter?.('manifest', 1, 'manifest', true); + reporter?.('novels', 1, 'novels', true); + reporter?.('categories', 1, 'categories', true); + reporter?.('settings', 1, 'settings', true); + reporter?.('plugins', 1, 'plugins', true); +}; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'DRIVE_RESTORE', + isRunning: false, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + }; + return { progressValues, setMeta, getMetadata: () => metadata }; +}; + +describe('Drive restore progress', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(exists).mockResolvedValue(dataFile); + jest.mocked(download).mockResolvedValue(''); + jest.mocked(clearBackupCache).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValue([]); + }); + + it('keeps remote retrieval and later phases monotonic through completion', async () => { + const result = createRestoreResult(false); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await driveRestore(backupFolder, capture.setMeta); + + expect(capture.progressValues[0]).toBe(0); + expect( + capture.progressValues.some(value => Math.abs(value - 0.3) < 0.000001), + ).toBe(true); + expect(capture.progressValues[capture.progressValues.length - 1]).toBe(1); + expect( + capture.progressValues.every( + (value, index) => + index === 0 || value >= capture.progressValues[index - 1], + ), + ).toBe(true); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + expect(download).toHaveBeenCalledWith(dataFile, CACHE_DIR_PATH); + }); + + it('does not report completion when a selected archive is missing', async () => { + const result = createRestoreResult(true); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + jest + .mocked(exists) + .mockImplementation(async name => + name === ZipBackupName.DATA ? dataFile : undefined, + ); + const capture = createProgressCapture(); + + await expect(driveRestore(backupFolder, capture.setMeta)).rejects.toThrow( + 'backupScreen.invalidBackupFolder', + ); + + expect(capture.progressValues.length).toBeGreaterThan(0); + expect(Math.max(...capture.progressValues)).toBeLessThan(1); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + }); + it('restores selected format-2 archives before finalizing plugins', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = createRestoreResult(true, true, 2, mappings); + const pluginFile = { + id: 'plugins-file', + name: ZipBackupName.PLUGINS, + } as DriveFile; + const novelFilesFile = { + id: 'novel-files-file', + name: ZipBackupName.NOVEL_FILES, + } as DriveFile; + const files: Record = { + [ZipBackupName.DATA]: dataFile, + [ZipBackupName.PLUGINS]: pluginFile, + [ZipBackupName.NOVEL_FILES]: novelFilesFile, + }; + jest.mocked(exists).mockImplementation(async name => files[name]); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await driveRestore(backupFolder, capture.setMeta); + + expect(download).toHaveBeenCalledWith(dataFile, CACHE_DIR_PATH); + expect(download).toHaveBeenCalledWith(pluginFile, '/storage/Plugins'); + expect(download).toHaveBeenCalledWith(novelFilesFile, '/cache/novel-files'); + expect(restoreNovelFiles).toHaveBeenCalledWith( + '/cache/novel-files', + mappings, + result.restoreRunId, + expect.any(Function), + ); + expect( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect(finalizeRestoredPlugins).toHaveBeenCalledWith(result); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ); + expect( + jest.mocked(getRestoreCompletionText).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + }); + + it('rejects a missing data archive before clearing the cache or downloading', async () => { + jest.mocked(exists).mockResolvedValue(undefined); + const capture = createProgressCapture(); + + await expect(driveRestore(backupFolder, capture.setMeta)).rejects.toThrow( + 'backupScreen.invalidBackupFolder', + ); + + expect(exists).toHaveBeenCalledWith( + ZipBackupName.DATA, + false, + backupFolder.id, + ); + expect(clearBackupCache).not.toHaveBeenCalled(); + expect(download).not.toHaveBeenCalled(); + expect(restoreData).not.toHaveBeenCalled(); + }); +}); diff --git a/src/services/backup/restore/__tests__/files.test.ts b/src/services/backup/restore/__tests__/files.test.ts new file mode 100644 index 0000000000..246602eef5 --- /dev/null +++ b/src/services/backup/restore/__tests__/files.test.ts @@ -0,0 +1,220 @@ +import NativeFile from '@modules/native-file'; +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { restoreLegacyFiles, restoreNovelFiles } from '../files'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', +})); + +describe('restore file sections', () => { + beforeEach(() => { + jest.mocked(getRestoreChapterMappings).mockReset(); + }); + + it('moves downloaded files with bounded staged mapping lookups', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + const chapterItems = Array.from({ length: 101 }, (_, index) => ({ + name: String(index + 1), + path: `/staging/source/1/${index + 1}`, + isDirectory: true, + })); + jest.mocked(NativeFile.readDir).mockImplementation(async path => { + if (path === '/staging/source/1') { + return [ + { + name: 'cover.png', + path: '/staging/source/1/cover.png', + isDirectory: false, + }, + ...chapterItems, + ]; + } + if (path === '/staging/source/1/10') { + return [ + { + name: 'index.html', + path: '/staging/source/1/10/index.html', + isDirectory: false, + }, + ]; + } + if (path === '/staging/source/1/101') { + return [ + { + name: 'index.html', + path: '/staging/source/1/101/index.html', + isDirectory: false, + }, + ]; + } + return []; + }); + jest + .mocked(getRestoreChapterMappings) + .mockImplementation( + async (_restoreRunId, _backupNovelId, backupChapterIds) => + backupChapterIds + .filter(id => id === 10 || id === 101) + .map(backupChapterId => ({ + backupChapterId, + restoredChapterId: backupChapterId === 10 ? 99 : 199, + })), + ); + + const progress: { completed: number; total: number }[] = []; + await restoreNovelFiles( + '/staging', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'restore-run-1', + (completed, total) => progress.push({ completed, total }), + ); + + expect(getRestoreChapterMappings).toHaveBeenCalledTimes(2); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 1, + 'restore-run-1', + 1, + Array.from({ length: 100 }, (_, index) => index + 1), + ); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 2, + 'restore-run-1', + 1, + [101], + ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/cover.png', + '/storage/Novels/source/7/cover.png', + ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/10/index.html', + '/storage/Novels/source/7/99/index.html', + ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/101/index.html', + '/storage/Novels/source/7/199/index.html', + ); + expect(NativeFile.moveFile).not.toHaveBeenCalledWith( + expect.stringContaining('/20/'), + expect.any(String), + ); + expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); + expect(progress).toEqual([ + { completed: 1, total: 2 }, + { completed: 2, total: 2 }, + ]); + }); + + it('counts missing novel directories and only completes after staging cleanup', async () => { + jest.mocked(NativeFile.exists).mockImplementation(async path => { + if (path === '/staging') { + return true; + } + return false; + }); + jest + .mocked(NativeFile.unlink) + .mockRejectedValue(new Error('staging cleanup failed')); + const progress: { completed: number; total: number }[] = []; + + await expect( + restoreNovelFiles( + '/staging', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'restore-run', + (completed, total) => progress.push({ completed, total }), + ), + ).rejects.toThrow('staging cleanup failed'); + + expect(progress).toEqual([{ completed: 1, total: 2 }]); + }); + + it('reports the staging cleanup for an empty mapping set', async () => { + const events: string[] = []; + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.unlink).mockImplementation(async () => { + events.push('cleanup'); + }); + const progress: { completed: number; total: number }[] = []; + + await restoreNovelFiles( + '/staging', + [], + 'restore-run', + (completed, total) => { + progress.push({ completed, total }); + events.push(`progress:${completed}/${total}`); + }, + ); + + expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); + expect(progress).toEqual([{ completed: 1, total: 1 }]); + expect(events).toEqual(['cleanup', 'progress:1/1']); + }); + + it('passes the restore run ID through legacy file restoration', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockImplementation(async path => { + if (path === '/legacy/Novels/source/1') { + return [ + { + name: '10', + path: '/legacy/Novels/source/1/10', + isDirectory: true, + }, + ]; + } + return []; + }); + jest + .mocked(getRestoreChapterMappings) + .mockResolvedValue([{ backupChapterId: 10, restoredChapterId: 99 }]); + + const progress: { completed: number; total: number }[] = []; + await restoreLegacyFiles( + '/legacy', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'legacy-run', + (completed, total) => progress.push({ completed, total }), + ); + + expect(getRestoreChapterMappings).toHaveBeenCalledWith('legacy-run', 1, [ + 10, + ]); + expect(NativeFile.unlink).toHaveBeenCalledWith('/legacy'); + expect(progress).toEqual([ + { completed: 1, total: 3 }, + { completed: 2, total: 3 }, + { completed: 3, total: 3 }, + ]); + }); +}); diff --git a/src/services/backup/restore/__tests__/index.test.ts b/src/services/backup/restore/__tests__/index.test.ts new file mode 100644 index 0000000000..e2bb4ea97d --- /dev/null +++ b/src/services/backup/restore/__tests__/index.test.ts @@ -0,0 +1,1282 @@ +import { + clearRestoreChapterMappings, + _restoreNovelAndChapters, + _restoreNovelsAndChapters, +} from '@database/queries/NovelRestoreQueries'; +import { _restoreCategory } from '@database/queries/CategoryQueries'; +import NativeFile from '@modules/native-file'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { clearRestoreChapterMappingsSafely, restoreData } from '../index'; +import { encodeNovelBatch } from '../../novelPayload'; +import type { + BackupNovel, + ChapterInfo, + RestoredNovelMapping, +} from '@database/types'; +import type { BackupOptions } from '../../options'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(async () => undefined), + _restoreNovelAndChapters: jest.fn(), + _restoreNovelsAndChapters: jest.fn(), +})); + +jest.mock('@database/queries/CategoryQueries', () => ({ + _restoreCategory: jest.fn(), +})); + +jest.mock('@utils/mmkv/mmkv', () => ({ + MMKVStorage: { + getAllKeys: jest.fn(() => []), + getBoolean: jest.fn(), + getString: jest.fn(), + set: jest.fn(), + }, +})); + +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); + +jest.mock('@plugins/pluginManager', () => ({ + INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + ROOT_STORAGE: '/storage', +})); + +const pluginOnlyOptions: BackupOptions = { + library: false, + settings: false, + plugins: true, + downloadedFiles: false, +}; +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: null, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); + +describe('selective backup data', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(NativeFile.exists).mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + const restoreNovel = async ( + novel: BackupNovel, + _options?: { includeChapterMappings?: boolean; restoreRunId?: string }, + ): Promise => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id, + }); + jest.mocked(_restoreNovelAndChapters).mockImplementation(restoreNovel); + jest.mocked(_restoreNovelsAndChapters).mockImplementation( + async ( + novels: BackupNovel[], + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => Promise.all(novels.map(novel => restoreNovel(novel, _options))), + ); + }); + + it('accepts v3 section manifests', async () => { + const options: BackupOptions = { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }; + jest.mocked(NativeFile.readFile).mockResolvedValueOnce( + JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }), + ); + + const result = await restoreData('/cache'); + + expect(result.manifest).toEqual({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }); + expect(result.restoreRunId).toEqual(expect.any(String)); + }); + + it('does not warn about sections intentionally omitted by the manifest', async () => { + jest + .mocked(NativeFile.readFile) + .mockResolvedValueOnce( + JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 2, + sections: pluginOnlyOptions, + }), + ) + .mockResolvedValueOnce('[]'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path.endsWith('/Plugins.json')); + + const result = await restoreData('/cache'); + + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + failedNovelCount: 0, + failedCategoryCount: 0, + failedSectionCount: 0, + settingsRestored: true, + manifest: { + formatVersion: 2, + sections: pluginOnlyOptions, + }, + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(_restoreCategory).not.toHaveBeenCalled(); + expect(MMKVStorage.set).toHaveBeenCalledWith('INSTALL_PLUGINS', '[]'); + }); + + it('merges restored plugins with the existing registry', async () => { + const options: BackupOptions = { + library: false, + settings: true, + plugins: true, + downloadedFiles: false, + }; + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => + path.endsWith('/Version.json') || + path.endsWith('/Setting.json') || + path.endsWith('/Plugins.json'), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/Setting.json')) { + return JSON.stringify({ + INSTALL_PLUGINS: JSON.stringify([ + { id: 'restored', name: 'Restored' }, + ]), + THEME: 'dark', + }); + } + return JSON.stringify([{ id: 'restored', name: 'Restored' }]); + }); + jest + .mocked(MMKVStorage.getString) + .mockReturnValueOnce( + JSON.stringify([{ id: 'existing', name: 'Existing' }]), + ); + + await restoreData('/cache'); + + expect(MMKVStorage.set).toHaveBeenCalledWith('THEME', 'dark'); + expect(MMKVStorage.set).toHaveBeenCalledWith( + 'INSTALL_PLUGINS', + JSON.stringify([ + { id: 'existing', name: 'Existing' }, + { id: 'restored', name: 'Restored' }, + ]), + ); + }); + it.each(['{}', 'null'])( + 'isolates malformed category payload %s from other restore sections', + async categoryPayload => { + const options: BackupOptions = { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/Version.json', + '/cache/NovelAndChapters', + '/cache/Category.json', + '/cache/Setting.json', + '/cache/Plugins.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue([]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 3, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return categoryPayload; + } + if (path.endsWith('/Setting.json')) { + return JSON.stringify({ reviewSetting: true }); + } + return JSON.stringify([{ id: 'review-plugin', name: 'Review plugin' }]); + }); + + const result = await restoreData('/cache'); + + expect(result).toMatchObject({ + failedSectionCount: 1, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + }); + expect(_restoreCategory).not.toHaveBeenCalled(); + expect(MMKVStorage.set).toHaveBeenCalledWith('reviewSetting', true); + expect(MMKVStorage.set).toHaveBeenCalledWith( + 'INSTALL_PLUGINS', + JSON.stringify([{ id: 'review-plugin', name: 'Review plugin' }]), + ); + }, + ); + + it('propagates category progress interruptions before restoring settings', async () => { + const options: BackupOptions = { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }; + const interruption = new Error('category progress interrupted'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/Version.json', + '/cache/NovelAndChapters', + '/cache/Category.json', + '/cache/Setting.json', + '/cache/Plugins.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue([]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 3, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return '[]'; + } + if (path.endsWith('/Setting.json')) { + return JSON.stringify({ reviewSetting: true }); + } + return '[]'; + }); + const progressReporter = jest.fn((phase: string) => { + if (phase === 'categories') { + throw interruption; + } + }); + + await expect(restoreData('/cache', progressReporter)).rejects.toBe( + interruption, + ); + expect(MMKVStorage.set).not.toHaveBeenCalledWith('reviewSetting', true); + }); + + it('merges the plugin registry from legacy settings', async () => { + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path.endsWith('/Setting.json')); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ version: '2.0.0' }); + } + return JSON.stringify({ + INSTALL_PLUGINS: JSON.stringify([{ id: 'legacy', name: 'Legacy' }]), + }); + }); + jest + .mocked(MMKVStorage.getString) + .mockReturnValueOnce( + JSON.stringify([{ id: 'existing', name: 'Existing' }]), + ); + + await restoreData('/cache'); + + expect(MMKVStorage.set).toHaveBeenCalledWith( + 'INSTALL_PLUGINS', + JSON.stringify([ + { id: 'existing', name: 'Existing' }, + { id: 'legacy', name: 'Legacy' }, + ]), + ); + }); + + it('restores stored covers from library data and preserves missing covers', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.2', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/1.json')) { + return JSON.stringify({ + id: 1, + name: 'Stored cover', + path: '/stored-cover', + pluginId: 'source', + cover: '/Novels/source/1/cover.png?123', + chapters: [], + }); + } + if (path.endsWith('/2.json')) { + return JSON.stringify({ + id: 2, + name: 'Missing cover', + path: '/missing-cover', + pluginId: 'source', + cover: null, + chapters: [], + }); + } + return '[]'; + }); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/NovelAndChapters', + '/cache/Covers/1', + '/cache/Category.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: '1.json', + path: '/cache/NovelAndChapters/1.json', + isDirectory: false, + }, + { + name: '2.json', + path: '/cache/NovelAndChapters/2.json', + isDirectory: false, + }, + ]); + + await restoreData('/cache'); + + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/1', + '/storage/Novels/source/1/cover.png', + ); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 1, + cover: 'file:///storage/Novels/source/1/cover.png?123', + }), + expect.objectContaining({ id: 2, cover: null }), + ], + { includeChapterMappings: false }, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + }); + + it('keeps restored mappings when a stored cover copy fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const novels = [1, 2, 3].map(id => ({ + ...makeTestNovel(id), + cover: `/Novels/source/${id}/cover.png`, + })); + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + const category = { id: 1, name: 'Category', novelIds: [1, 2, 3] }; + + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/NovelAndChapters', + '/cache/Covers/1', + '/cache/Covers/2', + '/cache/Covers/3', + '/cache/Category.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return JSON.stringify([category]); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + if (!novel) { + throw new Error(`Unexpected read: ${path}`); + } + return JSON.stringify(novel); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + jest.mocked(NativeFile.copyFile).mockImplementation(async source => { + if (source === '/cache/Covers/1') { + throw new Error('Cover copy failed'); + } + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + novels.map(novel => ({ + ...novel, + cover: `file:///storage${novel.cover}`, + })), + { includeChapterMappings: false }, + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/2', + '/storage/Novels/source/102/cover.png', + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/3', + '/storage/Novels/source/103/cover.png', + ); + expect(_restoreCategory).toHaveBeenCalledWith( + expect.objectContaining({ novelIds: [1, 2, 3] }), + new Map([ + [1, 101], + [2, 102], + [3, 103], + ]), + ); + expect(result).toMatchObject({ + novelCount: 3, + failedNovelCount: 1, + failedSectionCount: 0, + novelMappings: mappings, + }); + }); + + it('restores downloaded-file novels in one mapping-aware batch', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 11, + name: 'First', + path: '/first', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 101, + novelId: 11, + name: 'First chapter', + path: '/first/1', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 22, + name: 'Second', + path: '/second', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 202, + novelId: 22, + name: 'Second chapter', + path: '/second/1', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ]; + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'source', + backupNovelId: 11, + restoredNovelId: 111, + }, + { + pluginId: 'source', + backupNovelId: 22, + restoredNovelId: 222, + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(result.restoreRunId).toBe( + jest.mocked(_restoreNovelsAndChapters).mock.calls[0][1]?.restoreRunId, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + + it('restores scrambled compact batches in order with downloaded-file mappings', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const firstNovel = makeTestNovel(11, 'plugin-a', [makeTestChapter(11, 1)]); + const secondNovel = makeTestNovel(22, 'plugin-b', [makeTestChapter(22, 1)]); + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'plugin-a', + backupNovelId: 11, + restoredNovelId: 111, + }, + { + pluginId: 'plugin-b', + backupNovelId: 22, + restoredNovelId: 222, + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'batch-000002.json', + path: '/cache/NovelAndChapters/batch-000002.json', + isDirectory: false, + }, + { + name: 'ignored-directory', + path: '/cache/NovelAndChapters/ignored-directory', + isDirectory: true, + }, + { + name: 'batch-000001.json', + path: '/cache/NovelAndChapters/batch-000001.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + if (path.endsWith('/batch-000001.json')) { + return JSON.stringify(encodeNovelBatch([firstNovel])); + } + if (path.endsWith('/batch-000002.json')) { + return JSON.stringify(encodeNovelBatch([secondNovel])); + } + throw new Error(`Unexpected read: ${path}`); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + const restoredNovels = jest.mocked(_restoreNovelsAndChapters).mock + .calls[0][0] as BackupNovel[]; + expect(restoredNovels.map(novel => novel.id)).toEqual([11, 22]); + expect( + restoredNovels.map(novel => + novel.chapters.map(chapter => ({ + id: chapter.id, + novelId: chapter.novelId, + isDownloaded: chapter.isDownloaded, + })), + ), + ).toEqual([ + [{ id: 1101, novelId: 11, isDownloaded: true }], + [{ id: 2201, novelId: 22, isDownloaded: true }], + ]); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(expect.any(Array), { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + pluginIds: ['plugin-a', 'plugin-b'], + novelMappings: mappings, + }); + }); + + it('retries each downloaded-file novel when its batch restore fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 31, + name: 'First', + path: '/first', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [], + }, + { + id: 32, + name: 'Second', + path: '/second', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [], + }, + ]; + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest + .mocked(_restoreNovelsAndChapters) + .mockRejectedValueOnce(new Error('batch failed')); + jest.mocked(_restoreNovelAndChapters).mockImplementation( + async ( + novel, + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => mappings.find(mapping => mapping.backupNovelId === novel.id)!, + ); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(1, novels[0], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(2, novels[1], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + + it.each([ + { + label: 'v1', + manifest: { version: '1.0.0' }, + formatVersion: 1, + }, + { + label: 'v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 2, + }, + { + label: 'v3', + manifest: { + appVersion: '3.0.0', + formatVersion: 3, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 3, + }, + ])( + '$label object payloads remain restorable', + async ({ manifest, formatVersion }) => { + const novel = makeTestNovel(301, 'legacy-source'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify(manifest); + } + return JSON.stringify(novel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 301, + pluginId: 'legacy-source', + chapters: [ + expect.objectContaining({ + id: 30101, + novelId: 301, + }), + ], + }), + ], + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, + ); + expect(result.manifest).toMatchObject({ formatVersion }); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + novelMappings: [ + expect.objectContaining({ + backupNovelId: 301, + pluginId: 'legacy-source', + }), + ], + }); + }, + ); + + it('normalizes legacy object payloads before restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + chapters: [ + { + id: 50101, + novelId: 999, + path: '/legacy/1', + name: 'Legacy chapter', + }, + ], + ignoredLegacyField: 'ignored', + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 50101, + novelId: 501, + path: '/legacy/1', + name: 'Legacy chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], + }, + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + }); + }); + it('restores legacy novels whose stored name is empty', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + chapters: [], + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'empty-name.json', + path: '/cache/NovelAndChapters/empty-name.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + }), + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + novelCount: 1, + failedNovelCount: 0, + }); + }); + + it('rejects duplicate source IDs, novel identities, and chapter identities before restoring invalid files', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(601, 'source'); + const duplicateIdNovel = makeTestNovel(601, 'other-source'); + const duplicateIdentityNovel = { + ...makeTestNovel(602, 'source'), + path: validNovel.path, + }; + const duplicateChapterIdentityNovel = { + ...makeTestNovel(603, 'source'), + chapters: [ + makeTestChapter(603, 1), + { ...makeTestChapter(603, 1), id: 60302 }, + ], + }; + const payloads = { + 'batch-000001.json': JSON.stringify(encodeNovelBatch([validNovel])), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([duplicateIdNovel])), + 'batch-000003.json': JSON.stringify( + encodeNovelBatch([duplicateIdentityNovel]), + ), + 'batch-000004.json': JSON.stringify( + encodeNovelBatch([duplicateChapterIdentityNovel]), + ), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(payloads) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return payloads[path.split('/').pop() as keyof typeof payloads]; + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith([validNovel], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 3, + pluginIds: ['source'], + }); + }); + + it('skips malformed compact files without partially restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(901, 'valid-source'); + const [validRecord] = encodeNovelBatch([validNovel]); + const malformedFiles: Record = { + 'batch-000001.json': '{', + 'batch-000002.json': JSON.stringify({}), + 'batch-000003.json': JSON.stringify([{ ...validRecord, n: undefined }]), + 'batch-000004.json': JSON.stringify([ + validRecord, + { + ...validRecord, + c: [[...validRecord.c[0]].slice(0, 14)], + }, + ]), + 'batch-000005.json': JSON.stringify([{ ...validRecord, id: 'bad' }]), + 'batch-000006.json': JSON.stringify([{ ...validRecord, p: 123 }]), + 'batch-000007.json': JSON.stringify([validRecord]), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(malformedFiles) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return malformedFiles[path.split('/').pop() ?? '']; + }); + + const result = await restoreData('/cache'); + expect( + jest + .mocked(NativeFile.readFile) + .mock.calls.map(([path]) => path) + .filter(path => path.includes('/NovelAndChapters/')), + ).toEqual( + Object.keys(malformedFiles) + .sort() + .map(name => `/cache/NovelAndChapters/${name}`), + ); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 901, + pluginId: 'valid-source', + chapters: [ + expect.objectContaining({ + id: 90101, + novelId: 901, + }), + ], + }), + ], + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 7, + }); + }); + + it('treats backups without a section manifest as legacy full backups', async () => { + jest + .mocked(NativeFile.readFile) + .mockResolvedValueOnce(JSON.stringify({ version: '2.0.0' })); + + const result = await restoreData('/cache'); + + expect(result.manifest).toMatchObject({ + appVersion: '2.0.0', + formatVersion: 1, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }); + expect(result.restoreRunId).toEqual(expect.any(String)); + }); + it('does not reject when restore mapping cleanup fails', async () => { + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + clearRestoreChapterMappingsSafely('restore-run'), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith('restore-run'); + }); +}); diff --git a/src/services/backup/restore/__tests__/local.test.ts b/src/services/backup/restore/__tests__/local.test.ts new file mode 100644 index 0000000000..6f4eac785d --- /dev/null +++ b/src/services/backup/restore/__tests__/local.test.ts @@ -0,0 +1,536 @@ +import NativeFile from '@modules/native-file'; +import NativeZipArchive from '@modules/native-zip-archive'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { restoreBackup } from '../local'; +import * as fileSections from '../files'; +import { finalizeRestoredPlugins, getRestoreCompletionText } from '../result'; +import { restoreData } from '../index'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import type { UnzipStats } from '@modules/native-zip-archive/src/NativeZipArchiveModule'; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + }; + return { progressValues, setMeta }; +}; + +const unzipStats: UnzipStats = { + fileCount: 7, + compressedBytes: 420, + uncompressedBytes: 840, + largestUncompressedEntryBytes: 300, + elapsedMs: 1.5, + categories: { + NovelAndChapters: { + fileCount: 2, + compressedBytes: 120, + uncompressedBytes: 240, + }, + Covers: { fileCount: 1, compressedBytes: 30, uncompressedBytes: 60 }, + NovelFiles: { fileCount: 3, compressedBytes: 240, uncompressedBytes: 480 }, + other: { fileCount: 1, compressedBytes: 30, uncompressedBytes: 60 }, + }, +}; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(), + getRestoreChapterMappings: jest.fn(), +})); + +jest.mock('../files', () => { + const actual = jest.requireActual('../files'); + return { + ...actual, + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), + }; +}); + +jest.mock('../../cache', () => ({ + CACHE_DIR_PATH: '/cache/BackupData', + clearBackupCache: jest.fn(), +})); + +jest.mock('../index', () => ({ + clearRestoreChapterMappingsSafely: jest.fn(async (restoreRunId: string) => { + try { + await jest + .requireMock('@database/queries/NovelRestoreQueries') + .clearRestoreChapterMappings(restoreRunId); + } catch { + // Match the production helper's best-effort cleanup. + } + }), + restoreData: jest.fn(), +})); + +jest.mock('../result', () => ({ + finalizeRestoredPlugins: jest.fn(), + getRestoreCompletionText: jest.fn(), +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', + ROOT_STORAGE: '/storage', +})); + +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); + +describe('local backup restore', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + beforeEach(() => { + jest.mocked(restoreData).mockReset(); + jest.mocked(NativeZipArchive.zip).mockReset().mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.zipDirectories) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.unzip) + .mockReset() + .mockResolvedValue(unzipStats); + jest.mocked(NativeFile.copyFile).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.exists).mockReset().mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockReset().mockResolvedValue([]); + jest.mocked(finalizeRestoredPlugins).mockReset().mockResolvedValue([]); + jest + .mocked(clearRestoreChapterMappings) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreLegacyFiles) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreNovelFiles) + .mockReset() + .mockResolvedValue(undefined); + }); + + it('loads restored plugins after their archive is extracted', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: ['restored'], + novelMappings: [], + restoreRunId: 'restore-run-plugins', + manifest: { + appVersion: '2.1.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }, + }, + }; + jest + .mocked(restoreData) + .mockImplementationOnce(async (_cacheDirPath, reporter) => { + reporter?.('manifest', 1, 'manifest', true); + reporter?.('novels', 0.25, 'novels', true); + reporter?.('novels', 0.75, 'novels', true); + reporter?.('categories', 1, 'categories', true); + reporter?.('settings', 1, 'settings', true); + reporter?.('plugins', 1, 'plugins', true); + return restoreResult; + }); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + const capture = createProgressCapture(); + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + await restoreBackup({ sourceUri: 'content://backup.zip' }, capture.setMeta); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/plugins.zip', + '/storage/Plugins', + ); + expect(finalizeRestoredPlugins).toHaveBeenCalledWith(restoreResult); + expect(logSpy).toHaveBeenCalledWith( + expect.stringContaining( + `local:outer-unzip:done ${JSON.stringify(unzipStats)}`, + ), + ); + expect( + capture.progressValues.some(value => Math.abs(value - 0.09) < 0.000001), + ).toBe(true); + expect( + capture.progressValues.some(value => Math.abs(value - 0.3) < 0.000001), + ).toBe(true); + const novelProgress = capture.progressValues.filter( + value => value > 0.302 && value < 0.982, + ); + expect(new Set(novelProgress).size).toBeGreaterThan(1); + expect(capture.progressValues[capture.progressValues.length - 1]).toBe(1); + logSpy.mockRestore(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], + ); + }); + it('keeps a successful restore successful when mapping cleanup fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-cleanup-failure', + manifest: { + appVersion: '2.1.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('preserves the restore error when mapping cleanup also fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-restore-failure', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('extracts the v1 downloaded archive into the legacy staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v1', + manifest: { + appVersion: '1.0.0', + formatVersion: 1 as const, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/download.zip', + '/cache/BackupData/RestoredLegacyFiles', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + expect(fileSections.restoreLegacyFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredLegacyFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + expect.any(Function), + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('extracts the v2 novel-files archive into the novel staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + '/cache/BackupData/RestoredNovelFiles', + ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredNovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + expect.any(Function), + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('restores v3 novel files from the outer archive without nested extraction', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v3', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + novelDataFormat: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/plugins.zip', + '/storage/Plugins', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/NovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + expect.any(Function), + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + it('rejects a v3 downloaded-file restore without NovelFiles', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v3-missing', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + const capture = createProgressCapture(); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }, capture.setMeta), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(capture.progressValues.every(value => value < 1)).toBe(true); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + it('does not report completion when plugin finalization fails', async () => { + const restoreResult = { + novelCount: 0, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-finalization-failure', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }, + }; + const failure = new Error('plugin finalization failed'); + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(finalizeRestoredPlugins).mockRejectedValueOnce(failure); + const capture = createProgressCapture(); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }, capture.setMeta), + ).rejects.toBe(failure); + + expect(capture.progressValues.every(value => value < 1)).toBe(true); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + }); + + it('does not report success when the restore pipeline fails and resets benchmark timing before a retry', async () => { + const interruption = new Error('restore interrupted'); + let completionUpdateCount = 0; + const setMeta: TaskProgressUpdater = transform => { + const next = transform({ + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }); + if (next.completionText) { + completionUpdateCount++; + } + }; + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + for (let attempt = 0; attempt < 2; attempt++) { + jest.mocked(restoreData).mockRejectedValueOnce(interruption); + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }, setMeta), + ).rejects.toBe(interruption); + } + + const startEvents = logSpy.mock.calls + .map(([message]) => String(message)) + .filter(message => message.includes('local:start')); + expect(startEvents).toHaveLength(2); + expect(startEvents[1]).not.toContain('duration:'); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + expect(completionUpdateCount).toBe(0); + logSpy.mockRestore(); + }); +}); diff --git a/src/services/backup/restore/__tests__/pipeline.test.ts b/src/services/backup/restore/__tests__/pipeline.test.ts new file mode 100644 index 0000000000..e3c56f8b20 --- /dev/null +++ b/src/services/backup/restore/__tests__/pipeline.test.ts @@ -0,0 +1,749 @@ +import { + _restoreNovelAndChapters, + _restoreNovelsAndChapters, + clearRestoreChapterMappings, +} from '@database/queries/NovelRestoreQueries'; +import { _restoreCategory } from '@database/queries/CategoryQueries'; +import type { + BackupNovel, + ChapterInfo, + RestoredNovelMapping, +} from '@database/types'; +import NativeFile from '@modules/native-file'; +import { restoreData } from '../index'; +import { restoreNovels } from '../novels'; +import { encodeNovelBatch } from '../../novelPayload'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { createRestoreProgressReporter } from '../progress'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + _restoreNovelAndChapters: jest.fn(), + _restoreNovelsAndChapters: jest.fn(), + clearRestoreChapterMappings: jest.fn(async () => undefined), +})); +jest.mock('@database/queries/CategoryQueries', () => ({ + _restoreCategory: jest.fn(), +})); + +jest.mock('@i18n/translations', () => ({ + getString: (key: string, options?: Record) => { + if (options?.current === undefined) { + return key; + } + return options.total === undefined + ? `${key}:${options.current}` + : `${key}:${options.current}/${options.total}`; + }, +})); + +jest.mock('@utils/mmkv/mmkv', () => ({ + MMKVStorage: { + getAllKeys: jest.fn(() => []), + getBoolean: jest.fn(), + getString: jest.fn(), + set: jest.fn(), + }, +})); + +jest.mock('@plugins/pluginManager', () => ({ + INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + ROOT_STORAGE: '/storage', +})); + +const options = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, +}; + +const makeTestChapter = (novelId: number): ChapterInfo => ({ + id: novelId * 100 + 1, + novelId, + path: `/novel/${novelId}/chapter/1`, + name: `Chapter ${novelId}`, + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, +}); + +const makeTestNovel = (id: number, path = `/novel/${id}`): BackupNovel => ({ + id, + name: `Novel ${id}`, + path, + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: true, + isLocal: false, + totalPages: null, + chapters: [makeTestChapter(id)], +}); + +const makeMappings = (novels: BackupNovel[]): RestoredNovelMapping[] => + novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 10_000, + })); + +const createDeferred = () => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const manifestContent = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, +}); + +const configureFiles = ( + files: Record, + manifest = manifestContent, +) => { + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => + path === '/cache/NovelAndChapters' || path === '/cache/Category.json', + ); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(files) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path === '/cache/Version.json') { + return manifest; + } + if (path === '/cache/Category.json') { + return JSON.stringify([ + { id: 8, name: 'Reading list', novelIds: [1, 100, 200, 201] }, + ]); + } + return files[path.split('/').pop() ?? '']; + }); +}; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const progressTexts: string[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + if (metadata.progressText !== undefined) { + progressTexts.push(metadata.progressText); + } + }; + return { + progressValues, + progressTexts, + reporter: createRestoreProgressReporter(setMeta, 'local'), + }; +}; + +const getNovelReadPaths = () => + jest + .mocked(NativeFile.readFile) + .mock.calls.map(([path]) => path) + .filter(path => path.includes('/NovelAndChapters/')); + +describe('bounded novel restore pipeline', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + it('settles progress and logs zeroed metrics when the novel directory is absent', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(false); + const events: string[] = []; + const capture = createProgressCapture(); + const result = await restoreNovels( + '/cache', + { + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }, + 'restore-run', + capture.reporter, + event => { + events.push(event); + }, + ); + + expect(result.failedSectionCount).toBe(1); + expect(capture.progressTexts).toContain('backupScreen.restoringNovels'); + expect( + capture.progressValues[capture.progressValues.length - 1], + ).toBeCloseTo(0.982); + const summaryPrefix = 'restoreData:database:summary '; + const summaryEvent = events.find(event => event.startsWith(summaryPrefix)); + expect(summaryEvent).toBeDefined(); + expect(JSON.parse(summaryEvent!.slice(summaryPrefix.length))).toMatchObject( + { + uniqueInputChapterCount: 0, + novelUpsertCalls: 0, + novelIdentityLookupCalls: 0, + chapterWriteChunkCalls: 0, + chapterMappingRowsAttempted: 0, + statsRefreshCalls: 0, + novelBatchFallbacks: 0, + chapterBatchFallbacks: 0, + fallbackCauses: [], + }, + ); + }); + + it('completes the progress ranges for omitted library, settings, and plugins', async () => { + const manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + sections: { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }); + configureFiles({}, manifest); + const capture = createProgressCapture(); + + await restoreData('/cache', capture.reporter); + + expect(capture.progressTexts).toContain('backupScreen.restoringNovels'); + expect(capture.progressTexts).toContain('backupScreen.restoringCategories'); + expect(capture.progressTexts).toContain('backupScreen.restoringSettings'); + expect(capture.progressTexts).toContain('backupScreen.restoringPlugins'); + expect( + capture.progressValues[capture.progressValues.length - 1], + ).toBeLessThan(1); + }); + + it('keeps the 25-novel denominator until mappings persist', async () => { + const novels = Array.from({ length: 25 }, (_, index) => + makeTestNovel(index + 1), + ); + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 25, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(novels)), + }, + v3Manifest, + ); + const mappingsResult = createDeferred(); + const mappingsStarted = createDeferred(); + jest.mocked(_restoreNovelsAndChapters).mockImplementation(() => { + mappingsStarted.resolve(undefined); + return mappingsResult.promise; + }); + const progressTexts: string[] = []; + const restorePromise = restoreData('/cache', (phase, _fraction, text) => { + if (phase === 'novels') { + progressTexts.push(text); + } + }); + await mappingsStarted.promise; + expect(progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/25', + ); + expect(progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:25/25', + ); + mappingsResult.resolve(makeMappings(novels)); + await restorePromise; + + expect(progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:25/25', + ); + }); + + it('uses the v2 novel file count before any mappings persist', async () => { + const novels = [makeTestNovel(1), makeTestNovel(2), makeTestNovel(3)]; + const v2Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + configureFiles( + Object.fromEntries( + novels.map(novel => [`novel-${novel.id}.json`, JSON.stringify(novel)]), + ), + v2Manifest, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async batch => makeMappings(batch)); + const firstProgress = createDeferred(); + const progressTexts: string[] = []; + const restorePromise = restoreData('/cache', (phase, _fraction, text) => { + if (phase !== 'novels') { + return; + } + progressTexts.push(text); + if (text.startsWith('backupScreen.restoringNovelsProgress:')) { + firstProgress.resolve(text); + } + }); + + await expect(firstProgress.promise).resolves.toBe( + 'backupScreen.restoringNovelsProgress:0/3', + ); + const result = await restorePromise; + + expect(result.novelCount).toBe(3); + expect(progressTexts).toContain('backupScreen.restoringNovelsProgress:3/3'); + }); + + it('restores compact v2 batches using their total novel count', async () => { + const firstBatch = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const lastNovel = makeTestNovel(101); + const v2Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + novelCount: 101, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstBatch)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([lastNovel])), + }, + v2Manifest, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async batch => makeMappings(batch)); + const capture = createProgressCapture(); + + const result = await restoreData('/cache', capture.reporter); + + expect(result.novelCount).toBe(101); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/101', + ); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:101/101', + ); + }); + + it('restores successful novels after a failed three-novel batch', async () => { + const novels = Array.from({ length: 3 }, (_, index) => + makeTestNovel(index + 1), + ); + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 3, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(novels)), + }, + v3Manifest, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockRejectedValueOnce(new Error('batch restore failed')); + jest.mocked(_restoreNovelAndChapters).mockImplementation(async novel => { + if (novel.id === 2) { + throw new Error('single novel restore failed'); + } + return makeMappings([novel])[0]; + }); + const capture = createProgressCapture(); + + const result = await restoreData('/cache', capture.reporter); + + expect(result.novelMappings).toEqual([ + makeMappings([novels[0]])[0], + makeMappings([novels[2]])[0], + ]); + expect(result).toMatchObject({ novelCount: 2, failedNovelCount: 1 }); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:2/3', + ); + expect( + capture.progressValues.every( + (value, index, values) => index === 0 || value >= values[index - 1], + ), + ).toBe(true); + expect(capture.progressValues.every(value => value < 1)).toBe(true); + }); + + it('validates one later file during a 100-novel write', async () => { + const firstFileNovels = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const secondFileNovel = makeTestNovel(200); + const duplicateNovel = makeTestNovel(201, secondFileNovel.path); + const files = { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstFileNovels)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([secondFileNovel])), + 'batch-000003.json': JSON.stringify(encodeNovelBatch([duplicateNovel])), + }; + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 101, + sections: options, + }); + configureFiles(files, v3Manifest); + const firstWrite = createDeferred(); + const secondFileRead = createDeferred(); + const readFile = jest.mocked(NativeFile.readFile); + const readFileImplementation = readFile.getMockImplementation()!; + readFile.mockImplementation(async path => { + if (path.endsWith('batch-000002.json')) { + secondFileRead.resolve(undefined); + } + return readFileImplementation(path); + }); + let writeCount = 0; + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async (novels, restoreOptions) => { + writeCount++; + restoreOptions?.onProgress?.({ + stage: 'chapters', + completed: 0, + total: novels.length, + }); + if (writeCount === 1) { + return firstWrite.promise; + } + return makeMappings(novels); + }); + const capture = createProgressCapture(); + + const restorePromise = restoreData('/cache', capture.reporter); + await secondFileRead.promise; + + expect(getNovelReadPaths()).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + ]); + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenNthCalledWith( + 1, + firstFileNovels, + expect.objectContaining({ + includeChapterMappings: true, + restoreRunId: expect.any(String), + }), + ); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/101', + ); + expect(capture.progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + + firstWrite.resolve(makeMappings(firstFileNovels)); + const result = await restorePromise; + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:101/101', + ); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(2); + expect(_restoreNovelsAndChapters).toHaveBeenNthCalledWith( + 2, + [secondFileNovel], + expect.objectContaining({ + includeChapterMappings: true, + restoreRunId: result.restoreRunId, + }), + ); + expect(getNovelReadPaths()).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + '/cache/NovelAndChapters/batch-000003.json', + ]); + expect(result).toMatchObject({ + novelCount: 101, + categoryCount: 1, + failedNovelCount: 1, + failedCategoryCount: 0, + pluginIds: ['source'], + novelMappings: [ + ...makeMappings(firstFileNovels), + ...makeMappings([secondFileNovel]), + ], + }); + const [restoredCategory, restoredNovelIdMap] = ( + _restoreCategory as jest.Mock + ).mock.calls[0]; + expect(restoredCategory).toEqual({ + id: 8, + name: 'Reading list', + novelIds: [1, 100, 200], + }); + expect(restoredNovelIdMap.get(1)).toBe(10_001); + expect(restoredNovelIdMap.get(100)).toBe(10_100); + expect(restoredNovelIdMap.get(200)).toBe(10_200); + }); + it('reports completion for empty and invalid files without novel writes', async () => { + configureFiles({ + 'empty.json': JSON.stringify(encodeNovelBatch([])), + 'invalid.json': '{', + }); + const capture = createProgressCapture(); + const result = await restoreNovels( + '/cache', + { + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }, + 'restore-run', + capture.reporter, + ); + + expect(_restoreNovelsAndChapters).not.toHaveBeenCalled(); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/1', + ); + expect(result).toMatchObject({ novelCount: 0, failedNovelCount: 1 }); + }); + + it('settles the active write when the progress updater throws undefined', async () => { + const firstFileNovels = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const nextFileNovel = makeTestNovel(200); + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 101, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstFileNovels)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([nextFileNovel])), + }, + v3Manifest, + ); + const firstWrite = createDeferred(); + let firstWriteSettled = false; + const writePromise = firstWrite.promise.then(mappings => { + firstWriteSettled = true; + return mappings; + }); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async (_batch, restoreOptions) => { + restoreOptions?.onProgress?.({ + stage: 'chapters', + completed: 0, + total: 1, + }); + return writePromise; + }); + const secondFileRead = createDeferred(); + const readFile = jest.mocked(NativeFile.readFile); + const readFileImplementation = readFile.getMockImplementation()!; + readFile.mockImplementation(async path => { + if (path.endsWith('batch-000002.json')) { + secondFileRead.resolve(undefined); + } + return readFileImplementation(path); + }); + const interruption = undefined; + const progressTexts: string[] = []; + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progressText) { + progressTexts.push(metadata.progressText); + } + if (metadata.progress !== undefined && metadata.progress > 0.4) { + throw interruption; + } + }; + + const restorePromise = restoreData( + '/cache', + createRestoreProgressReporter(setMeta, 'local'), + ); + await secondFileRead.promise; + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).not.toHaveBeenCalled(); + expect(progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/101', + ); + expect(progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + + firstWrite.resolve(makeMappings(firstFileNovels)); + await expect(restorePromise).rejects.toBe(interruption); + expect(progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + + expect(firstWriteSettled).toBe(true); + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(clearRestoreChapterMappings).toHaveBeenCalledTimes(1); + }); + + it.each([ + { label: 'Error', interruption: new Error('progress updater failed') }, + { label: 'undefined', interruption: undefined }, + ])( + 'does not start fallback after prefetched progress throws $label', + async ({ interruption }) => { + const firstFileNovels = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const nextFileNovel = makeTestNovel(200); + configureFiles({ + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstFileNovels)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([nextFileNovel])), + }); + + const firstWrite = createDeferred(); + const databaseError = new Error('database batch failed'); + let firstWriteSettled = false; + const writePromise = firstWrite.promise.then( + mappings => { + firstWriteSettled = true; + return mappings; + }, + error => { + firstWriteSettled = true; + throw error; + }, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(() => writePromise); + jest + .mocked(_restoreNovelAndChapters) + .mockImplementation(async novel => makeMappings([novel])[0]); + + const secondFileRead = createDeferred(); + const updaterInterrupted = createDeferred(); + let currentTime = 1_000; + const dateNow = jest + .spyOn(Date, 'now') + .mockImplementation(() => currentTime); + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + let interruptionThrown = false; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if ( + !interruptionThrown && + metadata.progress !== undefined && + metadata.progress > 0.4 + ) { + interruptionThrown = true; + updaterInterrupted.resolve(undefined); + throw interruption; + } + }; + const readFile = jest.mocked(NativeFile.readFile); + const readFileImplementation = readFile.getMockImplementation()!; + readFile.mockImplementation(async path => { + if (path.endsWith('batch-000002.json')) { + currentTime += 250; + secondFileRead.resolve(undefined); + } + return readFileImplementation(path); + }); + jest.mocked(clearRestoreChapterMappings).mockImplementation(async () => { + expect(firstWriteSettled).toBe(true); + }); + + try { + const restorePromise = restoreData( + '/cache', + createRestoreProgressReporter(setMeta, 'local'), + ); + await secondFileRead.promise; + await updaterInterrupted.promise; + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).not.toHaveBeenCalled(); + + firstWrite.reject(databaseError); + await expect(restorePromise).rejects.toBe(interruption); + + expect(firstWriteSettled).toBe(true); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).toHaveBeenCalledTimes(1); + } finally { + dateNow.mockRestore(); + } + }, + ); +}); diff --git a/src/services/backup/restore/__tests__/progress.test.ts b/src/services/backup/restore/__tests__/progress.test.ts new file mode 100644 index 0000000000..704f169071 --- /dev/null +++ b/src/services/backup/restore/__tests__/progress.test.ts @@ -0,0 +1,149 @@ +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { createRestoreProgressReporter } from '../progress'; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + completionText: 'preserved', + }; + const updates: BackgroundTaskMetadata[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + updates.push(metadata); + }; + return { setMeta, updates }; +}; + +describe('restore progress reporter', () => { + afterEach(() => { + jest.useRealTimers(); + }); + it('maps local phases into benchmark-weighted ranges and reaches one at finalize', () => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + reporter('source', 1, 'copy'); + reporter('extract', 1, 'extract'); + reporter('manifest', 1, 'manifest'); + reporter('novels', 1, 'novels'); + reporter('categories', 1, 'categories'); + reporter('settings', 1, 'settings'); + reporter('plugins', 1, 'plugins'); + reporter('selectedFiles', 1, 'selected files'); + reporter('finalize', 0, 'finishing'); + reporter('finalize', 1, 'done'); + + const values = updates.map(update => update.progress!); + expect(values[0]).toBeCloseTo(0.09); + expect(values[1]).toBeCloseTo(0.3); + expect(values[2]).toBeCloseTo(0.302); + expect(values[3]).toBeCloseTo(0.982); + expect(values[4]).toBeCloseTo(0.986); + expect(values[5]).toBeCloseTo(0.988); + expect(values[6]).toBeCloseTo(0.992); + expect(values[7]).toBeCloseTo(0.997); + expect(values[8]).toBeCloseTo(0.997); + expect(values[9]).toBe(1); + expect( + values.every((value, index) => index === 0 || value >= values[index - 1]), + ).toBe(true); + expect(updates.every(update => update.completionText === 'preserved')).toBe( + true, + ); + }); + + it('combines remote retrieval and extraction in the 0.30 source range', () => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'remote')!; + + reporter('source', 1, 'download'); + reporter('extract', 1, 'download'); + reporter('manifest', 1, 'manifest'); + + expect(updates[0].progress).toBeCloseTo(0.3); + expect(updates[updates.length - 1].progress).toBeCloseTo(0.302); + }); + it.each(['local', 'remote'] as const)( + 'reserves completion for finalization fraction one for %s restores', + source => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, source)!; + + for (const fraction of [0, 0.25, 0.75, 1 - Number.EPSILON]) { + reporter('finalize', fraction, `finishing ${fraction}`, true); + } + + const partialValues = updates.map(update => update.progress!); + expect(partialValues.every(value => value < 1)).toBe(true); + expect( + partialValues.every( + (value, index) => index === 0 || value >= partialValues[index - 1], + ), + ).toBe(true); + + reporter('finalize', 1, 'done', true); + expect(updates[updates.length - 1].progress).toBe(1); + }, + ); + + it('clamps fractions and keeps progress monotonic across interleaved phases', () => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + reporter('source', -4, 'start'); + reporter('extract', 2, 'extracted'); + reporter('source', 0, 'interleaved source'); + reporter('manifest', Number.NaN, 'manifest'); + + const values = updates.map(update => update.progress!); + expect(values[0]).toBe(0); + expect(values[1]).toBeCloseTo(0.3); + expect(values[2]).toBeCloseTo(0.3); + expect(values[3]).toBeCloseTo(0.3); + expect(values.every(value => value >= 0 && value <= 1)).toBe(true); + expect( + values.every((value, index) => index === 0 || value >= values[index - 1]), + ).toBe(true); + }); + + it('throttles ordinary updates but emits phase changes, forced updates, and completion', () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + reporter('source', 0, 'source'); + reporter('source', 0.2, 'ordinary'); + expect(updates).toHaveLength(1); + + jest.advanceTimersByTime(250); + reporter('source', 0.2, 'ordinary'); + expect(updates).toHaveLength(2); + + reporter('source', 0.3, 'forced', true); + reporter('source', 0.3, 'forced', true); + expect(updates).toHaveLength(3); + + reporter('extract', 0, 'next phase'); + expect(updates).toHaveLength(4); + reporter('extract', 1, 'next phase complete'); + expect(updates).toHaveLength(5); + }); + + it('does not swallow updater exceptions', () => { + const setMeta: TaskProgressUpdater = () => { + throw new Error('metadata update failed'); + }; + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + expect(() => reporter('source', 0, 'copy')).toThrow( + 'metadata update failed', + ); + }); +}); diff --git a/src/services/backup/__tests__/restoreResult.test.ts b/src/services/backup/restore/__tests__/result.test.ts similarity index 98% rename from src/services/backup/__tests__/restoreResult.test.ts rename to src/services/backup/restore/__tests__/result.test.ts index 74932e74cc..7b14243602 100644 --- a/src/services/backup/__tests__/restoreResult.test.ts +++ b/src/services/backup/restore/__tests__/result.test.ts @@ -5,7 +5,7 @@ import { getMissingRestorePluginIds, getRestoreCompletionText, type RestoreResult, -} from '../restoreResult'; +} from '../result'; jest.mock('@i18n/translations', () => ({ getString: (key: string, options?: Record) => { @@ -65,6 +65,7 @@ const successfulResult: RestoreResult = { failedSectionCount: 0, pluginIds: ['installed'], novelMappings: [], + restoreRunId: 'restore-run-1', manifest: { appVersion: '2.1.0', formatVersion: 2, diff --git a/src/services/backup/restore/__tests__/selfhost.test.ts b/src/services/backup/restore/__tests__/selfhost.test.ts new file mode 100644 index 0000000000..44b35f745d --- /dev/null +++ b/src/services/backup/restore/__tests__/selfhost.test.ts @@ -0,0 +1,258 @@ +import { download } from '@api/remote'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { CACHE_DIR_PATH, clearBackupCache } from '../../cache'; +import { ZipBackupName } from '../../types'; +import { restoreLegacyFiles, restoreNovelFiles } from '../files'; +import { selfHostRestore } from '../selfhost'; + +import { restoreData } from '../index'; +import { + finalizeRestoredPlugins, + getRestoreCompletionText, + type RestoreResult, +} from '../result'; +import type { RestoreProgressReporter } from '../progress'; + +jest.mock('@api/remote', () => ({ download: jest.fn() })); +jest.mock('../../cache', () => ({ + CACHE_DIR_PATH: '/cache/BackupData', + clearBackupCache: jest.fn(), +})); +jest.mock('../files', () => ({ + getLegacyFilesRestorePath: jest.fn(() => '/cache/legacy-files'), + getNovelFilesRestorePath: jest.fn(() => '/cache/novel-files'), + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), +})); +jest.mock('../index', () => ({ + clearRestoreChapterMappingsSafely: jest.fn(), + restoreData: jest.fn(), +})); +jest.mock('../result', () => ({ + finalizeRestoredPlugins: jest.fn(), + getRestoreCompletionText: jest.fn(() => 'completed'), +})); +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', +})); + +const backupFolder = 'backup-folder'; +const host = 'https://example.invalid'; +const restoreDataFor = ( + plugins: boolean, + downloadedFiles = false, + formatVersion: 1 | 2 | 3 = 3, + novelMappings: RestoreResult['novelMappings'] = [], +): RestoreResult => ({ + novelCount: 0, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: plugins ? ['restored-plugin'] : [], + novelMappings, + restoreRunId: 'selfhost-restore-run', + manifest: { + appVersion: '2.1.3', + formatVersion, + novelDataFormat: 2, + sections: { + library: downloadedFiles, + settings: false, + plugins, + downloadedFiles, + }, + }, +}); + +const reportRestoreStages = (reporter: RestoreProgressReporter | undefined) => { + reporter?.('manifest', 1, 'manifest', true); + reporter?.('novels', 1, 'novels', true); + reporter?.('categories', 1, 'categories', true); + reporter?.('settings', 1, 'settings', true); + reporter?.('plugins', 1, 'plugins', true); +}; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'SELF_HOST_RESTORE', + isRunning: false, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + }; + return { progressValues, setMeta, getMetadata: () => metadata }; +}; + +describe('self-host restore progress', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(download).mockResolvedValue(''); + jest.mocked(clearBackupCache).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValue([]); + }); + + it('keeps remote retrieval and later phases monotonic through completion', async () => { + const result = restoreDataFor(false); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await selfHostRestore({ host, backupFolder }, capture.setMeta); + + expect(capture.progressValues[0]).toBe(0); + expect( + capture.progressValues.some(value => Math.abs(value - 0.3) < 0.000001), + ).toBe(true); + expect(capture.progressValues[capture.progressValues.length - 1]).toBe(1); + expect( + capture.progressValues.every( + (value, index) => + index === 0 || value >= capture.progressValues[index - 1], + ), + ).toBe(true); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.DATA, + CACHE_DIR_PATH, + ); + }); + + it('does not report completion when a selected archive download fails', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = restoreDataFor(true, true, 2, mappings); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + jest.mocked(download).mockImplementation(async (_host, _folder, name) => { + if (name === ZipBackupName.NOVEL_FILES) { + throw new Error('selected archive download failed'); + } + return ''; + }); + const capture = createProgressCapture(); + + await expect( + selfHostRestore({ host, backupFolder }, capture.setMeta), + ).rejects.toThrow('selected archive download failed'); + + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.NOVEL_FILES, + '/cache/novel-files', + ); + expect(restoreNovelFiles).not.toHaveBeenCalled(); + expect(capture.progressValues.length).toBeGreaterThan(0); + expect(Math.max(...capture.progressValues)).toBeLessThan(1); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + }); + + it('restores selected format-2 archives before finalizing plugins', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = restoreDataFor(true, true, 2, mappings); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await selfHostRestore({ host, backupFolder }, capture.setMeta); + + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.PLUGINS, + '/storage/Plugins', + ); + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.NOVEL_FILES, + '/cache/novel-files', + ); + expect(restoreNovelFiles).toHaveBeenCalledWith( + '/cache/novel-files', + mappings, + result.restoreRunId, + expect.any(Function), + ); + expect( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect(finalizeRestoredPlugins).toHaveBeenCalledWith(result); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ); + expect( + jest.mocked(getRestoreCompletionText).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + }); + + it('restores format-1 downloads from the legacy staging path', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = restoreDataFor(false, true, 1, mappings); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await selfHostRestore({ host, backupFolder }, capture.setMeta); + + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.DOWNLOAD, + '/cache/legacy-files', + ); + expect(restoreLegacyFiles).toHaveBeenCalledWith( + '/cache/legacy-files', + mappings, + result.restoreRunId, + expect.any(Function), + ); + expect(restoreNovelFiles).not.toHaveBeenCalled(); + expect(capture.getMetadata().completionText).toBe('completed'); + }); +}); diff --git a/src/services/backup/restore/__tests__/validation.test.ts b/src/services/backup/restore/__tests__/validation.test.ts new file mode 100644 index 0000000000..01ab70d641 --- /dev/null +++ b/src/services/backup/restore/__tests__/validation.test.ts @@ -0,0 +1,763 @@ +import type { BackupNovel, ChapterInfo } from '@database/types'; +import { decodeNovelBatch, encodeNovelBatch } from '../../novelPayload'; +import type { ResolvedBackupManifest } from '../../types'; +import { + decodeAndValidateNovelFile, + decodeRestoreNovelFile, + getNovelFileRecordCount, + NovelFileValidationError, +} from '../validation'; + +jest.mock('@utils/Storages', () => ({ ROOT_STORAGE: '/storage' })); + +const sections = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, +}; + +const makeManifest = (novelDataFormat: 1 | 2): ResolvedBackupManifest => ({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat, + sections, +}); + +const legacyManifest = makeManifest(1); +const compactManifest = makeManifest(2); + +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: `2024-03-${String(chapterNumber).padStart(2, '0')}`, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); + +type MutableLegacyNovel = Record & { + chapters: Record[]; +}; + +type MutableCompactNovel = Record & { c: unknown[][] }; + +type SeenRecords = { + novelIds: Set; + novelIdentities: Set; + chapterIdentities: Set; +}; + +const newSeenRecords = (): SeenRecords => ({ + novelIds: new Set(), + novelIdentities: new Set(), + chapterIdentities: new Set(), +}); + +const makeLegacyPayload = (): MutableLegacyNovel => ({ + id: 1, + path: '/legacy/one', + pluginId: 'legacy-source', + name: 'Legacy novel', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 101, + novelId: 1, + path: '/legacy/one/chapter/1', + name: 'Legacy chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], +}); + +const makeCompactRecord = ( + novel: BackupNovel = makeTestNovel(1), +): MutableCompactNovel => + encodeNovelBatch([novel])[0] as unknown as MutableCompactNovel; + +const setLegacyNovelField = (field: string, value: unknown) => { + const payload = makeLegacyPayload(); + payload[field] = value; + return payload; +}; + +const setLegacyChapterField = (field: string, value: unknown) => { + const payload = makeLegacyPayload(); + const chapter = payload.chapters[0]; + if (!chapter) { + throw new Error('Expected a legacy chapter fixture'); + } + chapter[field] = value; + return payload; +}; + +const setCompactNovelField = (field: string, value: unknown) => { + const record = makeCompactRecord(); + record[field] = value; + return [record]; +}; + +const setCompactChapterField = (index: number, value: unknown) => { + const record = makeCompactRecord(); + const chapter = record.c[0]; + if (!chapter) { + throw new Error('Expected a compact chapter fixture'); + } + chapter[index] = value; + return [record]; +}; + +const encodeFile = (novels: BackupNovel[]) => + JSON.stringify(encodeNovelBatch(novels)); + +const captureValidationError = ( + fileContent: string, + manifest: ResolvedBackupManifest, + seen: SeenRecords = newSeenRecords(), +): NovelFileValidationError => { + try { + decodeAndValidateNovelFile( + fileContent, + manifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ); + } catch (error) { + expect(error).toBeInstanceOf(NovelFileValidationError); + return error as NovelFileValidationError; + } + throw new Error('Expected novel file validation to fail'); +}; + +describe('decodeRestoreNovelFile', () => { + it('normalizes legacy values and ignores fields outside the backup shape', () => { + const payload = { + id: 22, + path: '/legacy/empty-name', + pluginId: 'legacy-source', + name: '', + cover: '/covers/legacy.jpg', + chapters: [ + { + id: 2201, + novelId: 999, + path: '/legacy/empty-name/chapter/1', + name: '', + extraChapterField: 'discard me', + }, + ], + extraNovelField: 'discard me', + }; + + expect(decodeRestoreNovelFile(payload, legacyManifest)).toEqual([ + { + id: 22, + path: '/legacy/empty-name', + pluginId: 'legacy-source', + name: '', + cover: 'file:///storage/covers/legacy.jpg', + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 2201, + novelId: 22, + path: '/legacy/empty-name/chapter/1', + name: '', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], + }, + ]); + }); + + it('maps every compact chapter tuple field and removes restore-only aggregates', () => { + const chapter: ChapterInfo = { + id: 1402, + novelId: 14, + path: '/novel/14/chapter/2', + name: 'Mapped chapter', + releaseTime: 'released-at', + bookmark: true, + unread: false, + readTime: 'read-at', + isDownloaded: false, + updatedTime: 'updated-at', + chapterNumber: 2.5, + page: 'page-token', + position: 3.5, + progress: 0.625, + scanlator: 'scanlator-team', + timeSpent: 91, + }; + const novel: BackupNovel = { + ...makeTestNovel(14, 'compact-source', [chapter]), + name: 'Compact mapping novel', + path: '/novel/14', + cover: '/covers/round-trip.jpg', + }; + const novelWithAggregates = { + ...novel, + chaptersDownloaded: 4, + chaptersUnread: 6, + totalChapters: 12, + lastReadAt: '2024-03-01T10:20:30.000Z', + lastUpdatedAt: '2024-03-02T10:20:30.000Z', + }; + const compactPayload = encodeNovelBatch([novelWithAggregates]); + + expect(compactPayload[0]?.c).toEqual([ + [ + 1402, + '/novel/14/chapter/2', + 'Mapped chapter', + 'released-at', + true, + false, + 'read-at', + false, + 'updated-at', + 2.5, + 'page-token', + 3.5, + 0.625, + 'scanlator-team', + 91, + ], + ]); + expect(decodeNovelBatch(compactPayload)).toEqual([novelWithAggregates]); + + expect(decodeRestoreNovelFile(compactPayload, compactManifest)).toEqual([ + { + ...novel, + cover: 'file:///storage/covers/round-trip.jpg', + }, + ]); + }); + + it.each([ + ['HTTP', 'https://cdn.example/cover.jpg', 'https://cdn.example/cover.jpg'], + [ + '/covers/relative.jpg', + '/covers/relative.jpg', + 'file:///storage/covers/relative.jpg', + ], + ['empty', '', ''], + ['null', null, null], + ] as const)( + 'keeps the existing %s compact cover handling', + (_description, cover, expectedCover) => { + const novel = { ...makeTestNovel(40), cover }; + const [restored] = decodeRestoreNovelFile( + encodeNovelBatch([novel]), + compactManifest, + ); + + expect(restored?.cover).toBe(expectedCover); + }, + ); +}); + +type InvalidKind = 'string' | 'boolean' | 'number'; +type MalformedInputCase = { + label: string; + manifest: ResolvedBackupManifest; + payload: () => unknown; +}; + +const invalidValueForKind: Record = { + string: 42, + boolean: 'not-a-boolean', + number: 'not-a-number', +}; + +const nullableFieldCases = ( + prefix: string, + manifest: ResolvedBackupManifest, + fields: readonly (readonly [string, InvalidKind])[], + makePayload: (field: string, value: unknown) => unknown, +): MalformedInputCase[] => + fields.map(([field, kind]) => ({ + label: `${prefix} ${field}`, + manifest, + payload: () => makePayload(field, invalidValueForKind[kind]), + })); + +const compactChapterNullableFields: readonly (readonly [ + string, + number, + InvalidKind, +])[] = [ + ['releaseTime', 3, 'string'], + ['bookmark', 4, 'boolean'], + ['unread', 5, 'boolean'], + ['readTime', 6, 'string'], + ['isDownloaded', 7, 'boolean'], + ['updatedTime', 8, 'string'], + ['chapterNumber', 9, 'number'], + ['page', 10, 'string'], + ['position', 11, 'number'], + ['progress', 12, 'number'], + ['scanlator', 13, 'string'], + ['timeSpent', 14, 'number'], +]; + +const malformedInputCases: MalformedInputCase[] = [ + { + label: 'legacy novel id', + manifest: legacyManifest, + payload: () => setLegacyNovelField('id', 0), + }, + { + label: 'legacy novel path', + manifest: legacyManifest, + payload: () => setLegacyNovelField('path', ''), + }, + { + label: 'legacy plugin id', + manifest: legacyManifest, + payload: () => setLegacyNovelField('pluginId', ''), + }, + { + label: 'legacy novel name type', + manifest: legacyManifest, + payload: () => setLegacyNovelField('name', null), + }, + { + label: 'legacy chapter id', + manifest: legacyManifest, + payload: () => setLegacyChapterField('id', 0), + }, + { + label: 'legacy chapter novel id', + manifest: legacyManifest, + payload: () => setLegacyChapterField('novelId', 0), + }, + { + label: 'legacy chapter path', + manifest: legacyManifest, + payload: () => setLegacyChapterField('path', ''), + }, + { + label: 'legacy chapter name type', + manifest: legacyManifest, + payload: () => setLegacyChapterField('name', false), + }, + { + label: 'legacy chapters collection', + manifest: legacyManifest, + payload: () => setLegacyNovelField('chapters', {}), + }, + ...nullableFieldCases( + 'legacy novel', + legacyManifest, + [ + ['cover', 'string'], + ['summary', 'string'], + ['author', 'string'], + ['artist', 'string'], + ['status', 'string'], + ['genres', 'string'], + ['inLibrary', 'boolean'], + ['isLocal', 'boolean'], + ['totalPages', 'number'], + ], + setLegacyNovelField, + ), + ...nullableFieldCases( + 'legacy chapter', + legacyManifest, + [ + ['releaseTime', 'string'], + ['readTime', 'string'], + ['bookmark', 'boolean'], + ['unread', 'boolean'], + ['isDownloaded', 'boolean'], + ['updatedTime', 'string'], + ['chapterNumber', 'number'], + ['page', 'string'], + ['progress', 'number'], + ['position', 'number'], + ['scanlator', 'string'], + ['timeSpent', 'number'], + ], + setLegacyChapterField, + ), + { + label: 'compact novel id', + manifest: compactManifest, + payload: () => setCompactNovelField('id', 0), + }, + { + label: 'compact novel path', + manifest: compactManifest, + payload: () => setCompactNovelField('p', ''), + }, + { + label: 'compact plugin id', + manifest: compactManifest, + payload: () => setCompactNovelField('pi', ''), + }, + { + label: 'compact novel name type', + manifest: compactManifest, + payload: () => setCompactNovelField('n', null), + }, + { + label: 'compact chapter id', + manifest: compactManifest, + payload: () => setCompactChapterField(0, 0), + }, + { + label: 'compact chapter path', + manifest: compactManifest, + payload: () => setCompactChapterField(1, ''), + }, + { + label: 'compact chapter name type', + manifest: compactManifest, + payload: () => setCompactChapterField(2, false), + }, + { + label: 'compact chapter collection', + manifest: compactManifest, + payload: () => setCompactNovelField('c', {}), + }, + ...nullableFieldCases( + 'compact novel', + compactManifest, + [ + ['co', 'string'], + ['s', 'string'], + ['a', 'string'], + ['ar', 'string'], + ['st', 'string'], + ['g', 'string'], + ['l', 'boolean'], + ['lo', 'boolean'], + ['t', 'number'], + ['d', 'number'], + ['u', 'number'], + ['tc', 'number'], + ['lr', 'string'], + ['lu', 'string'], + ], + setCompactNovelField, + ), + ...compactChapterNullableFields.map(([field, index, kind]) => ({ + label: `compact chapter ${field}`, + manifest: compactManifest, + payload: () => setCompactChapterField(index, invalidValueForKind[kind]), + })), +]; + +describe('malformed backup records', () => { + it.each(malformedInputCases)('rejects $label', ({ manifest, payload }) => { + expect(() => decodeRestoreNovelFile(payload(), manifest)).toThrow(); + }); + + it.each([ + ['missing', (record: MutableCompactNovel) => delete record.p], + ['extra', (record: MutableCompactNovel) => (record.extra = true)], + ])('rejects compact records with %s novel keys', (_shape, mutate) => { + const record = makeCompactRecord(); + mutate(record); + + expect(() => decodeNovelBatch([record])).toThrow(); + }); + + it.each([ + ['short', 14], + ['long', 16], + ] as const)('rejects a %s compact chapter tuple', (_shape, length) => { + const record = makeCompactRecord(); + const chapter = record.c[0]; + if (!chapter) { + throw new Error('Expected a compact chapter fixture'); + } + record.c[0] = + length === 14 ? chapter.slice(0, length) : [...chapter, 'extra']; + + expect(() => decodeNovelBatch([record])).toThrow( + 'Invalid compact novel record', + ); + }); + + it.each([ + ['novel id', () => setCompactNovelField('id', Number.NaN)], + [ + 'aggregate count', + () => setCompactNovelField('d', Number.POSITIVE_INFINITY), + ], + [ + 'chapter number', + () => setCompactChapterField(9, Number.NEGATIVE_INFINITY), + ], + ])('rejects non-finite %s values in the compact codec', (_field, payload) => { + expect(() => decodeNovelBatch(payload())).toThrow( + 'Invalid compact novel record', + ); + }); +}); + +describe('decodeAndValidateNovelFile', () => { + it.each([ + { + label: 'novel ID', + novels: [makeTestNovel(31), makeTestNovel(31, 'other-source')], + message: 'Duplicate backup novel identity', + }, + { + label: 'plugin and path identity', + novels: [ + makeTestNovel(32, 'source'), + { ...makeTestNovel(33, 'source'), path: '/novel/32' }, + ], + message: 'Duplicate backup novel identity', + }, + { + label: 'chapter path identity', + novels: [ + makeTestNovel(34, 'source', [ + makeTestChapter(34, 1), + { + ...makeTestChapter(34, 2), + path: makeTestChapter(34, 1).path, + }, + ]), + ], + message: 'Duplicate backup chapter identity', + }, + ])('rejects duplicate $label values within a file', ({ novels, message }) => { + expect( + captureValidationError(encodeFile(novels), compactManifest).message, + ).toBe(message); + }); + + it.each([ + { + label: 'novel ID', + candidate: (accepted: BackupNovel) => ({ + ...makeTestNovel(502, 'other-source'), + id: accepted.id, + path: '/novel/different-path', + }), + }, + { + label: 'plugin and path identity', + candidate: (accepted: BackupNovel) => ({ + ...makeTestNovel(503, accepted.pluginId), + path: accepted.path, + }), + }, + ])( + 'rejects duplicate $label identities from a prior file', + ({ candidate }) => { + const seen = newSeenRecords(); + const accepted = makeTestNovel(501, 'source'); + expect( + decodeAndValidateNovelFile( + encodeFile([accepted]), + compactManifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ), + ).toHaveLength(1); + + const error = captureValidationError( + encodeFile([candidate(accepted)]), + compactManifest, + seen, + ); + expect(error.message).toBe('Duplicate backup novel identity'); + }, + ); + + it('preserves all identity sets after late failure and accepts valid records', () => { + const seen: SeenRecords = { + novelIds: new Set([900]), + novelIdentities: new Set(['prior-source\u0000/prior-novel']), + chapterIdentities: new Set(['900\u0000/prior-chapter']), + }; + const before = { + novelIds: new Set(seen.novelIds), + novelIdentities: new Set(seen.novelIdentities), + chapterIdentities: new Set(seen.chapterIdentities), + }; + const first = makeTestNovel(601); + const second = makeTestNovel(602); + const duplicateChapter = makeTestChapter(603, 1); + const lateFailure = makeTestNovel(603, 'source', [ + duplicateChapter, + { ...makeTestChapter(603, 2), path: duplicateChapter.path }, + ]); + + const error = captureValidationError( + encodeFile([first, second, lateFailure]), + compactManifest, + seen, + ); + + expect(error.recordCount).toBe(3); + expect(seen.novelIds).toEqual(before.novelIds); + expect(seen.novelIdentities).toEqual(before.novelIdentities); + expect(seen.chapterIdentities).toEqual(before.chapterIdentities); + + expect( + decodeAndValidateNovelFile( + encodeFile([first, second]), + compactManifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ), + ).toHaveLength(2); + expect([...seen.novelIds]).toEqual([900, first.id, second.id]); + expect(seen.novelIdentities.size).toBe(3); + expect(seen.chapterIdentities.size).toBe(3); + }); + + it('counts a late malformed chapter without accepting earlier records', () => { + const seen = newSeenRecords(); + const first = makeTestNovel(701); + const second = makeTestNovel(702); + const last = makeCompactRecord(makeTestNovel(703)); + const lastChapter = last.c[0]; + if (!lastChapter) { + throw new Error('Expected a compact chapter fixture'); + } + lastChapter[1] = ''; + + const error = captureValidationError( + JSON.stringify([ + makeCompactRecord(first), + makeCompactRecord(second), + last, + ]), + compactManifest, + seen, + ); + + expect(error.recordCount).toBe(3); + expect(seen.novelIds.size).toBe(0); + expect(seen.novelIdentities.size).toBe(0); + expect(seen.chapterIdentities.size).toBe(0); + }); + + it('counts failures and accepts an empty compact batch', () => { + expect(captureValidationError('{', compactManifest).recordCount).toBe(1); + expect(captureValidationError('{}', compactManifest).recordCount).toBe(1); + expect( + captureValidationError(JSON.stringify({}), legacyManifest).recordCount, + ).toBe(1); + + const oversizedPayload = Array.from({ length: 101 }, () => null); + expect(getNovelFileRecordCount(oversizedPayload, compactManifest)).toBe( + 101, + ); + expect( + captureValidationError(JSON.stringify(oversizedPayload), compactManifest) + .recordCount, + ).toBe(101); + + const malformedRecord = makeCompactRecord(); + malformedRecord.n = null; + expect( + captureValidationError( + JSON.stringify([makeCompactRecord(), malformedRecord]), + compactManifest, + ).recordCount, + ).toBe(2); + + expect(getNovelFileRecordCount({}, compactManifest)).toBe(1); + expect(getNovelFileRecordCount([], compactManifest)).toBe(1); + expect(getNovelFileRecordCount([null, null], compactManifest)).toBe(2); + expect(getNovelFileRecordCount([null, null], legacyManifest)).toBe(1); + expect(getNovelFileRecordCount({}, legacyManifest)).toBe(1); + + expect(decodeRestoreNovelFile([], compactManifest)).toEqual([]); + const seen = newSeenRecords(); + expect( + decodeAndValidateNovelFile( + '[]', + compactManifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ), + ).toEqual([]); + expect(seen.novelIds.size).toBe(0); + expect(seen.novelIdentities.size).toBe(0); + expect(seen.chapterIdentities.size).toBe(0); + }); +}); diff --git a/src/services/backup/restore/drive.ts b/src/services/backup/restore/drive.ts new file mode 100644 index 0000000000..22f32db126 --- /dev/null +++ b/src/services/backup/restore/drive.ts @@ -0,0 +1,43 @@ +import { DriveFile } from '@api/drive/types'; +import { exists } from '@api/drive'; +import { download } from '@api/drive/request'; +import { getString } from '@i18n/translations'; +import { CACHE_DIR_PATH, clearBackupCache } from '../cache'; +import { ZipBackupName } from '../types'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; +import { runRestore } from './run'; + +export const driveRestore = async ( + backupFolder: DriveFile, + setMeta: TaskProgressUpdater, +) => { + await runRestore( + { + kind: 'remote', + prepare: async report => { + const progressText = getString('backupScreen.downloadingData'); + report?.('source', 0, progressText, true); + const dataFile = await exists( + ZipBackupName.DATA, + false, + backupFolder.id, + ); + if (!dataFile) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + await clearBackupCache(); + await download(dataFile, CACHE_DIR_PATH); + report?.('source', 1, progressText, true); + report?.('extract', 1, progressText, true); + }, + loadArchive: async (name, destination) => { + const archive = await exists(name, false, backupFolder.id); + if (!archive) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + await download(archive, destination); + }, + }, + setMeta, + ); +}; diff --git a/src/services/backup/restore/files.ts b/src/services/backup/restore/files.ts new file mode 100644 index 0000000000..495849f853 --- /dev/null +++ b/src/services/backup/restore/files.ts @@ -0,0 +1,116 @@ +import { NOVEL_STORAGE, PLUGIN_STORAGE } from '@utils/Storages'; +import NativeFile from '@modules/native-file'; +import type { RestoredNovelMapping } from '@database/types'; +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; + +export const getNovelFilesRestorePath = (cacheDirPath: string) => + `${cacheDirPath}/RestoredNovelFiles`; + +export const getLegacyFilesRestorePath = (cacheDirPath: string) => + `${cacheDirPath}/RestoredLegacyFiles`; + +const moveDirectoryContents = async (source: string, destination: string) => { + if (!(await NativeFile.exists(source))) { + return; + } + await NativeFile.mkdir(destination); + for (const item of await NativeFile.readDir(source)) { + const destinationPath = `${destination}/${item.name}`; + if (item.isDirectory) { + await moveDirectoryContents(item.path, destinationPath); + } else { + await NativeFile.moveFile(item.path, destinationPath); + } + } +}; + +const RESTORE_CHAPTER_LOOKUP_BATCH_SIZE = 100; + +export const restoreNovelFiles = async ( + stagingPath: string, + novelMappings: RestoredNovelMapping[], + restoreRunId: string, + onProgress?: (completed: number, total: number) => void, +) => { + const total = novelMappings.length + 1; + for (const [index, mapping] of novelMappings.entries()) { + const sourceNovelPath = `${stagingPath}/${mapping.pluginId}/${mapping.backupNovelId}`; + if (!(await NativeFile.exists(sourceNovelPath))) { + onProgress?.(index + 1, total); + continue; + } + + const destinationNovelPath = `${NOVEL_STORAGE}/${mapping.pluginId}/${mapping.restoredNovelId}`; + await NativeFile.mkdir(destinationNovelPath); + const items = await NativeFile.readDir(sourceNovelPath); + for (const item of items) { + if (!item.isDirectory) { + await NativeFile.moveFile( + item.path, + `${destinationNovelPath}/${item.name}`, + ); + } + } + + const chapterItems = items.filter(item => item.isDirectory); + for ( + let start = 0; + start < chapterItems.length; + start += RESTORE_CHAPTER_LOOKUP_BATCH_SIZE + ) { + const chapterBatch = chapterItems.slice( + start, + start + RESTORE_CHAPTER_LOOKUP_BATCH_SIZE, + ); + const backupChapterIds = chapterBatch + .map(item => Number(item.name)) + .filter(id => Number.isInteger(id) && id > 0); + const chapterMappings = await getRestoreChapterMappings( + restoreRunId, + mapping.backupNovelId, + backupChapterIds, + ); + const restoredChapterIds = new Map( + chapterMappings.map(chapter => [ + String(chapter.backupChapterId), + chapter.restoredChapterId, + ]), + ); + for (const item of chapterBatch) { + const restoredChapterId = restoredChapterIds.get(item.name); + if (restoredChapterId !== undefined) { + await moveDirectoryContents( + item.path, + `${destinationNovelPath}/${restoredChapterId}`, + ); + } + } + } + onProgress?.(index + 1, total); + } + + if (await NativeFile.exists(stagingPath)) { + await NativeFile.unlink(stagingPath); + } + onProgress?.(total, total); +}; + +export const restoreLegacyFiles = async ( + stagingPath: string, + novelMappings: RestoredNovelMapping[], + restoreRunId: string, + onProgress?: (completed: number, total: number) => void, +) => { + const total = novelMappings.length + 2; + await moveDirectoryContents(`${stagingPath}/Plugins`, PLUGIN_STORAGE); + await restoreNovelFiles( + `${stagingPath}/Novels`, + novelMappings, + restoreRunId, + onProgress ? completed => onProgress(completed, total) : undefined, + ); + if (await NativeFile.exists(stagingPath)) { + await NativeFile.unlink(stagingPath); + } + onProgress?.(total, total); +}; diff --git a/src/services/backup/restore/index.ts b/src/services/backup/restore/index.ts new file mode 100644 index 0000000000..85a0c3f851 --- /dev/null +++ b/src/services/backup/restore/index.ts @@ -0,0 +1,318 @@ +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { _restoreCategory } from '@database/queries/CategoryQueries'; +import type { BackupCategory } from '@database/types'; +import type { RestoreProgressReporter } from './progress'; +import NativeFile from '@modules/native-file'; +import { getString } from '@i18n/translations'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { INSTALLED_PLUGINS_KEY } from '@plugins/pluginManager'; +import type { PluginItem } from '@plugins/types'; +import { DEFAULT_BACKUP_OPTIONS, resolveBackupOptions } from '../options'; +import { + BackupEntryName, + type BackupManifest, + type ResolvedBackupManifest, +} from '../types'; +import { restoreNovels } from './novels'; +import type { RestoreResult } from './result'; + +const parsePluginList = (value: unknown): PluginItem[] => { + const parsed: unknown = typeof value === 'string' ? JSON.parse(value) : value; + if (!Array.isArray(parsed)) { + throw new Error('Invalid installed plugin registry'); + } + return parsed as PluginItem[]; +}; + +const restoreMMKVData = (data: Record) => { + for (const key in data) { + const value = data[key]; + if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' + ) { + throw new Error('Invalid backup setting'); + } + MMKVStorage.set(key, value); + } +}; + +const getBackupManifest = async ( + cacheDirPath: string, +): Promise => { + try { + const fileContent = await NativeFile.readFile( + cacheDirPath + '/' + BackupEntryName.VERSION, + ); + const data = JSON.parse(fileContent) as Partial & { + version?: string; + }; + if ( + (data.formatVersion === 2 || data.formatVersion === 3) && + data.sections + ) { + const novelDataFormat = + data.novelDataFormat === 1 || data.novelDataFormat === 2 + ? data.novelDataFormat + : undefined; + const novelCount = + typeof data.novelCount === 'number' && + Number.isSafeInteger(data.novelCount) && + data.novelCount >= 0 + ? data.novelCount + : undefined; + return { + appVersion: data.appVersion ?? data.version ?? '', + formatVersion: data.formatVersion, + ...(novelDataFormat === undefined ? {} : { novelDataFormat }), + ...((data.formatVersion === 3 || novelDataFormat === 2) && + novelCount !== undefined + ? { novelCount } + : {}), + sections: resolveBackupOptions(data.sections), + }; + } + + return { + appVersion: data.version, + formatVersion: 1, + sections: DEFAULT_BACKUP_OPTIONS, + }; + } catch { + return { + formatVersion: 1, + sections: DEFAULT_BACKUP_OPTIONS, + }; + } +}; + +const createRestoreRunId = () => + `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + +type RestoreBenchmarkLogger = (message: string) => void; + +const restoreDataInternal = async ( + cacheDirPath: string, + progressReporter: RestoreProgressReporter | undefined, + benchmarkLog: RestoreBenchmarkLogger | undefined, + restoreRunId: string, +): Promise => { + progressReporter?.( + 'manifest', + 0, + getString('backupScreen.restoringData'), + true, + ); + const manifest = await getBackupManifest(cacheDirPath); + benchmarkLog?.('restoreData:manifest:loaded'); + progressReporter?.( + 'manifest', + 1, + getString('backupScreen.restoringData'), + true, + ); + const novelSummary = manifest.sections.library + ? await restoreNovels( + cacheDirPath, + manifest, + restoreRunId, + progressReporter, + benchmarkLog, + ) + : { + novelCount: 0, + failedNovelCount: 0, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + novelIdMap: new Map(), + }; + if (!manifest.sections.library) { + progressReporter?.( + 'novels', + 1, + getString('backupScreen.restoringNovels'), + true, + ); + } + let failedSectionCount = novelSummary.failedSectionCount; + const novelIdMap = novelSummary.novelIdMap; + const installedPluginsBeforeRestore = (() => { + try { + return parsePluginList( + MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', + ); + } catch { + return []; + } + })(); + let pluginsFromSettings: PluginItem[] = []; + + benchmarkLog?.('restoreData:categories:start'); + const categoriesProgressText = getString('backupScreen.restoringCategories'); + let categoryCount = 0; + let failedCategoryCount = 0; + if (!manifest.sections.library) { + progressReporter?.('categories', 1, categoriesProgressText, true); + } else { + progressReporter?.('categories', 0, categoriesProgressText, true); + const categoryFilePath = cacheDirPath + '/' + BackupEntryName.CATEGORY; + let categories: BackupCategory[] | undefined; + if (!(await NativeFile.exists(categoryFilePath))) { + failedSectionCount++; + } else { + try { + const fileContent = await NativeFile.readFile(categoryFilePath); + const parsed: unknown = JSON.parse(fileContent); + if (!Array.isArray(parsed)) { + throw new Error('Invalid backup categories'); + } + categories = parsed as BackupCategory[]; + } catch { + failedSectionCount++; + } + } + + if (categories) { + for (const [index, category] of categories.entries()) { + try { + await _restoreCategory( + { + ...category, + novelIds: category.novelIds.filter(novelId => + novelIdMap.has(novelId), + ), + }, + novelIdMap, + ); + categoryCount++; + } catch { + failedCategoryCount++; + } + progressReporter?.( + 'categories', + (index + 1) / Math.max(1, categories.length), + getString('backupScreen.restoringCategoriesProgress', { + current: index + 1, + total: categories.length, + }), + index + 1 === categories.length, + ); + } + } + progressReporter?.('categories', 1, categoriesProgressText, true); + } + benchmarkLog?.( + `restoreData:categories:done count=${categoryCount} failed=${failedCategoryCount}`, + ); + + benchmarkLog?.('restoreData:settings:start'); + const settingsProgressText = getString('backupScreen.restoringSettings'); + const settingsFilePath = cacheDirPath + '/' + BackupEntryName.SETTING; + let settingsRestored = !manifest.sections.settings; + + if (!manifest.sections.settings) { + progressReporter?.('settings', 1, settingsProgressText, true); + } else { + progressReporter?.('settings', 0, settingsProgressText, true); + if (!(await NativeFile.exists(settingsFilePath))) { + // Reported as a settings warning in the completion summary. + } else { + try { + const fileContent = await NativeFile.readFile(settingsFilePath); + const settingsData: Record = JSON.parse(fileContent); + if (INSTALLED_PLUGINS_KEY in settingsData) { + pluginsFromSettings = parsePluginList( + settingsData[INSTALLED_PLUGINS_KEY], + ); + delete settingsData[INSTALLED_PLUGINS_KEY]; + } + restoreMMKVData(settingsData); + settingsRestored = true; + } catch { + // Included in the completion warning below. + } + } + progressReporter?.('settings', 1, settingsProgressText, true); + } + benchmarkLog?.(`restoreData:settings:done restored=${settingsRestored}`); + + benchmarkLog?.('restoreData:plugins:start'); + let restoredPlugins = pluginsFromSettings; + const pluginsProgressText = getString('backupScreen.restoringPlugins'); + if (!manifest.sections.plugins) { + progressReporter?.('plugins', 1, pluginsProgressText, true); + } else { + progressReporter?.('plugins', 0, pluginsProgressText, true); + if (manifest.formatVersion === 2 || manifest.formatVersion === 3) { + const pluginMetadataPath = + cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA; + if (!(await NativeFile.exists(pluginMetadataPath))) { + failedSectionCount++; + } else { + try { + restoredPlugins = parsePluginList( + await NativeFile.readFile(pluginMetadataPath), + ); + } catch { + failedSectionCount++; + } + } + } + const mergedPlugins = [ + ...new Map( + [...installedPluginsBeforeRestore, ...restoredPlugins].map(plugin => [ + plugin.id, + plugin, + ]), + ).values(), + ]; + MMKVStorage.set(INSTALLED_PLUGINS_KEY, JSON.stringify(mergedPlugins)); + progressReporter?.('plugins', 1, pluginsProgressText, true); + } + benchmarkLog?.(`restoreData:plugins:done count=${restoredPlugins.length}`); + benchmarkLog?.('restoreData:done'); + + return { + novelCount: novelSummary.novelCount, + failedNovelCount: novelSummary.failedNovelCount, + categoryCount, + failedCategoryCount, + settingsRestored, + failedSectionCount, + pluginIds: novelSummary.pluginIds, + novelMappings: novelSummary.novelMappings, + restoreRunId, + manifest, + }; +}; + +export const clearRestoreChapterMappingsSafely = async ( + restoreRunId: string, +): Promise => { + try { + await clearRestoreChapterMappings(restoreRunId); + } catch { + // Restore mappings are run-scoped and do not affect later restores. + } +}; + +export const restoreData = async ( + cacheDirPath: string, + progressReporter?: RestoreProgressReporter, + benchmarkLog?: RestoreBenchmarkLogger, +): Promise => { + const restoreRunId = createRestoreRunId(); + try { + return await restoreDataInternal( + cacheDirPath, + progressReporter, + benchmarkLog, + restoreRunId, + ); + } catch (error) { + await clearRestoreChapterMappingsSafely(restoreRunId); + throw error; + } +}; diff --git a/src/services/backup/restore/local.ts b/src/services/backup/restore/local.ts new file mode 100644 index 0000000000..42d48a074c --- /dev/null +++ b/src/services/backup/restore/local.ts @@ -0,0 +1,106 @@ +import { CACHE_DIR_PATH, clearBackupCache } from '../cache'; +import NativeZipArchive from '@modules/native-zip-archive'; +import NativeFile from '@modules/native-file'; +import { ZipBackupName } from '../types'; +import { getString } from '@i18n/translations'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; +import { runRestore } from './run'; + +const printTime = (time?: number) => { + if (!time) return ''; + if (time >= 1000) { + return (time / 1000).toFixed(2) + 's'; + } + return time.toFixed(2) + 'ms'; +}; + +let startTime: number; +let lastTime: number | undefined; +const logRestoreBenchmark = (message: string) => { + if (!__DEV__) { + return; + } + const currentTime = performance.now(); + if (message === 'local:start') { + lastTime = undefined; + startTime = currentTime; + } + const diff = lastTime ? printTime(currentTime - lastTime) : undefined; + lastTime = currentTime; + // Benchmark output is consumed from the Metro client log. + // eslint-disable-next-line no-console + console.log( + `[restore-benchmark] ${currentTime.toFixed(2)},${ + !diff ? '' : ' duration: ' + diff.padEnd(15, ' ') + } ${message}`, + ); + if (message === 'local:finalize:done') { + // eslint-disable-next-line no-console + console.log('Total time:', printTime(currentTime - startTime)); + lastTime = undefined; + } +}; + +export const restoreBackup = async ( + { sourceUri }: { sourceUri: string }, + setMeta?: TaskProgressUpdater, +) => { + logRestoreBenchmark('local:start'); + try { + await runRestore( + { + kind: 'local', + prepare: async report => { + report?.('source', 0, getString('backupScreen.copyingBackup'), true); + await clearBackupCache(); + const localPath = `${CACHE_DIR_PATH}-source.zip`; + await NativeFile.copyFile(sourceUri, localPath); + logRestoreBenchmark('local:copy:done'); + report?.('source', 1, getString('backupScreen.copyingBackup'), true); + report?.( + 'extract', + 0, + getString('backupScreen.extractingBackup'), + true, + ); + const outerArchiveStats = await NativeZipArchive.unzip( + localPath, + CACHE_DIR_PATH, + ); + logRestoreBenchmark( + `local:outer-unzip:done ${JSON.stringify(outerArchiveStats)}`, + ); + report?.( + 'extract', + 1, + getString('backupScreen.extractingBackup'), + true, + ); + }, + loadArchive: async (name, destination) => { + const archivePath = `${CACHE_DIR_PATH}/${name}`; + if (!(await NativeFile.exists(archivePath))) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + const archiveStats = await NativeZipArchive.unzip( + archivePath, + destination, + ); + logRestoreBenchmark( + `${ + name === ZipBackupName.DOWNLOAD + ? 'local:legacy-archive' + : 'local:selected-archive' + }-unzip:done ${JSON.stringify(archiveStats)}`, + ); + }, + benchmarkLog: logRestoreBenchmark, + }, + setMeta, + ); + } catch (error) { + lastTime = undefined; + setMeta?.(meta => ({ ...meta, isRunning: false })); + throw error; + } +}; diff --git a/src/services/backup/restore/novels.ts b/src/services/backup/restore/novels.ts new file mode 100644 index 0000000000..791b5edc6f --- /dev/null +++ b/src/services/backup/restore/novels.ts @@ -0,0 +1,515 @@ +import { + _restoreNovelAndChapters, + _restoreNovelsAndChapters, + recordRestoreFallback, + type RestoreNovelMetrics, + type RestoreNovelOptions, + type RestoreNovelProgress, +} from '@database/queries/NovelRestoreQueries'; +import type { BackupNovel, RestoredNovelMapping } from '@database/types'; +import NativeFile from '@modules/native-file'; +import { getString } from '@i18n/translations'; +import { NOVEL_STORAGE, ROOT_STORAGE } from '@utils/Storages'; +import type { RestoreProgressReporter } from './progress'; +import { BackupEntryName, type ResolvedBackupManifest } from '../types'; +import { + NovelFileValidationError, + decodeAndValidateNovelFile, +} from './validation'; + +const RESTORE_NOVEL_BATCH_SIZE = 100; +const BACKUP_FILE_CONCURRENCY = 8; +const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; + +type BackupNovelFileDescriptor = { + name: string; + path: string; +}; + +type RestoreBenchmarkLogger = (message: string) => void; + +type RestoreNovelTelemetry = { + database: RestoreNovelMetrics; + collectDatabaseMetrics: boolean; + uniqueInputChapterCount: number; +}; + +export type NovelRestoreSummary = { + novelCount: number; + failedNovelCount: number; + failedSectionCount: number; + pluginIds: string[]; + novelMappings: RestoredNovelMapping[]; + novelIdMap: Map; +}; + +const restoreNovelsWithTelemetry = async ( + cacheDirPath: string, + manifest: ResolvedBackupManifest, + restoreRunId: string, + progressReporter: RestoreProgressReporter | undefined, + benchmarkLog: RestoreBenchmarkLogger | undefined, + telemetry: RestoreNovelTelemetry, +): Promise => { + const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; + const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; + const summary: NovelRestoreSummary = { + novelCount: 0, + failedNovelCount: 0, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + novelIdMap: new Map(), + }; + + benchmarkLog?.('restoreData:novels:validation:start'); + progressReporter?.( + 'novels', + 0, + getString('backupScreen.validatingNovels'), + true, + ); + if (!(await NativeFile.exists(novelDirPath))) { + summary.failedSectionCount++; + progressReporter?.( + 'novels', + 1, + getString('backupScreen.restoringNovels'), + true, + ); + return summary; + } + + let items: BackupNovelFileDescriptor[]; + try { + items = (await NativeFile.readDir(novelDirPath)) + .filter(item => !item.isDirectory) + .sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, + ) + .map(item => ({ name: item.name, path: item.path })); + } catch { + summary.failedSectionCount++; + progressReporter?.( + 'novels', + 1, + getString('backupScreen.restoringNovels'), + true, + ); + return summary; + } + + const seenNovelIds = new Set(); + const seenNovelIdentities = new Set(); + const seenChapterIdentities = new Set(); + const pluginIds = new Set(); + const pendingNovels: BackupNovel[] = []; + let filesProcessed = 0; + let inputRecordCount = 0; + let persistedNovelCount = 0; + const totalNovelCount = + manifest.formatVersion === 3 || + (manifest.formatVersion === 2 && manifest.novelDataFormat === 2) + ? manifest.novelCount + : items.length; + let completedFileWork = 0; + let pendingFileWork = 0; + let activeFileWork = 0; + let activeWorkFraction = 0; + let activeDatabaseStage: RestoreNovelProgress['stage'] | undefined; + let progressInterruptionLatched = false; + let progressInterruption: unknown; + let readMs = 0; + let parseMs = 0; + let databaseMs = 0; + let coverProcessingMs = 0; + let coverCandidates = 0; + let coverFilesFound = 0; + let coverFilesMissing = 0; + let coverFilesCopied = 0; + let coverProcessingFailures = 0; + + const getNovelCountText = () => + getString('backupScreen.restoringNovelsProgress', { + current: persistedNovelCount, + total: totalNovelCount ?? inputRecordCount, + }); + const getNovelPhaseFraction = () => { + if (items.length === 0) { + return 1; + } + return ( + (0.25 * filesProcessed + + 0.75 * (completedFileWork + activeFileWork * activeWorkFraction)) / + items.length + ); + }; + const publishNovelProgress = (force = false) => { + if (progressInterruptionLatched) { + return; + } + try { + progressReporter?.( + 'novels', + getNovelPhaseFraction(), + getNovelCountText(), + force, + ); + } catch (error) { + if (!progressInterruptionLatched) { + progressInterruptionLatched = true; + progressInterruption = error; + } + throw error; + } + }; + const processFile = async (item: BackupNovelFileDescriptor) => { + const readStartedAt = performance.now(); + let fileContent: string | undefined; + try { + fileContent = await NativeFile.readFile(item.path); + } catch { + summary.failedNovelCount++; + } finally { + readMs += performance.now() - readStartedAt; + } + + if (fileContent === undefined) { + inputRecordCount++; + completedFileWork++; + } else { + const parseStartedAt = performance.now(); + try { + const novels = decodeAndValidateNovelFile( + fileContent, + manifest, + seenNovelIds, + seenNovelIdentities, + seenChapterIdentities, + ); + inputRecordCount += novels.length; + if (novels.length === 0) { + completedFileWork++; + } else { + pendingFileWork++; + } + for (const novel of novels) { + pluginIds.add(novel.pluginId); + pendingNovels.push(novel); + } + if (telemetry.collectDatabaseMetrics) { + telemetry.uniqueInputChapterCount = seenChapterIdentities.size; + } + } catch (error) { + const rejectedRecordCount = + error instanceof NovelFileValidationError ? error.recordCount : 1; + summary.failedNovelCount += rejectedRecordCount; + inputRecordCount += rejectedRecordCount; + completedFileWork++; + } finally { + parseMs += performance.now() - parseStartedAt; + } + } + + filesProcessed++; + publishNovelProgress(); + }; + + publishNovelProgress(true); + + const restoreNovelBatch = async () => { + if (pendingNovels.length === 0) { + return; + } + const batch = pendingNovels.splice(0, pendingNovels.length); + const fileWorkForBatch = pendingFileWork; + pendingFileWork = 0; + activeFileWork = fileWorkForBatch; + activeWorkFraction = 0; + activeDatabaseStage = undefined; + + const reportDatabaseCheckpoint = ( + progress: RestoreNovelProgress, + fallbackIndex?: number, + ) => { + if (progressInterruptionLatched) { + return; + } + try { + const stageFraction = + progress.total > 0 + ? Math.max(0, Math.min(1, progress.completed / progress.total)) + : 1; + let databaseFraction: number; + if (progress.stage === 'novels') { + databaseFraction = 0.1 * stageFraction; + } else if (progress.stage === 'chapters') { + databaseFraction = 0.1 + 0.8 * stageFraction; + } else { + databaseFraction = 0.9 + 0.1 * stageFraction; + } + if (fallbackIndex !== undefined) { + databaseFraction = (fallbackIndex + databaseFraction) / batch.length; + } + activeWorkFraction = Math.max( + activeWorkFraction, + 0.8 * databaseFraction, + ); + const stageChanged = activeDatabaseStage !== progress.stage; + activeDatabaseStage = progress.stage; + const stageCompleted = + progress.total <= 0 || progress.completed >= progress.total; + publishNovelProgress(stageChanged || stageCompleted); + } catch (error) { + if (!progressInterruptionLatched) { + progressInterruptionLatched = true; + progressInterruption = error; + } + } + }; + const onBatchProgress = (progress: RestoreNovelProgress) => + reportDatabaseCheckpoint(progress); + const restoreOptions: RestoreNovelOptions = { + includeChapterMappings: manifest.sections.downloadedFiles, + ...(manifest.sections.downloadedFiles ? { restoreRunId } : {}), + ...(progressReporter ? { onProgress: onBatchProgress } : {}), + }; + let restoredNovels: { + backupNovel: BackupNovel; + mapping: RestoredNovelMapping; + }[] = []; + const databaseStartedAt = performance.now(); + try { + try { + const mappings = telemetry.collectDatabaseMetrics + ? await _restoreNovelsAndChapters( + batch, + restoreOptions, + telemetry.database, + ) + : await _restoreNovelsAndChapters(batch, restoreOptions); + if (mappings.length !== batch.length) { + throw new Error('Restore returned incomplete novel mappings'); + } + restoredNovels = batch.map((backupNovel, index) => ({ + backupNovel, + mapping: mappings[index], + })); + persistedNovelCount += batch.length; + } catch (databaseError) { + if (progressInterruptionLatched) { + throw progressInterruption; + } + if (telemetry.collectDatabaseMetrics) { + telemetry.database.novelBatchFallbacks++; + recordRestoreFallback(telemetry.database, databaseError); + } + for (const [index, backupNovel] of batch.entries()) { + if (telemetry.collectDatabaseMetrics) { + telemetry.database.novelFallbackRowsAttempted++; + } + const fallbackOptions: RestoreNovelOptions = progressReporter + ? { + ...restoreOptions, + onProgress: progress => + reportDatabaseCheckpoint(progress, index), + } + : restoreOptions; + let mapping: RestoredNovelMapping; + try { + mapping = telemetry.collectDatabaseMetrics + ? await _restoreNovelAndChapters( + backupNovel, + fallbackOptions, + telemetry.database, + ) + : await _restoreNovelAndChapters(backupNovel, fallbackOptions); + } catch (fallbackError) { + if (progressInterruptionLatched) { + throw progressInterruption; + } + if (telemetry.collectDatabaseMetrics) { + recordRestoreFallback(telemetry.database, fallbackError); + } + summary.failedNovelCount++; + continue; + } + restoredNovels.push({ backupNovel, mapping }); + persistedNovelCount++; + if (progressInterruptionLatched) { + throw progressInterruption; + } + publishNovelProgress(true); + } + } + } finally { + databaseMs += performance.now() - databaseStartedAt; + } + + if (progressInterruptionLatched) { + throw progressInterruption; + } + activeWorkFraction = 0.8; + activeDatabaseStage = undefined; + publishNovelProgress(true); + + const shouldRestoreCover = (backupNovel: BackupNovel) => + !manifest.sections.downloadedFiles && + backupNovel.cover?.startsWith(APP_STORAGE_URI); + const hasCoverWork = restoredNovels.some(({ backupNovel }) => + shouldRestoreCover(backupNovel), + ); + const coverStartedAt = performance.now(); + if (hasCoverWork) { + let attemptedCoverRecords = 0; + for ( + let start = 0; + start < restoredNovels.length; + start += BACKUP_FILE_CONCURRENCY + ) { + const coverBatch = restoredNovels.slice( + start, + start + BACKUP_FILE_CONCURRENCY, + ); + await Promise.all( + coverBatch.map(async ({ backupNovel, mapping: novelMapping }) => { + try { + if (shouldRestoreCover(backupNovel)) { + coverCandidates++; + const coverBackupPath = coversDirPath + '/' + backupNovel.id; + if (await NativeFile.exists(coverBackupPath)) { + coverFilesFound++; + const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; + await NativeFile.mkdir( + coverPath.slice(0, Math.max(0, coverPath.lastIndexOf('/'))), + ); + await NativeFile.copyFile(coverBackupPath, coverPath); + coverFilesCopied++; + } else { + coverFilesMissing++; + } + } + } catch { + coverProcessingFailures++; + summary.failedNovelCount++; + } + }), + ); + attemptedCoverRecords += coverBatch.length; + activeWorkFraction = + 0.8 + (0.2 * attemptedCoverRecords) / restoredNovels.length; + publishNovelProgress(attemptedCoverRecords === restoredNovels.length); + } + } + coverProcessingMs += performance.now() - coverStartedAt; + completedFileWork += activeFileWork; + activeFileWork = 0; + activeWorkFraction = 0; + publishNovelProgress(true); + + for (const { backupNovel, mapping: novelMapping } of restoredNovels) { + summary.novelMappings.push(novelMapping); + summary.novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); + summary.novelCount++; + } + }; + + let nextFileIndex = 0; + while (nextFileIndex < items.length || pendingNovels.length > 0) { + if (pendingNovels.length >= RESTORE_NOVEL_BATCH_SIZE) { + const writePromise = restoreNovelBatch(); + if (nextFileIndex < items.length) { + const filePromise = processFile(items[nextFileIndex++]); + const [writeResult, fileResult] = await Promise.allSettled([ + writePromise, + filePromise, + ]); + if (fileResult.status === 'rejected') { + throw fileResult.reason; + } + if (writeResult.status === 'rejected') { + throw writeResult.reason; + } + } else { + await writePromise; + } + continue; + } + + if (nextFileIndex < items.length) { + await processFile(items[nextFileIndex++]); + continue; + } + + await restoreNovelBatch(); + } + publishNovelProgress(true); + + benchmarkLog?.(`restoreData:novels:pipeline:done total=${filesProcessed}`); + benchmarkLog?.( + `restoreData:novels:done count=${summary.novelCount} failed=${ + summary.failedNovelCount + } readMs=${readMs.toFixed(1)} parseMs=${parseMs.toFixed( + 1, + )} databaseMs=${databaseMs.toFixed( + 1, + )} coverProcessingMs=${coverProcessingMs.toFixed( + 1, + )} coverCandidates=${coverCandidates} coverFilesFound=${coverFilesFound} coverFilesMissing=${coverFilesMissing} coverFilesCopied=${coverFilesCopied} coverProcessingFailures=${coverProcessingFailures}`, + ); + + summary.pluginIds = [...pluginIds]; + return summary; +}; + +export const restoreNovels = async ( + cacheDirPath: string, + manifest: ResolvedBackupManifest, + restoreRunId: string, + progressReporter?: RestoreProgressReporter, + benchmarkLog?: RestoreBenchmarkLogger, +): Promise => { + const telemetry: RestoreNovelTelemetry = { + collectDatabaseMetrics: __DEV__ && benchmarkLog !== undefined, + database: { + novelUpsertCalls: 0, + novelUpsertRowsAttempted: 0, + novelUpsertMs: 0, + novelIdentityLookupCalls: 0, + novelIdentityLookupRowsReturned: 0, + novelIdentityLookupMs: 0, + chapterWriteChunkCalls: 0, + chapterWriteRowsAttempted: 0, + chapterWriteMs: 0, + chapterMappingRowsAttempted: 0, + statsRefreshCalls: 0, + statsRefreshNovels: 0, + statsRefreshMs: 0, + novelBatchFallbacks: 0, + novelFallbackRowsAttempted: 0, + chapterBatchFallbacks: 0, + chapterFallbackRowsAttempted: 0, + fallbackCauses: [], + }, + uniqueInputChapterCount: 0, + }; + try { + return await restoreNovelsWithTelemetry( + cacheDirPath, + manifest, + restoreRunId, + progressReporter, + benchmarkLog, + telemetry, + ); + } finally { + if (telemetry.collectDatabaseMetrics) { + benchmarkLog?.( + `restoreData:database:summary ${JSON.stringify({ + uniqueInputChapterCount: telemetry.uniqueInputChapterCount, + ...telemetry.database, + })}`, + ); + } + } +}; diff --git a/src/services/backup/restore/progress.ts b/src/services/backup/restore/progress.ts new file mode 100644 index 0000000000..9ab0bd8b24 --- /dev/null +++ b/src/services/backup/restore/progress.ts @@ -0,0 +1,109 @@ +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; + +export type RestorePhase = + | 'source' + | 'extract' + | 'manifest' + | 'novels' + | 'categories' + | 'settings' + | 'plugins' + | 'selectedFiles' + | 'finalize'; + +export type RestoreProgressReporter = ( + phase: RestorePhase, + fraction: number, + progressText: string, + force?: boolean, +) => void; + +const PROGRESS_UPDATE_INTERVAL_MS = 250; + +const LOCAL_PHASE_WEIGHTS: Record = { + source: 0.09, + extract: 0.21, + manifest: 0.002, + novels: 0.68, + categories: 0.004, + settings: 0.002, + plugins: 0.004, + selectedFiles: 0.005, + finalize: 0.003, +}; + +const PHASES = Object.keys(LOCAL_PHASE_WEIGHTS) as RestorePhase[]; + +export const createRestoreProgressReporter = ( + setMeta: TaskProgressUpdater | undefined, + source: 'local' | 'remote', +): RestoreProgressReporter | undefined => { + if (!setMeta) { + return undefined; + } + + const weights = { ...LOCAL_PHASE_WEIGHTS }; + if (source === 'remote') { + weights.source = 0.3; + weights.extract = 0; + } + + const phaseStarts = {} as Record; + let nextStart = 0; + for (const phase of PHASES) { + phaseStarts[phase] = nextStart; + nextStart += weights[phase]; + } + + let highWaterMark = 0; + let currentPhase: RestorePhase | undefined; + let lastEmittedProgress: number | undefined; + let lastEmittedText: string | undefined; + let lastEmittedAt = Number.NEGATIVE_INFINITY; + + return (phase, fraction, progressText, force = false) => { + const normalizedFraction = Number.isNaN(fraction) + ? 0 + : Math.max(0, Math.min(1, fraction)); + const mappedProgress = Math.max( + 0, + Math.min( + phase === 'finalize' && normalizedFraction === 1 + ? 1 + : 1 - Number.EPSILON, + phaseStarts[phase] + weights[phase] * normalizedFraction, + ), + ); + highWaterMark = Math.max(highWaterMark, mappedProgress); + + const phaseChanged = currentPhase !== phase; + currentPhase = phase; + + if ( + lastEmittedProgress === highWaterMark && + lastEmittedText === progressText + ) { + return; + } + + const now = Date.now(); + const completed = normalizedFraction === 1; + if ( + !phaseChanged && + !force && + !completed && + now - lastEmittedAt < PROGRESS_UPDATE_INTERVAL_MS + ) { + return; + } + + lastEmittedProgress = highWaterMark; + lastEmittedText = progressText; + lastEmittedAt = now; + setMeta(meta => ({ + ...meta, + progress: highWaterMark, + progressText, + })); + }; +}; diff --git a/src/services/backup/restoreResult.ts b/src/services/backup/restore/result.ts similarity index 97% rename from src/services/backup/restoreResult.ts rename to src/services/backup/restore/result.ts index dc11bd9880..24327ff913 100644 --- a/src/services/backup/restoreResult.ts +++ b/src/services/backup/restore/result.ts @@ -5,8 +5,8 @@ import { reloadInstalledPlugins, } from '@plugins/pluginManager'; import { PLUGIN_STORAGE } from '@utils/Storages'; -import type { ResolvedBackupManifest } from './types'; import type { RestoredNovelMapping } from '@database/types'; +import type { ResolvedBackupManifest } from '../types'; export type RestoreResult = { novelCount: number; @@ -17,6 +17,7 @@ export type RestoreResult = { failedSectionCount: number; pluginIds: string[]; novelMappings: RestoredNovelMapping[]; + restoreRunId: string; manifest: ResolvedBackupManifest; }; diff --git a/src/services/backup/restore/run.ts b/src/services/backup/restore/run.ts new file mode 100644 index 0000000000..b199880fdc --- /dev/null +++ b/src/services/backup/restore/run.ts @@ -0,0 +1,146 @@ +import { getString } from '@i18n/translations'; +import NativeFile from '@modules/native-file'; +import { BackupEntryName, ZipBackupName } from '../types'; +import { getSelectedBackupFileSections } from '../fileSections'; +import { CACHE_DIR_PATH } from '../cache'; +import { + getLegacyFilesRestorePath, + getNovelFilesRestorePath, + restoreLegacyFiles, + restoreNovelFiles, +} from './files'; +import { restoreData, clearRestoreChapterMappingsSafely } from './index'; +import { + finalizeRestoredPlugins, + getRestoreCompletionText, + type RestoreResult, +} from './result'; +import { + createRestoreProgressReporter, + type RestoreProgressReporter, +} from './progress'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; + +type RestoreSource = { + kind: 'local' | 'remote'; + prepare: (report: RestoreProgressReporter | undefined) => Promise; + loadArchive: (name: ZipBackupName, destination: string) => Promise; + benchmarkLog?: (message: string) => void; +}; + +export const runRestore = async ( + source: RestoreSource, + setMeta?: TaskProgressUpdater, +): Promise => { + const progressReporter = createRestoreProgressReporter(setMeta, source.kind); + let restoreResult: RestoreResult | undefined; + + setMeta?.(meta => ({ ...meta, isRunning: true })); + + try { + await source.prepare(progressReporter); + restoreResult = await restoreData( + CACHE_DIR_PATH, + progressReporter, + source.benchmarkLog, + ); + source.benchmarkLog?.('local:restore-data:done'); + + const selectedFilesText = getString('backupScreen.restoringSelectedFiles'); + const publishSelectedFilesProgress = (fraction: number, force = false) => { + progressReporter?.('selectedFiles', fraction, selectedFilesText, force); + }; + const reportSelectedFileMoves = (completed: number, total: number) => { + publishSelectedFilesProgress( + 0.5 + 0.5 * (total > 0 ? completed / total : 1), + completed >= total, + ); + }; + publishSelectedFilesProgress(0, true); + + if (restoreResult.manifest.formatVersion === 1) { + const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); + await source.loadArchive(ZipBackupName.DOWNLOAD, legacyFilesRestorePath); + publishSelectedFilesProgress(0.5, true); + await restoreLegacyFiles( + legacyFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + reportSelectedFileMoves, + ); + } else { + const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); + const sections = getSelectedBackupFileSections( + restoreResult.manifest.sections, + source.kind === 'local' ? restoreResult.manifest.formatVersion : 2, + ); + if (sections.length === 0) { + publishSelectedFilesProgress(0.5, true); + } + + let novelFilesSource: string | undefined; + if ( + source.kind === 'local' && + restoreResult.manifest.formatVersion === 3 && + restoreResult.manifest.sections.downloadedFiles + ) { + novelFilesSource = `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}`; + if (!(await NativeFile.exists(novelFilesSource))) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + } + + for (const [index, section] of sections.entries()) { + const destination = + section.archiveName === ZipBackupName.NOVEL_FILES + ? novelFilesRestorePath + : section.storagePath; + await source.loadArchive(section.archiveName, destination); + publishSelectedFilesProgress( + 0.5 * ((index + 1) / sections.length), + index + 1 === sections.length, + ); + } + source.benchmarkLog?.('local:selected-archives:done'); + + if (restoreResult.manifest.sections.downloadedFiles) { + await restoreNovelFiles( + novelFilesSource ?? novelFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + reportSelectedFileMoves, + ); + } else { + publishSelectedFilesProgress(1, true); + } + } + + source.benchmarkLog?.('local:downloaded-files:done'); + source.benchmarkLog?.('local:selected-files:done'); + + progressReporter?.( + 'finalize', + 0, + getString('backupScreen.finalizingRestore'), + true, + ); + const missingPluginIds = await finalizeRestoredPlugins(restoreResult); + const completionText = getRestoreCompletionText( + restoreResult, + missingPluginIds, + ); + progressReporter?.('finalize', 1, completionText, true); + source.benchmarkLog?.('local:finalize:done'); + + setMeta?.(meta => ({ + ...meta, + isRunning: false, + progressText: completionText, + completionText, + })); + } finally { + if (restoreResult) { + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); + } + } +}; diff --git a/src/services/backup/restore/selfhost.ts b/src/services/backup/restore/selfhost.ts new file mode 100644 index 0000000000..8dd98657e0 --- /dev/null +++ b/src/services/backup/restore/selfhost.ts @@ -0,0 +1,32 @@ +import { download } from '@api/remote'; +import { getString } from '@i18n/translations'; +import { CACHE_DIR_PATH, clearBackupCache } from '../cache'; +import { ZipBackupName } from '../types'; +import type { + SelfHostData, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { runRestore } from './run'; + +export const selfHostRestore = async ( + { host, backupFolder }: SelfHostData, + setMeta: TaskProgressUpdater, +) => { + await runRestore( + { + kind: 'remote', + prepare: async report => { + const progressText = getString('backupScreen.downloadingData'); + report?.('source', 0, progressText, true); + await clearBackupCache(); + await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); + report?.('source', 1, progressText, true); + report?.('extract', 1, progressText, true); + }, + loadArchive: async (name, destination) => { + await download(host, backupFolder, name, destination); + }, + }, + setMeta, + ); +}; diff --git a/src/services/backup/restore/validation.ts b/src/services/backup/restore/validation.ts new file mode 100644 index 0000000000..ba35e3f2ad --- /dev/null +++ b/src/services/backup/restore/validation.ts @@ -0,0 +1,127 @@ +import type { BackupNovel } from '@database/types'; +import { ROOT_STORAGE } from '@utils/Storages'; +import { decodeNovelBatch, normalizeLegacyNovel } from '../novelPayload'; +import type { ResolvedBackupManifest } from '../types'; + +const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; + +export class NovelFileValidationError extends Error { + recordCount: number; + + constructor(recordCount: number, cause: unknown) { + super(cause instanceof Error ? cause.message : 'Invalid novel file'); + this.name = 'NovelFileValidationError'; + this.recordCount = recordCount; + } +} + +export const decodeRestoreNovelFile = ( + payload: unknown, + manifest: ResolvedBackupManifest, +) => { + const novels = + manifest.novelDataFormat === 2 + ? decodeNovelBatch(payload) + : [normalizeLegacyNovel(payload)]; + return novels.map(novel => { + const { + chaptersDownloaded: _chaptersDownloaded, + chaptersUnread: _chaptersUnread, + totalChapters: _totalChapters, + lastReadAt: _lastReadAt, + lastUpdatedAt: _lastUpdatedAt, + ...normalized + } = novel as BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; + }; + return normalized.cover && !normalized.cover.startsWith('http') + ? { ...normalized, cover: APP_STORAGE_URI + normalized.cover } + : normalized; + }); +}; + +export const getNovelFileRecordCount = ( + payload: unknown, + manifest: ResolvedBackupManifest, +) => { + if (manifest.novelDataFormat !== 2) { + return 1; + } + return Array.isArray(payload) ? Math.max(1, payload.length) : 1; +}; + +export const validateNovelFileRecords = ( + novels: BackupNovel[], + seenNovelIds: Set, + seenNovelIdentities: Set, + seenChapterIdentities: Set, +) => { + const fileNovelIds = new Set(); + const fileNovelIdentities = new Set(); + const fileChapterIdentities = new Set(); + for (const novel of novels) { + const novelIdentity = `${novel.pluginId}\u0000${novel.path}`; + if ( + fileNovelIds.has(novel.id) || + seenNovelIds.has(novel.id) || + fileNovelIdentities.has(novelIdentity) || + seenNovelIdentities.has(novelIdentity) + ) { + throw new Error('Duplicate backup novel identity'); + } + fileNovelIds.add(novel.id); + fileNovelIdentities.add(novelIdentity); + for (const chapter of novel.chapters) { + const chapterIdentity = `${novel.id}\u0000${chapter.path}`; + if ( + fileChapterIdentities.has(chapterIdentity) || + seenChapterIdentities.has(chapterIdentity) + ) { + throw new Error('Duplicate backup chapter identity'); + } + fileChapterIdentities.add(chapterIdentity); + } + } + for (const novelId of fileNovelIds) { + seenNovelIds.add(novelId); + } + for (const novelIdentity of fileNovelIdentities) { + seenNovelIdentities.add(novelIdentity); + } + for (const chapterIdentity of fileChapterIdentities) { + seenChapterIdentities.add(chapterIdentity); + } +}; + +export const decodeAndValidateNovelFile = ( + fileContent: string, + manifest: ResolvedBackupManifest, + seenNovelIds: Set, + seenNovelIdentities: Set, + seenChapterIdentities: Set, +) => { + let payload: unknown; + try { + payload = JSON.parse(fileContent); + } catch (error) { + throw new NovelFileValidationError(1, error); + } + + const recordCount = getNovelFileRecordCount(payload, manifest); + try { + const novels = decodeRestoreNovelFile(payload, manifest); + validateNovelFileRecords( + novels, + seenNovelIds, + seenNovelIdentities, + seenChapterIdentities, + ); + return novels; + } catch (error) { + throw new NovelFileValidationError(recordCount, error); + } +}; diff --git a/src/services/backup/selfhost/index.ts b/src/services/backup/selfhost/index.ts index b25a8194ec..8e7662db52 100644 --- a/src/services/backup/selfhost/index.ts +++ b/src/services/backup/selfhost/index.ts @@ -1,29 +1,15 @@ import { sleep } from '@utils/sleep'; -import { download, upload } from '@api/remote'; +import { upload } from '@api/remote'; import { getString } from '@i18n/translations'; -import { - CACHE_DIR_PATH, - clearBackupCache, - prepareBackupData, - restoreData, -} from '../utils'; -import { - finalizeRestoredPlugins, - getRestoreCompletionText, -} from '../restoreResult'; +import { CACHE_DIR_PATH } from '../cache'; +import { prepareBackupData } from '../create'; import { getBackupCompletionText } from '../backupResult'; import { ZipBackupName } from '../types'; import type { SelfHostData, TaskProgressUpdater, } from '@services/backgroundTasks/contracts'; -import { - getLegacyFilesRestorePath, - getNovelFilesRestorePath, - getSelectedBackupFileSections, - restoreLegacyFiles, - restoreNovelFiles, -} from '../fileSections'; +import { getSelectedBackupFileSections } from '../fileSections'; import { resolveBackupOptions } from '../options'; export const createSelfHostBackup = async ( @@ -58,7 +44,7 @@ export const createSelfHostBackup = async ( await sleep(200); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await upload(host, backupFolder, section.archiveName, section.storagePath); } @@ -71,83 +57,3 @@ export const createSelfHostBackup = async ( completionText, })); }; - -export const selfHostRestore = async ( - { host, backupFolder }: SelfHostData, - setMeta: TaskProgressUpdater, -) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - await clearBackupCache(); - await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download( - host, - backupFolder, - ZipBackupName.DOWNLOAD, - legacyFilesRestorePath, - ); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - await download( - host, - backupFolder, - section.archiveName, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, - restoreResult.novelMappings, - ); - } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); - - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); -}; diff --git a/src/services/backup/types.ts b/src/services/backup/types.ts index aefd5cf65f..35ce7a649d 100644 --- a/src/services/backup/types.ts +++ b/src/services/backup/types.ts @@ -14,11 +14,14 @@ export enum BackupEntryName { PLUGIN_METADATA = 'Plugins.json', COVERS = 'Covers', NOVEL_AND_CHAPTERS = 'NovelAndChapters', + NOVEL_FILES = 'NovelFiles', } export type BackupManifest = { appVersion: string; - formatVersion: 2; + formatVersion: 2 | 3; + novelDataFormat?: 1 | 2; + novelCount?: number; sections: BackupOptions; }; @@ -27,5 +30,6 @@ export type ResolvedBackupManifest = | { appVersion?: string; formatVersion: 1; + novelDataFormat?: 1 | 2; sections: BackupOptions; }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts deleted file mode 100644 index d7b186ef80..0000000000 --- a/src/services/backup/utils.ts +++ /dev/null @@ -1,453 +0,0 @@ -import { SELF_HOST_BACKUP } from '@hooks/persisted/useSelfHost'; -import { OLD_TRACKED_NOVEL_PREFIX } from '@hooks/persisted/migrations/trackerMigration'; -import { LAST_UPDATE_TIME } from '@hooks/persisted/useUpdates'; -import { MMKVStorage } from '@utils/mmkv/mmkv'; -import { version } from '../../../package.json'; -import { - _restoreNovelAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; -import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; -import { - _restoreCategory, - getAllNovelCategories, - getCategoriesFromDb, -} from '@database/queries/CategoryQueries'; -import { - BackupCategory, - BackupNovel, - type RestoredNovelMapping, -} from '@database/types'; -import { - BackupEntryName, - type BackupManifest, - type ResolvedBackupManifest, -} from './types'; -import { NOVEL_STORAGE, ROOT_STORAGE } from '@utils/Storages'; -import { BACKGROUND_TASKS_STORE_KEY } from '@services/backgroundTasks/constants'; -import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; -import NativeFile from '@modules/native-file'; -import { getString } from '@i18n/translations'; -import type { RestoreResult } from './restoreResult'; -import type { BackupResult } from './backupResult'; -import { - DEFAULT_BACKUP_OPTIONS, - resolveBackupOptions, - type BackupOptions, -} from './options'; -import { INSTALLED_PLUGINS_KEY } from '@plugins/pluginManager'; -import type { PluginItem } from '@plugins/types'; - -const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; - -const stripUriSuffix = (uri: string) => uri.split(/[?#]/, 1)[0]; - -const parentDirectory = (path: string) => - path.slice(0, Math.max(0, path.lastIndexOf('/'))); - -const parsePluginList = (value: unknown): PluginItem[] => { - const parsed = typeof value === 'string' ? JSON.parse(value) : value; - if (!Array.isArray(parsed)) { - throw new Error('Invalid installed plugin registry'); - } - return parsed as PluginItem[]; -}; - -export const CACHE_DIR_PATH = - NativeFile.ExternalCachesDirectoryPath + '/BackupData'; - -export const clearBackupCache = async (cacheDirPath = CACHE_DIR_PATH) => { - if (await NativeFile.exists(cacheDirPath)) { - await NativeFile.unlink(cacheDirPath); - } -}; - -const backupMMKVData = () => { - const excludeKeys = [ - BACKGROUND_TASKS_STORE_KEY, - OLD_TRACKED_NOVEL_PREFIX, - SELF_HOST_BACKUP, - LAST_UPDATE_TIME, - INSTALLED_PLUGINS_KEY, - ]; - const keys = MMKVStorage.getAllKeys().filter( - key => !excludeKeys.includes(key), - ); - const data = {} as any; - for (const key of keys) { - let value: number | string | boolean | undefined = - MMKVStorage.getString(key); - if (!value) { - value = MMKVStorage.getBoolean(key); - } - if (key && value) { - data[key] = value; - } - } - return data; -}; - -const restoreMMKVData = (data: any) => { - for (const key in data) { - MMKVStorage.set(key, data[key]); - } -}; - -export const prepareBackupData = async ( - cacheDirPath: string, - requestedOptions?: BackupOptions, -): Promise => { - const options = resolveBackupOptions(requestedOptions); - const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; - const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; - let failedNovelCount = 0; - let failedSectionCount = 0; - - await clearBackupCache(cacheDirPath); - await NativeFile.mkdir(cacheDirPath); - - // version - const manifest: BackupManifest = { - appVersion: version, - formatVersion: 2, - sections: options, - }; - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.VERSION, - JSON.stringify(manifest), - ); - - // novels - if (options.library) { - await NativeFile.mkdir(novelDirPath); - await NativeFile.mkdir(coversDirPath); - await getAllNovels().then(async novels => { - for (const novel of novels) { - try { - const chapters = await getAllNovelChaptersForBackup(novel.id); - const backedUpChapters = options.downloadedFiles - ? chapters - : chapters.map(chapter => ({ - ...chapter, - isDownloaded: false, - })); - let cover = novel.cover; - if (cover?.startsWith(APP_STORAGE_URI)) { - try { - await NativeFile.copyFile( - stripUriSuffix(cover), - coversDirPath + '/' + novel.id, - ); - cover = cover.replace(APP_STORAGE_URI, ''); - } catch { - cover = options.downloadedFiles - ? cover.replace(APP_STORAGE_URI, '') - : null; - } - } - await NativeFile.writeFile( - novelDirPath + '/' + novel.id + '.json', - JSON.stringify({ - chapters: backedUpChapters, - ...novel, - cover, - }), - ); - } catch { - failedNovelCount++; - } - } - }); - - // categories - try { - const categories = await getCategoriesFromDb(); - const novelCategories = await getAllNovelCategories(); - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.CATEGORY, - JSON.stringify( - categories.map(category => { - return { - ...category, - novelIds: novelCategories - .filter(nc => nc.categoryId === category.id) - .map(nc => nc.novelId), - }; - }), - ), - ); - } catch { - failedSectionCount++; - } - } - - // settings - if (options.settings) { - try { - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.SETTING, - JSON.stringify(backupMMKVData()), - ); - } catch { - failedSectionCount++; - } - } - - // installed plugin registry - if (options.plugins) { - try { - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA, - MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', - ); - } catch { - failedSectionCount++; - } - } - - return { - failedNovelCount, - failedSectionCount, - }; -}; - -const getBackupManifest = async ( - cacheDirPath: string, -): Promise => { - try { - const fileContent = await NativeFile.readFile( - cacheDirPath + '/' + BackupEntryName.VERSION, - ); - const data = JSON.parse(fileContent) as Partial & { - version?: string; - }; - if (data.formatVersion === 2 && data.sections) { - return { - appVersion: data.appVersion ?? data.version ?? '', - formatVersion: 2, - sections: resolveBackupOptions(data.sections), - }; - } - - return { - appVersion: data.version, - formatVersion: 1, - sections: DEFAULT_BACKUP_OPTIONS, - }; - } catch { - return { - formatVersion: 1, - sections: DEFAULT_BACKUP_OPTIONS, - }; - } -}; - -const updateRestoreProgress = ( - setMeta: TaskProgressUpdater | undefined, - progressText: string, -) => { - setMeta?.(meta => ({ - ...meta, - progressText, - })); -}; - -export const restoreData = async ( - cacheDirPath: string, - setMeta?: TaskProgressUpdater, -): Promise => { - const manifest = await getBackupManifest(cacheDirPath); - const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; - const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; - const pluginIds = new Set(); - const novelIdMap = new Map(); - const novelMappings: RestoredNovelMapping[] = []; - const installedPluginsBeforeRestore = (() => { - try { - return parsePluginList( - MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', - ); - } catch { - return []; - } - })(); - let pluginsFromSettings: PluginItem[] = []; - - // version - // nothing to do - - // novels - if (manifest.sections.library) { - updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); - } - let novelCount = 0; - let failedCount = 0; - let failedSectionCount = 0; - - if (!manifest.sections.library) { - // Intentionally omitted from this backup. - } else if (!(await NativeFile.exists(novelDirPath))) { - failedSectionCount++; - } else { - try { - const items = (await NativeFile.readDir(novelDirPath)).filter( - item => !item.isDirectory, - ); - for (const [index, item] of items.entries()) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringNovelsProgress', { - current: index + 1, - total: items.length, - }), - ); - try { - const fileContent = await NativeFile.readFile(item.path); - const backupNovel = JSON.parse(fileContent) as BackupNovel; - pluginIds.add(backupNovel.pluginId); - - const hasStoredCover = - backupNovel.cover && !backupNovel.cover.startsWith('http'); - if (hasStoredCover) { - backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; - } - - const novelMapping = await _restoreNovelAndChapters(backupNovel); - novelMappings.push(novelMapping); - novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); - - if (hasStoredCover) { - const coverBackupPath = coversDirPath + '/' + backupNovel.id; - if (await NativeFile.exists(coverBackupPath)) { - const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; - await NativeFile.mkdir(parentDirectory(coverPath)); - await NativeFile.copyFile(coverBackupPath, coverPath); - } - } - novelCount++; - } catch { - failedCount++; - } - } - } catch { - failedSectionCount++; - } - } - - // categories - if (manifest.sections.library) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringCategories'), - ); - } - const categoryFilePath = cacheDirPath + '/' + BackupEntryName.CATEGORY; - let categoryCount = 0; - let failedCategoryCount = 0; - - if (!manifest.sections.library) { - // Intentionally omitted from this backup. - } else if (!(await NativeFile.exists(categoryFilePath))) { - failedSectionCount++; - } else { - try { - const fileContent = await NativeFile.readFile(categoryFilePath); - const categories: BackupCategory[] = JSON.parse(fileContent); - - for (const [index, category] of categories.entries()) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringCategoriesProgress', { - current: index + 1, - total: categories.length, - }), - ); - try { - await _restoreCategory( - { - ...category, - novelIds: category.novelIds.filter(novelId => - novelIdMap.has(novelId), - ), - }, - novelIdMap, - ); - categoryCount++; - } catch { - failedCategoryCount++; - } - } - } catch { - failedSectionCount++; - } - } - - // settings - if (manifest.sections.settings) { - updateRestoreProgress(setMeta, getString('backupScreen.restoringSettings')); - } - const settingsFilePath = cacheDirPath + '/' + BackupEntryName.SETTING; - let settingsRestored = !manifest.sections.settings; - - if (!manifest.sections.settings) { - // Intentionally omitted from this backup. - } else if (!(await NativeFile.exists(settingsFilePath))) { - // Reported as a settings warning in the completion summary. - } else { - try { - const fileContent = await NativeFile.readFile(settingsFilePath); - const settingsData = JSON.parse(fileContent); - if (INSTALLED_PLUGINS_KEY in settingsData) { - pluginsFromSettings = parsePluginList( - settingsData[INSTALLED_PLUGINS_KEY], - ); - delete settingsData[INSTALLED_PLUGINS_KEY]; - } - restoreMMKVData(settingsData); - settingsRestored = true; - } catch { - // Included in the completion warning below. - } - } - - // installed plugin registry - if (manifest.sections.plugins) { - let restoredPlugins = pluginsFromSettings; - if (manifest.formatVersion === 2) { - const pluginMetadataPath = - cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA; - if (!(await NativeFile.exists(pluginMetadataPath))) { - failedSectionCount++; - } else { - try { - restoredPlugins = parsePluginList( - await NativeFile.readFile(pluginMetadataPath), - ); - } catch { - failedSectionCount++; - } - } - } - const mergedPlugins = [ - ...new Map( - [...installedPluginsBeforeRestore, ...restoredPlugins].map(plugin => [ - plugin.id, - plugin, - ]), - ).values(), - ]; - MMKVStorage.set(INSTALLED_PLUGINS_KEY, JSON.stringify(mergedPlugins)); - } - - return { - novelCount, - failedNovelCount: failedCount, - categoryCount, - failedCategoryCount, - settingsRestored, - failedSectionCount, - pluginIds: [...pluginIds], - novelMappings, - manifest, - }; -}; diff --git a/src/services/updates/__tests__/index.test.ts b/src/services/updates/__tests__/index.test.ts index 2286bb738a..6e85407aea 100644 --- a/src/services/updates/__tests__/index.test.ts +++ b/src/services/updates/__tests__/index.test.ts @@ -155,4 +155,131 @@ describe('updateLibrary', () => { }, ); }); + it('chunks each download task immediately without changing its identity', async () => { + mockedGetMMKVObject.mockReturnValue({ downloadNewChapters: true }); + const novelId = 7; + const chapters = Array.from({ length: 101 }, (_, index) => ({ + chapterId: index + 1, + chapterName: `Chapter ${index + 1}`, + ...(index === 0 ? {} : { novelId }), + })); + const downloadTask = { + name: 'DOWNLOAD_CHAPTER' as const, + data: { + novelName: 'Example Novel', + novelId, + pluginId: 'source-a', + chapters, + }, + }; + const secondDownloadTask = { + name: 'DOWNLOAD_CHAPTER' as const, + data: { + novelName: 'Second Novel', + novelId: 8, + pluginId: 'source-a', + chapters: [ + { + chapterId: 102, + chapterName: 'Chapter 102', + novelId: 8, + }, + ], + }, + }; + const otherTask = { + name: 'UPDATE_LIBRARY' as const, + data: { categoryId: 4 }, + }; + mockedGetLibraryNovels.mockResolvedValue([ + novel(1, 'source-a', 'Example Novel'), + ]); + + let resolveUpdate!: () => void; + mockedUpdateNovel.mockImplementation( + async (_pluginId, _path, _novelId, options) => { + options.enqueue?.([downloadTask, secondDownloadTask, otherTask]); + await new Promise(resolve => { + resolveUpdate = resolve; + }); + }, + ); + const enqueue = jest.fn(); + + const updatePromise = updateLibrary({}, jest.fn(), enqueue); + await flushPromises(); + + expect(enqueue).toHaveBeenCalledTimes(4); + const enqueuedTasks = enqueue.mock.calls.map(([task]) => task); + const normalizedChapters = chapters.map(chapter => ({ + ...chapter, + novelId, + })); + expect(enqueuedTasks[0]).toEqual({ + ...downloadTask, + data: { + ...downloadTask.data, + chapters: normalizedChapters.slice(0, 100), + }, + }); + expect(enqueuedTasks[1]).toEqual({ + ...downloadTask, + data: { + ...downloadTask.data, + chapters: normalizedChapters.slice(100), + }, + }); + expect(enqueuedTasks[2]).toEqual(secondDownloadTask); + expect(enqueuedTasks[3]).toBe(otherTask); + + resolveUpdate(); + await updatePromise; + }); + + it('waits for every source worker before finalizing after an interruption', async () => { + mockedGetLibraryNovels.mockResolvedValue([ + novel(1, 'source-a', 'A One'), + novel(2, 'source-b', 'B One'), + ]); + let resolveSecondSource!: () => void; + const secondSource = new Promise(resolve => { + resolveSecondSource = resolve; + }); + mockedUpdateNovel.mockImplementation((_pluginId, _path, novelId) => + novelId === 2 ? secondSource : Promise.resolve(), + ); + + let metadata: BackgroundTaskMetadata = { + name: 'Update library', + isRunning: false, + progress: undefined, + progressText: undefined, + }; + let shouldInterrupt = true; + const setMeta = ( + transform: (meta: BackgroundTaskMetadata) => BackgroundTaskMetadata, + ) => { + const nextMetadata = transform(metadata); + if (shouldInterrupt && nextMetadata.progressText === 'A One') { + shouldInterrupt = false; + throw new Error('interrupted'); + } + metadata = nextMetadata; + }; + + const updatePromise = updateLibrary({}, setMeta, jest.fn()); + await flushPromises(); + + expect(metadata).toMatchObject({ + isRunning: true, + progress: 0, + }); + resolveSecondSource(); + await expect(updatePromise).rejects.toThrow('interrupted'); + expect(metadata).toMatchObject({ + isRunning: false, + progress: 1, + progressText: undefined, + }); + }); }); diff --git a/src/services/updates/index.ts b/src/services/updates/index.ts index 37a338fb33..626315cbfc 100644 --- a/src/services/updates/index.ts +++ b/src/services/updates/index.ts @@ -16,6 +16,7 @@ import { getGlobalUpdateCategoryFilters, } from '@hooks/persisted/useSettings'; import type { + BackgroundTask, BackgroundTaskEnqueuer, TaskProgressUpdater, } from '@services/backgroundTasks/contracts'; @@ -36,6 +37,42 @@ const groupNovelsByPlugin = (novels: DBNovelInfo[]) => { return [...groupedNovels.values()]; }; +const MAX_DOWNLOAD_CHAPTERS_PER_TASK = 100; + +const enqueueDownloadTasks = ( + enqueue: BackgroundTaskEnqueuer, + tasks: BackgroundTask | BackgroundTask[], +) => { + for (const task of Array.isArray(tasks) ? tasks : [tasks]) { + if (task.name !== 'DOWNLOAD_CHAPTER') { + enqueue(task); + continue; + } + + const chapters = task.data.chapters.map(chapter => + chapter.novelId === undefined && task.data.novelId !== undefined + ? { ...chapter, novelId: task.data.novelId } + : chapter, + ); + + for ( + let start = 0; + start < chapters.length; + start += MAX_DOWNLOAD_CHAPTERS_PER_TASK + ) { + enqueue({ + ...task, + data: { + ...task.data, + chapters: chapters.slice( + start, + start + MAX_DOWNLOAD_CHAPTERS_PER_TASK, + ), + }, + }); + } + } +}; const updateLibrary = async ( { @@ -67,79 +104,91 @@ const updateLibrary = async ( const options: UpdateNovelOptions = { downloadNewChapters: downloadNewChapters || false, refreshNovelMetadata: refreshNovelMetadata || false, - enqueue, + enqueue: tasks => enqueueDownloadTasks(enqueue, tasks), }; - let libraryNovels: DBNovelInfo[] = []; - if (categoryId) { - libraryNovels = await getLibraryWithCategory(categoryId, true); - } else { - libraryNovels = await getLibraryNovelsForGlobalUpdate( - getGlobalUpdateCategoryFilters(), - smartUpdateFilters, - ); - } + try { + let libraryNovels: DBNovelInfo[] = []; + if (categoryId) { + libraryNovels = await getLibraryWithCategory(categoryId, true); + } else { + libraryNovels = await getLibraryNovelsForGlobalUpdate( + getGlobalUpdateCategoryFilters(), + smartUpdateFilters, + ); + } - if (libraryNovels.length > 0) { - MMKVStorage.set(LAST_UPDATE_TIME, dayjs().format('YYYY-MM-DD HH:mm:ss')); - - const sourceQueues = groupNovelsByPlugin(libraryNovels); - const activeNovels = new Map(); - let completedNovels = 0; - let nextSourceQueue = 0; - - const publishProgress = () => { - setMeta(meta => ({ - ...meta, - progressText: [...activeNovels.values()].join('\n') || undefined, - progress: completedNovels / libraryNovels.length, - })); - }; - - const updateSourceQueue = async (sourceQueue: DBNovelInfo[]) => { - for (const novel of sourceQueue) { - activeNovels.set(novel.pluginId, novel.name); - publishProgress(); - - try { - await updateNovel(novel.pluginId, novel.path, novel.id, options); - await sleep(1000); - } catch (error: any) { - showToast(novel.name + ': ' + error.message); - } finally { - completedNovels += 1; - activeNovels.delete(novel.pluginId); + if (libraryNovels.length > 0) { + MMKVStorage.set(LAST_UPDATE_TIME, dayjs().format('YYYY-MM-DD HH:mm:ss')); + + const sourceQueues = groupNovelsByPlugin(libraryNovels); + const activeNovels = new Map(); + let completedNovels = 0; + let nextSourceQueue = 0; + + const publishProgress = () => { + setMeta(meta => ({ + ...meta, + progressText: [...activeNovels.values()].join('\n') || undefined, + progress: completedNovels / libraryNovels.length, + })); + }; + + const updateSourceQueue = async (sourceQueue: DBNovelInfo[]) => { + for (const novel of sourceQueue) { + activeNovels.set(novel.pluginId, novel.name); publishProgress(); + + try { + await updateNovel(novel.pluginId, novel.path, novel.id, options); + await sleep(1000); + } catch (error: unknown) { + showToast( + novel.name + + ': ' + + (error instanceof Error ? error.message : String(error)), + ); + } finally { + completedNovels += 1; + activeNovels.delete(novel.pluginId); + publishProgress(); + } } - } - }; + }; - const updateNextSource = async () => { - while (nextSourceQueue < sourceQueues.length) { - const sourceQueue = sourceQueues[nextSourceQueue]; - nextSourceQueue += 1; - await updateSourceQueue(sourceQueue); + const updateNextSource = async () => { + while (nextSourceQueue < sourceQueues.length) { + const sourceQueue = sourceQueues[nextSourceQueue]; + nextSourceQueue += 1; + await updateSourceQueue(sourceQueue); + } + }; + + const sourceResults = await Promise.allSettled( + Array.from( + { + length: Math.min(UPDATE_SOURCE_CONCURRENCY, sourceQueues.length), + }, + updateNextSource, + ), + ); + const rejectedSource = sourceResults.find( + result => result.status === 'rejected', + ); + if (rejectedSource?.status === 'rejected') { + throw rejectedSource.reason; } - }; - - await Promise.all( - Array.from( - { - length: Math.min(UPDATE_SOURCE_CONCURRENCY, sourceQueues.length), - }, - updateNextSource, - ), - ); - } else { - showToast("There's no novel to be updated"); + } else { + showToast("There's no novel to be updated"); + } + } finally { + setMeta(meta => ({ + ...meta, + progress: 1, + progressText: undefined, + isRunning: false, + })); } - - setMeta(meta => ({ - ...meta, - progress: 1, - progressText: undefined, - isRunning: false, - })); }; export { updateLibrary }; diff --git a/test/mocks/nativeModules.js b/test/mocks/nativeModules.js index e5f48124bf..a228365d9d 100644 --- a/test/mocks/nativeModules.js +++ b/test/mocks/nativeModules.js @@ -81,6 +81,7 @@ jest.mock('@modules/native-zip-archive', () => ({ __esModule: true, default: { zip: jest.fn().mockResolvedValue(), + zipDirectories: jest.fn().mockResolvedValue(), unzip: jest.fn().mockResolvedValue(), remoteUnzip: jest.fn().mockResolvedValue(), remoteZip: jest.fn().mockResolvedValue(''),